class WonderScrape::Recorder
Attributes
item_issues[RW]
items_scraped[R]
output[R]
progress_bar[R]
unexpected_fields[RW]
verbose[R]
Public Class Methods
new(output, options = {})
click to toggle source
# File lib/wonder_scrape/recorder.rb, line 6 def initialize(output, options = {}) @output = output @verbose = options[:verbose] || false @progress_bar = options[:progress_bar] @items_scraped = 0 @item_issues = {} @unexpected_fields = [] end
Public Instance Methods
increment_items_scraped(item)
click to toggle source
# File lib/wonder_scrape/recorder.rb, line 28 def increment_items_scraped(item) @items_scraped += 1 if verbose output.puts JSON.pretty_generate(item) else progress_bar&.advance(1) end end
print()
click to toggle source
# File lib/wonder_scrape/recorder.rb, line 15 def print output.puts "Successfully processed #{items_scraped} items!" if unexpected_fields.count > 0 output.puts "Encountered the following unexpected fields: #{unexpected_fields}" end if item_issues.count > 0 output.puts "Had issues with #{item_issues.count} items below" output.puts JSON.pretty_generate(item_issues) end end
record_unexpected_field(item_id, field_name)
click to toggle source
# File lib/wonder_scrape/recorder.rb, line 37 def record_unexpected_field(item_id, field_name) item_issues[item_id] ||= [] item_issues[item_id] << "Unexpected field: #{field_name}" unexpected_fields << field_name end