class AIPP::Downloader
AIP
downloader infrastructure
The downloader operates in the storage
directory where it creates two subdirectories “sources” and “work”. The initializer looks for the source
archive in “sources” and (if found) unzips its contents into “work”. When reading a document
, the downloader looks for the document
in “work” and (unless found) downloads it from url
. HTML documents are parsed to Nokogiri::HTML5::Document
, PDF
documents are parsed to AIPP::PDF
. Finally, the contents of “work” are written back to the source
archive.
@example
AIPP::Downloader.new(storage: options[:storage], source: "2018-11-08") do |downloader| html = downloader.read( document: 'ENR-5.1', url: 'https://www.sia.aviation-civile.gouv.fr/dvd/eAIP_08_NOV_2018/FRANCE/AIRAC-2018-11-08/html/eAIP/FR-ENR-5.1-fr-FR.html' ) pdf = downloader.read( document: 'VAC-LFMV', url: 'https://www.sia.aviation-civile.gouv.fr/dvd/eAIP_08_NOV_2018/Atlas-VAC/PDF_AIPparSSection/VAC/AD/AD-2.LFMV.pdf' ) end
Attributes
@return [String] name of the source archive (without extension “.zip”)
@return [Pathname] full path to the source archive
@return [Pathname] directory to operate within
Public Class Methods
@param storage [Pathname] directory to operate within @param source [String] name of the source archive (without extension “.zip”)
# File lib/aipp/downloader.rb 37 def initialize(storage:, source:) 38 @storage, @source = storage, source 39 fail(ArgumentError, 'bad storage directory') unless Dir.exist? storage 40 @source_file = sources_path.join("#{@source}.zip") 41 prepare 42 unzip if @source_file.exist? 43 yield self 44 zip 45 ensure 46 teardown 47 end
Public Instance Methods
Download and read document
@param document [String] document to read (without extension) @param url [String] URL to download the document from @param type [Symbol, nil] document type: nil
(default) to derive it from
the URL, :html, or :pdf
@return [Nokogiri::HTML5::Document, AIPP::PDF]
# File lib/aipp/downloader.rb 56 def read(document:, url:, type: nil) 57 type ||= Pathname(URI(url).path).extname[1..-1].to_sym 58 file = work_path.join([document, type].join('.')) 59 unless file.exist? 60 verbose_info "Downloading #{document}" 61 IO.copy_stream(URI.open(url), file) 62 end 63 convert file 64 end
Private Instance Methods
# File lib/aipp/downloader.rb 106 def convert(file) 107 case file.extname 108 when '.html' then Nokogiri.HTML5(file) 109 when '.pdf' then AIPP::PDF.new(file) 110 else 111 fail(ArgumentError, "invalid document type") 112 end 113 end
# File lib/aipp/downloader.rb 76 def prepare 77 teardown 78 sources_path.mkpath 79 work_path.mkpath 80 end
# File lib/aipp/downloader.rb 68 def sources_path 69 @storage.join('sources') 70 end
# File lib/aipp/downloader.rb 82 def teardown 83 if work_path.exist? 84 work_path.children.each(&:delete) 85 work_path.delete 86 end 87 end
# File lib/aipp/downloader.rb 89 def unzip 90 Zip::File.open(source_file).each do |entry| 91 entry.extract(work_path.join(entry.name)) 92 end 93 end
# File lib/aipp/downloader.rb 72 def work_path 73 @storage.join('work') 74 end
# File lib/aipp/downloader.rb 95 def zip 96 backup_file = source_file.sub(/$/, '.old') if source_file.exist? 97 source_file.rename(backup_file) if backup_file 98 Zip::File.open(source_file, Zip::File::CREATE) do |zip| 99 work_path.children.each do |entry| 100 zip.add(entry.basename.to_s, entry) unless entry.basename.to_s[0] == '.' 101 end 102 end 103 backup_file&.delete 104 end