diff --git a/AGENTS.md b/AGENTS.md index a474e89..5720a72 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -34,9 +34,11 @@ command-line familiarity. included.** `all_paths` prunes directories whose name starts with `.`, so `.git` and `.bundle` are skipped, but a dot-*file* is not a directory and so gets packed: a top-level `.env` is uploaded, as is `data.sqlite` and anything - else the person happens to have sitting there. Keep that distinction in mind - before changing `all_paths`, and don't describe the behaviour as "skips - hidden files", because it doesn't. + else the person happens to have sitting there. `data.sqlite` is deliberate — + it's how a run continues from existing data — and is announced in the + "Uploading" line, with `--skip-data` to leave it out. Keep that distinction + in mind before changing `all_paths`, and don't describe the behaviour as + "skips hidden files", because it doesn't. - **The server streams newline-delimited JSON, not plain text.** Each line has a `stream` (`stdout`, `internalout` or `stderr`) and `text`, and `log` raises on any other stream value. Partial chunks are buffered on the newline, so diff --git a/README.md b/README.md index ce9631b..5f747c4 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,12 @@ Yup, that's it. It runs the code that's there right now. It doesn't need to be checked into git or anything. +If there's a `data.sqlite` in the directory it gets uploaded along with the +code, so the scraper run picks up from the data it already has. If it's big and +you'd rather not wait for the upload, leave it out with + + morph --skip-data + The first time you run it, it will ask for your morph.io API key, which it saves in `~/.morph`. @@ -38,8 +44,7 @@ For help It uploads your code every time. So if it's big it might take a little while. Scrapers are not usually so I'm hoping this won't really be an issue. -It doesn't yet return you the resulting sqlite database (or use the one you -might have locally). +It doesn't yet return you the resulting sqlite database. ## Development diff --git a/lib/morph-cli.rb b/lib/morph-cli.rb index ddadb4d..33f9797 100644 --- a/lib/morph-cli.rb +++ b/lib/morph-cli.rb @@ -11,7 +11,7 @@ require 'minitar' module MorphCLI - def self.execute(directory, _development, env_config) + def self.execute(directory, _development, env_config, skip_data: false) all_paths = MorphCLI.all_paths(directory) unless all_paths.find { |file| /scraper\.[\S]+$/ =~ file } @@ -19,19 +19,17 @@ def self.execute(directory, _development, env_config) exit(1) end + database_path = MorphCLI.database_path(directory) + if skip_data + all_paths.delete(database_path) + database_path = nil + end + size = MorphCLI.get_dir_size(directory, all_paths) - puts "Uploading #{size}..." + puts "Uploading #{size}#{" (including #{database_path})" if database_path}..." file = MorphCLI.create_tar(directory, all_paths) - timeout = if env_config.key?(:timeout) - env_config[:timeout] - else - 600 # 10 minutes should be "enough for everyone", right? - # Setting to nil will disable the timeout entirely. - # Default is 60 seconds. - end - connection = Faraday.new(url: env_config[:base_url]) do |f| f.request :multipart f.response :raise_error @@ -44,7 +42,10 @@ def self.execute(directory, _development, env_config) api_key: env_config[:api_key], code: Faraday::Multipart::FilePart.new(file, "application/octet-stream") } - req.options.timeout = timeout + # 10 minutes should be "enough for everyone", right? + # Setting :timeout to nil in the config will disable the timeout + # entirely. The Faraday default is 60 seconds. + req.options.timeout = env_config.fetch(:timeout, 600) req.options.on_data = proc do |chunk, _overall_received_bytes, env| next unless env.status == 200 diff --git a/lib/morph-cli/cli.rb b/lib/morph-cli/cli.rb index ad7b093..73070ce 100644 --- a/lib/morph-cli/cli.rb +++ b/lib/morph-cli/cli.rb @@ -14,6 +14,8 @@ def self.exit_on_failure? desc '[execute]', 'execute morph scraper' option :directory, default: Dir.getwd + option :skip_data, default: false, type: :boolean, + desc: "Don't upload the local data.sqlite database with the scraper" def execute config = MorphCLI.load_config @@ -28,7 +30,7 @@ def execute api_key_is_valid = false until api_key_is_valid begin - MorphCLI.execute(options[:directory], options[:dev], env_config) + MorphCLI.execute(options[:directory], options[:dev], env_config, skip_data: options[:skip_data]) api_key_is_valid = true rescue Faraday::UnauthorizedError puts "Your key isn't working. Let's try again." diff --git a/spec/morph_cli/cli_spec.rb b/spec/morph_cli/cli_spec.rb index 324e559..bc7dad5 100644 --- a/spec/morph_cli/cli_spec.rb +++ b/spec/morph_cli/cli_spec.rb @@ -30,7 +30,16 @@ described_class.start(['execute', '--directory', '/somewhere']) expect(MorphCLI).to have_received(:execute) - .with('/somewhere', false, config[:production]) + .with('/somewhere', false, config[:production], skip_data: false) + end + + it 'leaves the database out when --skip-data is given' do + allow(MorphCLI).to receive(:execute) + + described_class.start(['execute', '--skip-data']) + + expect(MorphCLI).to have_received(:execute) + .with(anything, false, config[:production], skip_data: true) end it 'runs the scraper with the development config when --dev is given' do @@ -39,7 +48,7 @@ described_class.start(['execute', '--dev']) expect(MorphCLI).to have_received(:execute) - .with(anything, true, config[:development]) + .with(anything, true, config[:development], skip_data: false) end it 'asks for an API key and saves the config when none is set' do @@ -54,7 +63,7 @@ expect(config[:production][:api_key]).to eq('shiny-new-key') expect(MorphCLI).to have_received(:save_config).with(config) expect(MorphCLI).to have_received(:execute) - .with(anything, false, config[:production]) + .with(anything, false, config[:production], skip_data: false) end it 'asks for a new API key and retries when the server rejects it' do diff --git a/spec/morph_cli_spec.rb b/spec/morph_cli_spec.rb index 66111de..c321103 100644 --- a/spec/morph_cli_spec.rb +++ b/spec/morph_cli_spec.rb @@ -45,6 +45,36 @@ def with_scraper_directory end) end + it "uploads the local database and says so" do + stub_request(:post, "https://morph.io/run").to_return(status: 200, body: "") + + with_scraper_directory do |dir| + File.write(File.join(dir, "data.sqlite"), "sqlite data") + + expect { described_class.execute(dir, false, env_config) } + .to output(/\AUploading 21\.00 B \(including data\.sqlite\)\.\.\.\n\z/).to_stdout + end + + expect(WebMock).to(have_requested(:post, "https://morph.io/run").with do |req| + req.body.include?("data.sqlite") + end) + end + + it "leaves the database out of the upload when skip_data is true" do + stub_request(:post, "https://morph.io/run").to_return(status: 200, body: "") + + with_scraper_directory do |dir| + File.write(File.join(dir, "data.sqlite"), "sqlite data") + + expect { described_class.execute(dir, false, env_config, skip_data: true) } + .to output(/\AUploading 10\.00 B\.\.\.\n\z/).to_stdout + end + + expect(WebMock).to(have_requested(:post, "https://morph.io/run").with do |req| + !req.body.include?("data.sqlite") + end) + end + it "raises Faraday::UnauthorizedError when the API key is rejected" do stub_request(:post, "https://morph.io/run").to_return(status: 401, body: "")