Make data.sqlite upload explicit and optional

AGENTS.md

@@ -34,9 +34,11 @@ command-line familiarity.
   included.** `all_paths` prunes directories whose name starts with `.`, so
   `.git` and `.bundle` are skipped, but a dot-*file* is not a directory and so
   gets packed: a top-level `.env` is uploaded, as is `data.sqlite` and anything
-  else the person happens to have sitting there. Keep that distinction in mind
-  before changing `all_paths`, and don't describe the behaviour as "skips
-  hidden files", because it doesn't.
+  else the person happens to have sitting there. `data.sqlite` is deliberate —
+  it's how a run continues from existing data — and is announced in the
+  "Uploading" line, with `--skip-data` to leave it out. Keep that distinction
+  in mind before changing `all_paths`, and don't describe the behaviour as
+  "skips hidden files", because it doesn't.
 - **The server streams newline-delimited JSON, not plain text.** Each line has
   a `stream` (`stdout`, `internalout` or `stderr`) and `text`, and `log` raises
   on any other stream value. Partial chunks are buffered on the newline, so

README.md

@@ -26,6 +26,12 @@ Yup, that's it.
 It runs the code that's there right now. It doesn't need to be checked into git
 or anything.
 
+If there's a `data.sqlite` in the directory it gets uploaded along with the
+code, so the scraper run picks up from the data it already has. If it's big and
+you'd rather not wait for the upload, leave it out with
+
+    morph --skip-data
+
 The first time you run it, it will ask for your morph.io API key, which it
 saves in `~/.morph`.
 
@@ -38,8 +44,7 @@ For help
 It uploads your code every time. So if it's big it might take a little while.
 Scrapers are not usually so I'm hoping this won't really be an issue.
 
-It doesn't yet return you the resulting sqlite database (or use the one you
-might have locally).
+It doesn't yet return you the resulting sqlite database.
 
 ## Development
 

lib/morph-cli.rb

@@ -11,27 +11,25 @@
 require 'minitar'
 
 module MorphCLI
-  def self.execute(directory, _development, env_config)
+  def self.execute(directory, _development, env_config, skip_data: false)
     all_paths = MorphCLI.all_paths(directory)
 
     unless all_paths.find { |file| /scraper\.[\S]+$/ =~ file }
       warn "Can't find scraper to upload. Expected to find a file called scraper.rb, scraper.php, scraper.py, scraper.pl, scraper.js, etc to upload"
       exit(1)
     end
 
+    database_path = MorphCLI.database_path(directory)
+    if skip_data
+      all_paths.delete(database_path)
+      database_path = nil
+    end
+
     size = MorphCLI.get_dir_size(directory, all_paths)
-    puts "Uploading #{size}..."
+    puts "Uploading #{size}#{" (including #{database_path})" if database_path}..."
 
     file = MorphCLI.create_tar(directory, all_paths)
 
-    timeout = if env_config.key?(:timeout)
-                env_config[:timeout]
-              else
-                600 # 10 minutes should be "enough for everyone", right?
-                # Setting to nil will disable the timeout entirely.
-                # Default is 60 seconds.
-              end
-
     connection = Faraday.new(url: env_config[:base_url]) do |f|
       f.request :multipart
       f.response :raise_error
@@ -44,7 +42,10 @@ def self.execute(directory, _development, env_config)
         api_key: env_config[:api_key],
         code: Faraday::Multipart::FilePart.new(file, "application/octet-stream")
       }
-      req.options.timeout = timeout
+      # 10 minutes should be "enough for everyone", right?
+      # Setting :timeout to nil in the config will disable the timeout
+      # entirely. The Faraday default is 60 seconds.
+      req.options.timeout = env_config.fetch(:timeout, 600)
       req.options.on_data = proc do |chunk, _overall_received_bytes, env|
         next unless env.status == 200
 

lib/morph-cli/cli.rb

@@ -14,6 +14,8 @@ def self.exit_on_failure?
 
     desc '[execute]', 'execute morph scraper'
     option :directory, default: Dir.getwd
+    option :skip_data, default: false, type: :boolean,
+                       desc: "Don't upload the local data.sqlite database with the scraper"
 
     def execute
       config = MorphCLI.load_config
@@ -28,7 +30,7 @@ def execute
       api_key_is_valid = false
       until api_key_is_valid
         begin
-          MorphCLI.execute(options[:directory], options[:dev], env_config)
+          MorphCLI.execute(options[:directory], options[:dev], env_config, skip_data: options[:skip_data])
           api_key_is_valid = true
         rescue Faraday::UnauthorizedError
           puts "Your key isn't working. Let's try again."

spec/morph_cli/cli_spec.rb

@@ -30,7 +30,16 @@
       described_class.start(['execute', '--directory', '/somewhere'])
 
       expect(MorphCLI).to have_received(:execute)
-        .with('/somewhere', false, config[:production])
+        .with('/somewhere', false, config[:production], skip_data: false)
+    end
+
+    it 'leaves the database out when --skip-data is given' do
+      allow(MorphCLI).to receive(:execute)
+
+      described_class.start(['execute', '--skip-data'])
+
+      expect(MorphCLI).to have_received(:execute)
+        .with(anything, false, config[:production], skip_data: true)
     end
 
     it 'runs the scraper with the development config when --dev is given' do
@@ -39,7 +48,7 @@
       described_class.start(['execute', '--dev'])
 
       expect(MorphCLI).to have_received(:execute)
-        .with(anything, true, config[:development])
+        .with(anything, true, config[:development], skip_data: false)
     end
 
     it 'asks for an API key and saves the config when none is set' do
@@ -54,7 +63,7 @@
       expect(config[:production][:api_key]).to eq('shiny-new-key')
       expect(MorphCLI).to have_received(:save_config).with(config)
       expect(MorphCLI).to have_received(:execute)
-        .with(anything, false, config[:production])
+        .with(anything, false, config[:production], skip_data: false)
     end
 
     it 'asks for a new API key and retries when the server rejects it' do

spec/morph_cli_spec.rb

@@ -45,6 +45,36 @@ def with_scraper_directory
       end)
     end
 
+    it "uploads the local database and says so" do
+      stub_request(:post, "https://morph.io/run").to_return(status: 200, body: "")
+
+      with_scraper_directory do |dir|
+        File.write(File.join(dir, "data.sqlite"), "sqlite data")
+
+        expect { described_class.execute(dir, false, env_config) }
+          .to output(/\AUploading 21\.00 B \(including data\.sqlite\)\.\.\.\n\z/).to_stdout
+      end
+
+      expect(WebMock).to(have_requested(:post, "https://morph.io/run").with do |req|
+        req.body.include?("data.sqlite")
+      end)
+    end
+
+    it "leaves the database out of the upload when skip_data is true" do
+      stub_request(:post, "https://morph.io/run").to_return(status: 200, body: "")
+
+      with_scraper_directory do |dir|
+        File.write(File.join(dir, "data.sqlite"), "sqlite data")
+
+        expect { described_class.execute(dir, false, env_config, skip_data: true) }
+          .to output(/\AUploading 10\.00 B\.\.\.\n\z/).to_stdout
+      end
+
+      expect(WebMock).to(have_requested(:post, "https://morph.io/run").with do |req|
+        !req.body.include?("data.sqlite")
+      end)
+    end
+
     it "raises Faraday::UnauthorizedError when the API key is rejected" do
       stub_request(:post, "https://morph.io/run").to_return(status: 401, body: "")