From d687a378142fa6d4df6a9e3cef472bbb051494bd Mon Sep 17 00:00:00 2001
From: Eddie A Tejeda <669988+eddietejeda@users.noreply.github.com>
Date: Sun, 9 Aug 2026 09:57:28 -0700
Subject: [PATCH] fix(ingest): continuous datasources are not validate_only
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

new-datasource unconditionally set validate_only=true ("discover schema, don't
load"), so --continuous produced a spec with BOTH continuous and validate_only.
That is the exact combination the worker now rejects (422): the scheduler
re-runs it every tick but the drain falls to full-replace, reloading the whole
bucket forever. It was the root cause of the otel_telemetry incident.

A continuous datasource is a persistent, self-loading source the scheduler keeps
synced — the opposite of a one-off schema preview. Set validate_only only when
NOT continuous. Without this, --continuous is unusable against a worker carrying
the guard (dlthubworker #193): every attempt 422s.
---
 src/commands/ingest.rs | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/src/commands/ingest.rs b/src/commands/ingest.rs
index 117b4e9..1da1d0e 100644
--- a/src/commands/ingest.rs
+++ b/src/commands/ingest.rs
@@ -828,8 +828,14 @@ fn build_create_request(
             }
         }
     };
-    // Adding a datasource discovers the schema only — never loads data.
-    req.validate_only = true;
+    // Adding a datasource discovers the schema only — never loads data — EXCEPT
+    // a continuous one, which is a persistent, self-loading datasource the
+    // scheduler keeps synced. Sending validate_only with continuous is
+    // contradictory (a one-off preview that is also permanently synced): the
+    // worker rejects the pair 422, and before it did, the datasource was
+    // re-run every tick but fell to the full-replace path — reloading the whole
+    // bucket forever. So continuous datasources are created ready to sync.
+    req.validate_only = !req.continuous;
     req.name = args.name;
     req.database_id = args.database_id;
     Ok(req)
@@ -1896,6 +1902,9 @@ mod tests {
         assert_eq!(req.family, "filesystem");
         assert_eq!(req.bucket_url.as_deref(), Some("s3://b/prefix"));
         assert!(req.continuous); // --continuous rides through to the request body
+        // A continuous datasource is self-loading, so it is NOT validate_only —
+        // the worker 422s the pair, and it was the original full-replace bug.
+        assert!(!req.validate_only);
 
         // Default is off, and it serializes only when true (skip_serializing_if).
         let mut off = create_args();
@@ -1903,6 +1912,7 @@ mod tests {
         off.format = Some("jsonl".into());
         let req_off = build_create_request(&e, off, None).unwrap();
         assert!(!req_off.continuous);
+        assert!(req_off.validate_only); // a non-continuous add still discovers schema only
         assert!(
             !serde_json::to_string(&req_off)
                 .unwrap()