feat(ingest): --continuous flag on new-datasource (buckets)

skills/hotdata/SKILL.md

@@ -256,6 +256,8 @@ hotdata ingest new-datasource --service buckets --bucket-url s3://bucket/prefix
 # Files in S3/GCS/Azure buckets (csv, jsonl, parquet); --glob narrows the match.
 # Public buckets need no credentials; private ones take --config @creds.json
 # ({"aws_access_key_id": …, "aws_secret_access_key": …, "endpoint_url": …}).
+# --continuous keeps a bucket datasource synced: it's re-run incrementally on a
+# schedule, appending only newly-arrived objects (no re-read of the whole bucket).
 
 hotdata ingest new-datasource --service iceberg --config @catalog.json --table ns.orders
 # Iceberg via a REST catalog. --table is REQUIRED (repeatable, namespace.table).

src/client/ingest.rs

@@ -428,6 +428,10 @@ pub struct IngestRequest {
     pub tables: Vec<String>,
     #[serde(skip_serializing_if = "std::ops::Not::not")]
     pub validate_only: bool,
+    /// filesystem only: keep this datasource continuously synced — the scheduler
+    /// re-runs it incrementally (append only new objects). Ignored otherwise.
+    #[serde(skip_serializing_if = "std::ops::Not::not")]
+    pub continuous: bool,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub database_id: Option<String>,
 }

src/commands/ingest.rs

@@ -693,6 +693,11 @@ pub struct CreateArgs {
     #[arg(long)]
     glob: Option<String>,
 
+    /// Keep this datasource continuously synced — refreshed incrementally on a
+    /// schedule, appending only newly-arrived objects (buckets only)
+    #[arg(long)]
+    continuous: bool,
+
     /// Catalog type, e.g. rest (iceberg)
     #[arg(long = "catalog-type")]
     catalog_type: Option<String>,
@@ -713,6 +718,7 @@ impl CreateArgs {
             || self.bucket_url.is_some()
             || self.format.is_some()
             || self.glob.is_some()
+            || self.continuous
             || self.catalog_type.is_some()
             || self.database_id.is_some()
     }
@@ -779,6 +785,7 @@ fn build_create_request(
             bucket_url: Some(args.bucket_url.ok_or("buckets connectors need --bucket-url")?),
             file_glob: args.glob,
             file_format: args.format,
+            continuous: args.continuous,
             ..Default::default()
         },
         "iceberg" => IngestRequest {
@@ -1837,6 +1844,7 @@ mod tests {
             bucket_url: None,
             format: None,
             glob: None,
+            continuous: false,
             catalog_type: None,
             database_id: None,
         }
@@ -1869,6 +1877,31 @@ mod tests {
         assert_eq!(req.database_id.as_deref(), Some("db_1"));
     }
 
+    #[test]
+    fn create_request_filesystem_carries_continuous_flag() {
+        let e = entry("buckets", "filesystem");
+        let mut args = create_args();
+        args.bucket_url = Some("s3://b/prefix".into());
+        args.format = Some("jsonl".into());
+        args.continuous = true;
+        let req = build_create_request(&e, args, None).unwrap();
+        assert_eq!(req.family, "filesystem");
+        assert_eq!(req.bucket_url.as_deref(), Some("s3://b/prefix"));
+        assert!(req.continuous); // --continuous rides through to the request body
+
+        // Default is off, and it serializes only when true (skip_serializing_if).
+        let mut off = create_args();
+        off.bucket_url = Some("s3://b".into());
+        off.format = Some("jsonl".into());
+        let req_off = build_create_request(&e, off, None).unwrap();
+        assert!(!req_off.continuous);
+        assert!(
+            !serde_json::to_string(&req_off)
+                .unwrap()
+                .contains("continuous")
+        );
+    }
+
     #[test]
     fn create_request_rejects_invalid_names() {
         let e = entry("postgres", "sql");