fix(gateway): collect the tombstones the cluster has finished with

CHANGELOG.md

@@ -19,6 +19,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - http-client: a caller can bound the response body (`http_request_bounded`, `PrpcClient::with_max_response_bytes`). Nothing is bounded by default — `dstack vmm logs --lines 100000` is a legitimate multi-megabyte fetch — but every client that talks to a guest agent opts in, in the gateway and in the VMM, because a CVM is untrusted and one of them polls on a timer against the whole fleet
 
 ### Fixed
+- gateway: deleted KV records left a tombstone that nothing ever collected, so every deregistered CVM stayed on disk for the life of the deployment. Tombstones every peer has acknowledged are now dropped once every `tombstone_gc_writes` replicated writes (default 10000, zero disables); the trigger counts replicated writes rather than reading a clock, so nodes in a cluster collect in the same window without depending on time synchronization. A `SetTombstoneGcConfig` admin RPC stores an operator override in the KV itself, replicating one pace to every node
 - gateway: `Admin.RemoveCvm` now reports the outcome of the `inst/` tombstone alone. A failure to delete associated override or telemetry records is logged instead of failing the call, so a removal that did take effect is no longer reported as failed — which also aborted the local routing cleanup that follows it. Re-issuing a removal still sweeps up override and telemetry records orphaned by an earlier partial failure
 - sdk: `get_compose_hash` in the Python SDK mutated the dictionary it was given, stripping `docker_config` and `requirements` from it, so hashing the same manifest twice returned two different digests -- the second one for a manifest missing those blocks
 - sdk: the Go compose-hash helper HTML-escaped `<`, `>` and `&`, so an app-compose carrying an `os_version` bound such as `">=0.6.1"` hashed differently in Go than in Rust, Python and JavaScript. Any digest Go produced for such a manifest was wrong, and that digest is what gets whitelisted on chain

dstack/gateway/gateway.toml

@@ -215,3 +215,12 @@ wal_sync_window = "5s"
 sync_connections_enabled = true
 # Interval for syncing instance connections to KV store
 sync_connections_interval = "30s"
+# Collect tombstones every peer has already acknowledged once every this many
+# replicated writes, per store. Zero keeps every delete on disk for the life of
+# the deployment. The trigger counts replicated writes rather than reading a
+# clock so that every node in a cluster collects in the same window -- the
+# state digest covers tombstones, so nodes collecting on their own phase
+# report a mismatch and pay a full re-exchange. An operator override stored in
+# the KV itself (SetTombstoneGcConfig) replicates everywhere and takes
+# precedence over this per-node default.
+tombstone_gc_writes = 10000

dstack/gateway/rpc/proto/gateway_rpc.proto

@@ -547,6 +547,14 @@ service Admin {
   // to the new account; it does not deactivate the old ACME account at the CA.
   rpc RotateAcmeCredentials(google.protobuf.Empty) returns (RotateAcmeCredentialsResponse) {}
 
+  // ==================== Tombstone GC Configuration ====================
+  // Get the effective tombstone GC pacing on this node
+  rpc GetTombstoneGcConfig(google.protobuf.Empty) returns (TombstoneGcConfigResponse) {}
+  // Store the tombstone GC pacing in the replicated KV, overriding every
+  // node's config-file default so the whole cluster collects on the same
+  // write-count boundaries. Zero disables collection cluster-wide.
+  rpc SetTombstoneGcConfig(SetTombstoneGcConfigRequest) returns (google.protobuf.Empty) {}
+
   // ==================== Per-Instance Port Policy Override ====================
   // Set an admin override for an instance's port policy. Takes precedence
   // over the policy reported by the instance itself, and survives app
@@ -834,6 +842,23 @@ message SetCertbotConfigRequest {
   optional string acme_url = 4;
 }
 
+// ==================== Tombstone GC Configuration Messages ====================
+
+// Tombstone GC configuration response
+message TombstoneGcConfigResponse {
+  // Collect tombstones once every this many replicated writes (0 = disabled)
+  uint64 writes_per_collection = 1;
+  // True when the value is the replicated KV override; false when it is this
+  // node's config-file default (which other nodes may not share)
+  bool stored_in_kv = 2;
+}
+
+// Set tombstone GC configuration request
+message SetTombstoneGcConfigRequest {
+  // Collect tombstones once every this many replicated writes (0 = disabled)
+  uint64 writes_per_collection = 1;
+}
+
 // ==================== Per-Instance Port Policy Override Messages ====================
 
 // Set an admin override for an instance.

dstack/gateway/src/admin_service.rs

@@ -21,18 +21,18 @@ use dstack_gateway_rpc::{
     RemoveCvmResponse, RemoveNodeRequest, RemoveNodeResponse, RenewCertResponse,
     RenewZtDomainCertRequest, RenewZtDomainCertResponse, RotateAcmeCredentialsResponse,
     SetCertbotConfigRequest, SetDefaultDnsCredentialRequest, SetInstancePortPolicyRequest,
-    SetInstanceReadyRequest, SetNodeStatusRequest, SetNodeUrlRequest, StatusResponse,
-    StoreSyncStatus, UpdateDnsCredentialRequest, WaveKvStatusResponse, ZtDomainCertStatus,
-    ZtDomainConfig as ProtoZtDomainConfig, ZtDomainInfo,
+    SetInstanceReadyRequest, SetNodeStatusRequest, SetNodeUrlRequest, SetTombstoneGcConfigRequest,
+    StatusResponse, StoreSyncStatus, TombstoneGcConfigResponse, UpdateDnsCredentialRequest,
+    WaveKvStatusResponse, ZtDomainCertStatus, ZtDomainConfig as ProtoZtDomainConfig, ZtDomainInfo,
 };
 use ra_rpc::{CallContext, RpcCall};
 use tracing::{info, warn};
 use wavekv::node::NodeStatus as WaveKvNodeStatus;
 
 use crate::{
     kv::{
-        import::Rejection, DnsCredential, DnsProvider, GlobalCertbotConfig, NodeStatus, PortFlags,
-        PortPolicy, ZtDomainConfig,
+        import::Rejection, DnsCredential, DnsProvider, GlobalCertbotConfig,
+        GlobalTombstoneGcConfig, NodeStatus, PortFlags, PortPolicy, ZtDomainConfig,
     },
     main_service::Proxy,
     models::PortPolicyView,
@@ -718,6 +718,33 @@ impl AdminRpc for AdminRpcHandler {
         Ok(())
     }
 
+    // ==================== Tombstone GC Configuration ====================
+
+    async fn get_tombstone_gc_config(self) -> Result<TombstoneGcConfigResponse> {
+        Ok(match self.state.kv_store().get_tombstone_gc_config()? {
+            Some(stored) => TombstoneGcConfigResponse {
+                writes_per_collection: stored.writes_per_collection,
+                stored_in_kv: true,
+            },
+            None => TombstoneGcConfigResponse {
+                writes_per_collection: self.state.config.sync.tombstone_gc_writes,
+                stored_in_kv: false,
+            },
+        })
+    }
+
+    async fn set_tombstone_gc_config(self, request: SetTombstoneGcConfigRequest) -> Result<()> {
+        let config = GlobalTombstoneGcConfig {
+            writes_per_collection: request.writes_per_collection,
+        };
+        self.state.kv_store().set_tombstone_gc_config(&config)?;
+        info!(
+            "updated tombstone GC config: writes_per_collection={}",
+            config.writes_per_collection
+        );
+        Ok(())
+    }
+
     async fn set_instance_port_policy(self, request: SetInstancePortPolicyRequest) -> Result<()> {
         let proto = request.policy.context("port policy is required")?;
         let policy = port_policy_from_proto(proto)?;

dstack/gateway/src/config.rs

@@ -149,6 +149,15 @@ fn default_true() -> bool {
     true
 }
 
+/// Roughly hourly on a busy ephemeral store and rarer on the quiet persistent
+/// one -- the cadence follows write volume, which is also what produces
+/// tombstones. Any value works for correctness: the ack watermark decides what
+/// may go, not this, so it trades disk against how often every node stops to
+/// scan its data map.
+fn default_tombstone_gc_writes() -> u64 {
+    10_000
+}
+
 fn default_handshake_stale() -> Duration {
     Duration::from_secs(30 * 60)
 }
@@ -590,6 +599,18 @@ pub struct SyncConfig {
     /// zero turns the work off -- which is why this one is a window.
     #[serde(with = "serde_duration")]
     pub wal_sync_window: Duration,
+    /// Collect tombstones once every this many replicated writes, per store.
+    /// Zero disables collection, which keeps every delete on disk for the life
+    /// of the deployment.
+    ///
+    /// The trigger is a count of replicated writes rather than a clock so that
+    /// every node in a cluster collects in the same window without agreeing on
+    /// the time: see `start_tombstone_gc_task`. This value is only the default
+    /// -- an operator override stored in the KV itself (`SetTombstoneGcConfig`)
+    /// replicates to every node and takes precedence, which is also the only
+    /// way to keep the boundaries identical when config files drift.
+    #[serde(default = "default_tombstone_gc_writes")]
+    pub tombstone_gc_writes: u64,
     /// Enable periodic sync of instance connections to KV store
     pub sync_connections_enabled: bool,
     /// Interval for syncing instance connections to KV store