diff --git a/README.md b/README.md index f7fef97..009966c 100644 --- a/README.md +++ b/README.md @@ -62,8 +62,11 @@ dw doctor Data persists in `dev-server` under the CLI config directory. Use `--data-dir=./workflow-data` to choose a directory or `--port=9000` to choose a -port. Ctrl+C stops the runtime and preserves its data. Application workers -still run your workflow and activity code through an SDK. +port. Ctrl+C stops the runtime and preserves its data. When a newer `dw` +finds data written by an earlier one, it upgrades that development database +in place and prints the schema versions; an older `dw` refuses data written by +a newer one. Application workers still run your workflow and activity code +through an SDK. ## Connect diff --git a/rust/Cargo.lock b/rust/Cargo.lock index c7c7089..834d660 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -56,7 +56,7 @@ dependencies = [ [[package]] name = "apache-avro" version = "0.22.0" -source = "git+https://github.com/durable-workflow/server?rev=3d90086f73533ac4437735bae6b9d0446558577c#3d90086f73533ac4437735bae6b9d0446558577c" +source = "git+https://github.com/durable-workflow/server?rev=da655a8500691d6ce8e018d1552801491a696135#da655a8500691d6ce8e018d1552801491a696135" dependencies = [ "bigdecimal", "bon", @@ -487,7 +487,7 @@ dependencies = [ [[package]] name = "durable-workflow-server" version = "0.0.0" -source = "git+https://github.com/durable-workflow/server?rev=3d90086f73533ac4437735bae6b9d0446558577c#3d90086f73533ac4437735bae6b9d0446558577c" +source = "git+https://github.com/durable-workflow/server?rev=da655a8500691d6ce8e018d1552801491a696135#da655a8500691d6ce8e018d1552801491a696135" dependencies = [ "apache-avro 0.22.0", "axum", @@ -525,6 +525,7 @@ dependencies = [ "durable-workflow-server", "getrandom 0.2.17", "libc", + "miniz_oxide 0.9.1", "rcgen", "reqwest", "rustls", @@ -2016,7 +2017,7 @@ dependencies = [ [[package]] name = "sqlx-mysql" version = "0.9.0" -source = "git+https://github.com/durable-workflow/server?rev=3d90086f73533ac4437735bae6b9d0446558577c#3d90086f73533ac4437735bae6b9d0446558577c" +source = "git+https://github.com/durable-workflow/server?rev=da655a8500691d6ce8e018d1552801491a696135#da655a8500691d6ce8e018d1552801491a696135" dependencies = [ "bitflags", "byteorder", @@ -2042,7 +2043,7 @@ dependencies = [ [[package]] name = "sqlx-postgres" version = "0.9.0" -source = "git+https://github.com/durable-workflow/server?rev=3d90086f73533ac4437735bae6b9d0446558577c#3d90086f73533ac4437735bae6b9d0446558577c" +source = "git+https://github.com/durable-workflow/server?rev=da655a8500691d6ce8e018d1552801491a696135#da655a8500691d6ce8e018d1552801491a696135" dependencies = [ "atoi", "base64 0.22.1", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index f9ee5f5..20d4fab 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -15,7 +15,7 @@ path = "src/main.rs" [dependencies] # Native development Server, built into every standalone dw binary. Keep its # dependency patches on this exact source revision as well. -durable-workflow-server = { git = "https://github.com/durable-workflow/server", rev = "3d90086f73533ac4437735bae6b9d0446558577c" } +durable-workflow-server = { git = "https://github.com/durable-workflow/server", rev = "da655a8500691d6ce8e018d1552801491a696135" } axum = { version = "=0.8.9", default-features = false, features = ["http1", "tokio"] } # Published 4.18.0 carries sdk-rust#279: without its default msrv-1-86 # feature the SDK composes with the native Server's newer Avro dependency. @@ -34,6 +34,9 @@ tokio = { version = "1", features = ["rt", "rt-multi-thread", "macros", "net", " reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] } [dev-dependencies] +# Inflates the compressed development database fixtures; already in the +# graph through the native Server. +miniz_oxide = "0.9" # A self-signed HTTPS Server for the --tls-verify tests, on the ring provider # reqwest already uses. rcgen = { version = "0.13", default-features = false, features = ["ring", "pem"] } @@ -56,9 +59,9 @@ strip = "symbols" # Cargo only applies patches from the top-level manifest. These are the # reviewed Server dependency sources, including its Avro and SQL adapters. [patch.crates-io] -apache-avro = { git = "https://github.com/durable-workflow/server", rev = "3d90086f73533ac4437735bae6b9d0446558577c" } -sqlx-mysql = { git = "https://github.com/durable-workflow/server", rev = "3d90086f73533ac4437735bae6b9d0446558577c" } -sqlx-postgres = { git = "https://github.com/durable-workflow/server", rev = "3d90086f73533ac4437735bae6b9d0446558577c" } +apache-avro = { git = "https://github.com/durable-workflow/server", rev = "da655a8500691d6ce8e018d1552801491a696135" } +sqlx-mysql = { git = "https://github.com/durable-workflow/server", rev = "da655a8500691d6ce8e018d1552801491a696135" } +sqlx-postgres = { git = "https://github.com/durable-workflow/server", rev = "da655a8500691d6ce8e018d1552801491a696135" } [profile.dev] debug = 0 diff --git a/rust/src/commands/start_dev.rs b/rust/src/commands/start_dev.rs index d00511a..b3681cb 100644 --- a/rust/src/commands/start_dev.rs +++ b/rust/src/commands/start_dev.rs @@ -4,12 +4,14 @@ use super::start_dev_data::Data; use crate::cli::Input; use crate::error::CliError; use crate::{config, formatter}; -use durable_workflow_server::runtime::{router, Runtime}; +use durable_workflow_server::runtime::{router, schema_upgrade, Runtime, RuntimeError}; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use std::path::PathBuf; use std::time::Duration; const DEVELOPMENT_TOKEN: &str = "dev-token"; +// Relative to the held data directory, which `Data::enter` made current. +const DATABASE: &str = "database.sqlite"; const SHUTDOWN_GRACE: Duration = Duration::from_secs(5); struct Settings { @@ -107,6 +109,49 @@ async fn stopped() -> i32 { } } +/// Open the development database. Native startup never upgrades a schema, +/// but this Server holds its own data directory exclusively, so an earlier +/// native schema found there is upgraded in place with the Server's own +/// SQLite upgrade before the runtime opens it. A newer schema still refuses. +async fn open(data: &Data, token: &str) -> Result { + match Runtime::open(DATABASE, token.to_owned()).await { + Err(RuntimeError::Refused { + reason: "native_schema_upgrade_required", + .. + }) => {} + result => return result.map_err(open_error), + } + data.verify().map_err(data_error)?; + let report = schema_upgrade::upgrade_sqlite(DATABASE) + .await + .map_err(|error| { + CliError::Failure(format!("Cannot upgrade the development database: {error}")) + })?; + data.verify().map_err(data_error)?; + if report["status"] == "upgraded" { + formatter::writeln_out(&format!( + "Upgraded the development database from schema {} to {}.", + report["from_schema_version"], report["schema_version"] + )); + } + Runtime::open(DATABASE, token.to_owned()) + .await + .map_err(open_error) +} + +fn open_error(error: RuntimeError) -> CliError { + match error { + RuntimeError::Refused { + reason: reason @ "native_schema_newer_than_binary", + .. + } => CliError::Failure(format!( + "Cannot open development database: it was written by a newer dw ({reason}). \ + Start it with that newer dw, or move the data directory aside or choose another --data-dir to start fresh." + )), + error => CliError::Failure(format!("Cannot open development database: {error}")), + } +} + async fn serve(settings: Settings, mut data: Data) -> Result { let shutdown = stopped(); tokio::pin!(shutdown); @@ -125,7 +170,7 @@ async fn serve(settings: Settings, mut data: Data) -> Result { .map_err(|error| CliError::Failure(error.to_string()))?; data.prepare_database().map_err(data_error)?; let runtime = tokio::select! { - result = Runtime::open("database.sqlite", settings.token.clone()) => result.map_err(|error| CliError::Failure(format!("Cannot open development database: {error}")))?, + result = open(&data, &settings.token) => result?, code = &mut shutdown => return Ok(code), }; if let Err(error) = data.verify() { diff --git a/rust/tests/fixtures/dev-server-cli-2.9.0-schema-14.sqlite.zlib b/rust/tests/fixtures/dev-server-cli-2.9.0-schema-14.sqlite.zlib new file mode 100644 index 0000000..94638ff Binary files /dev/null and b/rust/tests/fixtures/dev-server-cli-2.9.0-schema-14.sqlite.zlib differ diff --git a/rust/tests/fixtures/dev-server-schema-99.sqlite.zlib b/rust/tests/fixtures/dev-server-schema-99.sqlite.zlib new file mode 100644 index 0000000..5ffa909 Binary files /dev/null and b/rust/tests/fixtures/dev-server-schema-99.sqlite.zlib differ diff --git a/rust/tests/start_dev.rs b/rust/tests/start_dev.rs index cbc371f..3954c47 100644 --- a/rust/tests/start_dev.rs +++ b/rust/tests/start_dev.rs @@ -207,7 +207,18 @@ fn starts_in_an_empty_directory_with_bundled_sqlite_and_real_cli_admission() { assert_eq!(ready["development"], true); assert_eq!(server.get("/api/cluster/info", None).0, 401); let cluster = project.client(&server, &["server:info"]); - assert!(cluster.get("capabilities").is_some()); + assert_eq!(cluster["capabilities"]["workflow_version_usage"], true); + let usage = project.client( + &server, + &[ + "workflow:version-usage", + "dev.example", + "change-a", + "--min-version=2", + ], + ); + assert_eq!(usage["counts"]["total"], 0, "{usage}"); + assert_eq!(usage["runs"], serde_json::json!([]), "{usage}"); let started = project.client( &server, &[ @@ -265,6 +276,94 @@ fn restart_preserves_workflow_data_and_releases_the_directory_lock() { second.stop(); } +/// A development database written by the published CLI 2.9.0 binary (native +/// schema 14): `dw server:start-dev --data-dir X --port 0`, then +/// `dw workflow:start --type=dev.upgrade --workflow-id=before-upgrade +/// --input='{"kept":true}'` and SIGTERM. Stored zlib-compressed. +const CLI_2_9_0_DATABASE: &[u8] = + include_bytes!("fixtures/dev-server-cli-2.9.0-schema-14.sqlite.zlib"); +/// A native ownership marker at schema 99, newer than any binary knows. +const NEWER_DATABASE: &[u8] = include_bytes!("fixtures/dev-server-schema-99.sqlite.zlib"); + +/// Create a data directory with this dw, then replace its database contents +/// in place, keeping the private file the Server created. +fn data_directory_with(project: &Project, fixture: &[u8]) -> (PathBuf, Vec) { + let directory = project.0.join("data"); + let mut server = DevServer::start(project, &["--data-dir", directory.to_str().unwrap()]); + server.stop(); + let database = miniz_oxide::inflate::decompress_to_vec_zlib(fixture).unwrap(); + fs::write(directory.join("database.sqlite"), &database).unwrap(); + (directory, database) +} + +#[test] +fn development_database_from_cli_2_9_0_is_upgraded_in_place_with_its_data() { + let project = Project::new(); + let (directory, _) = data_directory_with(&project, CLI_2_9_0_DATABASE); + let args = ["--data-dir", directory.to_str().unwrap()]; + let mut upgraded = DevServer::start(&project, &args); + let described = project.client(&upgraded, &["workflow:describe", "before-upgrade"]); + assert_eq!(described["run_id"], "01M7NVAVQN0FTYF4ND5GSGAADA"); + assert_eq!(described["workflow_type"], "dev.upgrade"); + assert_eq!(described["status"], "pending"); + assert_eq!(described["input"], serde_json::json!([true])); + let cluster = project.client(&upgraded, &["server:info"]); + assert_eq!(cluster["capabilities"]["workflow_version_usage"], true); + project.client( + &upgraded, + &[ + "workflow:start", + "--type=dev.upgrade", + "--workflow-id=after-upgrade", + "--input=null", + ], + ); + upgraded.stop(); + let text = upgraded.text.lock().unwrap().clone(); + assert_eq!( + text.matches("Upgraded the development database from schema 14 to 15.") + .count(), + 1, + "{text}" + ); + assert!( + text.find("Upgraded the development database").unwrap() + < text.find("listening on").unwrap() + ); + + let mut current = DevServer::start(&project, &args); + for id in ["before-upgrade", "after-upgrade"] { + let described = project.client(¤t, &["workflow:describe", id]); + assert_eq!(described["workflow_id"], id); + } + current.stop(); + assert!(!current.text.lock().unwrap().contains("Upgraded")); +} + +#[test] +fn newer_development_schema_refuses_without_changing_the_database() { + let project = Project::new(); + let (directory, database) = data_directory_with(&project, NEWER_DATABASE); + let output = project.run(&[ + "server:start-dev", + "--port=0", + "--data-dir", + directory.to_str().unwrap(), + ]); + assert_eq!(output.status.code(), Some(1)); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(stderr.contains("written by a newer dw"), "{stderr}"); + assert!( + stderr.contains("native_schema_newer_than_binary"), + "{stderr}" + ); + assert!(!String::from_utf8_lossy(&output.stdout).contains("listening on")); + assert_eq!( + fs::read(directory.join("database.sqlite")).unwrap(), + database + ); +} + #[test] fn same_data_directory_cannot_start_two_runtimes() { let project = Project::new();