From ea4a094be803d91234f13e35cfea9bc6af88c2c1 Mon Sep 17 00:00:00 2001 From: Jerry Combs Date: Tue, 20 Oct 2026 14:49:23 -0500 Subject: [PATCH 01/18] WIP: drivers per environment, names per environment, environment compare driver.xml environments (empty = every environment), filtered in Overrides.apply; drivers/.name overrides (renameDriver, toTreeNames, Secrets.withNames); keepOnImport for import-live and the web pull (--mark-new-drivers); EnvironmentCompare + vault.compare-envs (values, drift, case-only apart); driver.set environments. Co-Authored-By: Claude Opus 5.5 --- pom.xml | 2 +- .../java/com/pointblue/dirxml/dev/Cli.java | 22 +- .../dirxml/dev/ascode/AsCodeReader.java | 8 + .../dirxml/dev/ascode/AsCodeWriter.java | 3 +- .../dirxml/dev/deploy/DeployCli.java | 11 + .../pointblue/dirxml/dev/deploy/Deployer.java | 21 +- .../dirxml/dev/deploy/DriverEnvironments.java | 93 +++++++ .../dirxml/dev/deploy/EnvironmentCompare.java | 257 ++++++++++++++++++ .../dirxml/dev/deploy/ModelDiff.java | 13 +- .../dirxml/dev/deploy/Overrides.java | 151 ++++++++-- .../pointblue/dirxml/dev/deploy/Secrets.java | 26 ++ .../dirxml/dev/deploy/VaultDiff.java | 23 +- .../pointblue/dirxml/dev/edit/ConfigOps.java | 10 + .../pointblue/dirxml/dev/edit/Registry.java | 6 +- .../pointblue/dirxml/dev/model/Artifact.java | 4 +- .../pointblue/dirxml/dev/model/Driver.java | 22 +- .../pointblue/dirxml/dev/model/DriverSet.java | 61 +++++ .../dirxml/dev/simulate/Simulate.java | 2 +- .../dev/deploy/DriverEnvironmentsTest.java | 198 ++++++++++++++ .../dirxml/dev/deploy/VaultDiffTest.java | 2 +- 20 files changed, 894 insertions(+), 41 deletions(-) create mode 100644 src/main/java/com/pointblue/dirxml/dev/deploy/DriverEnvironments.java create mode 100644 src/main/java/com/pointblue/dirxml/dev/deploy/EnvironmentCompare.java create mode 100644 src/test/java/com/pointblue/dirxml/dev/deploy/DriverEnvironmentsTest.java diff --git a/pom.xml b/pom.xml index a9a27ec..206495d 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.pointblue.dirxml dirxml-dev - 0.48.0 + 0.49.0 jar DirXML Dev diff --git a/src/main/java/com/pointblue/dirxml/dev/Cli.java b/src/main/java/com/pointblue/dirxml/dev/Cli.java index 7b98098..ec950c3 100644 --- a/src/main/java/com/pointblue/dirxml/dev/Cli.java +++ b/src/main/java/com/pointblue/dirxml/dev/Cli.java @@ -62,9 +62,12 @@ public static void main(String[] args) { // import-live (connection from -Dldap.url/-Dldap.bindDn/-Dldap.password) java.util.List positional = new java.util.ArrayList<>(); String envName = null; + boolean markNewDrivers = false; for (int i = 1; i < args.length; i++) { if (args[i].equals("--env") && i + 1 < args.length) { envName = args[++i]; + } else if (args[i].equals("--mark-new-drivers")) { + markNewDrivers = true; } else { positional.add(args[i]); } @@ -78,7 +81,7 @@ public static void main(String[] args) { driverSetDn = positional.size() >= 2 ? positional.get(0) : env.driverSetDn; } else { if (positional.size() < 2) { - System.err.println("usage: import-live [] --env NAME | import-live (IDM_JAVA_OPTS=-Dldap.url/.bindDn/.password)"); + System.err.println("usage: import-live [] --env NAME [--mark-new-drivers] | import-live (IDM_JAVA_OPTS=-Dldap.url/.bindDn/.password)"); System.exit(2); } c = new com.pointblue.dirxml.dev.deploy.Vault.Config(); @@ -89,7 +92,7 @@ public static void main(String[] args) { driverSetDn = positional.get(0); } if (positional.isEmpty()) { - System.err.println("usage: import-live [] --env NAME | import-live (IDM_JAVA_OPTS=-Dldap.url/.bindDn/.password)"); + System.err.println("usage: import-live [] --env NAME [--mark-new-drivers] | import-live (IDM_JAVA_OPTS=-Dldap.url/.bindDn/.password)"); System.exit(2); } Path outDir = Paths.get(positional.get(positional.size() - 1)); @@ -110,9 +113,19 @@ public static void main(String[] args) { } } } + // drivers named differently in this vault take the tree's names; the tree's drivers this environment + // doesn't get stay; a driver new to the tree is marked as this environment's only with --mark-new-drivers + com.pointblue.dirxml.dev.model.DriverSet base = envName != null && java.nio.file.Files.isRegularFile(outDir.resolve(com.pointblue.dirxml.dev.ascode.AsCodeWriter.DRIVERSET_MANIFEST)) + ? com.pointblue.dirxml.dev.ascode.AsCodeReader.read(outDir) : null; + if (base != null) { + for (String n : com.pointblue.dirxml.dev.deploy.Overrides.toTreeNames(live, base.overrides.get(envName))) { + System.err.println("note: " + n); + } + for (String n : com.pointblue.dirxml.dev.deploy.DriverEnvironments.keepOnImport(live, envName, base, markNewDrivers, null)) { + System.err.println("note: " + n); + } + } if (envName != null && java.nio.file.Files.isDirectory(outDir.resolve(com.pointblue.dirxml.dev.deploy.Overrides.DIR))) { - com.pointblue.dirxml.dev.model.DriverSet base = java.nio.file.Files.isRegularFile(outDir.resolve(com.pointblue.dirxml.dev.ascode.AsCodeWriter.DRIVERSET_MANIFEST)) - ? com.pointblue.dirxml.dev.ascode.AsCodeReader.read(outDir) : null; live.overrides.putAll(com.pointblue.dirxml.dev.deploy.Overrides.read(outDir)); for (String n : com.pointblue.dirxml.dev.deploy.Overrides.foldBack(live, envName, base)) { System.err.println("override: " + n); @@ -465,6 +478,7 @@ private static void usage() { System.err.println(" import-live read the live vault (IDM_JAVA_OPTS=-Dldap.url/.bindDn/.password)"); System.err.println(" vault.diff --env E [--driver D…] [--json] the tree vs the live vault (exit 1 if they differ; docs/vault-deploy.md)"); System.err.println(" vault.verify --env E [--driver D…] [--json] same diff, read after a deploy"); + System.err.println(" vault.compare-envs [] --env A --with B [--json] two environments' vaults: drivers in one only, values, other drift"); System.err.println(" vault.deploy --env E [--driver D…] --dry-run|--yes|--step [--confirm E] [--no-restart] [--secrets none|missing|all]"); System.err.println(" [--allow-missing-secrets] [--capture-drift] [--capture-only] [--base-branch B] [--delete-driver D…] [--delete-all entitlements|jobs|rbe-policies|templates|forms|prds|roles|entities|s…] [--json]"); System.err.println(" plan → snapshot → write → restart → verify → audit line; never deletes a driver without --delete-driver"); diff --git a/src/main/java/com/pointblue/dirxml/dev/ascode/AsCodeReader.java b/src/main/java/com/pointblue/dirxml/dev/ascode/AsCodeReader.java index 3d8cdb5..b2ca2ea 100644 --- a/src/main/java/com/pointblue/dirxml/dev/ascode/AsCodeReader.java +++ b/src/main/java/com/pointblue/dirxml/dev/ascode/AsCodeReader.java @@ -84,6 +84,14 @@ private static Driver readDriver(Path dir) throws IOException { d.shimClass = attr(m, "shim-class"); d.shimAuthServer = attr(m, "shim-auth-server"); d.shimAuthId = attr(m, "shim-auth-id"); + String environments = attr(m, "environments"); + if (environments != null) { + for (String e : environments.split(",")) { + if (!e.isBlank()) { + d.environments.add(e.trim()); + } + } + } readMeta(m, d.meta); for (Element c : children(m, "config")) { d.config.put(attr(c, "kind"), xml(dir.resolve(attr(c, "file")))); diff --git a/src/main/java/com/pointblue/dirxml/dev/ascode/AsCodeWriter.java b/src/main/java/com/pointblue/dirxml/dev/ascode/AsCodeWriter.java index 59edff8..5c99216 100644 --- a/src/main/java/com/pointblue/dirxml/dev/ascode/AsCodeWriter.java +++ b/src/main/java/com/pointblue/dirxml/dev/ascode/AsCodeWriter.java @@ -119,7 +119,8 @@ private static void writeDriver(Driver d, Path dir) throws IOException { Files.createDirectories(dir); Manifest m = new Manifest("driver"); m.attr("name", d.name).attr("dn", d.dn).attr("shim-class", d.shimClass) - .attr("shim-auth-server", d.shimAuthServer).attr("shim-auth-id", d.shimAuthId); + .attr("shim-auth-server", d.shimAuthServer).attr("shim-auth-id", d.shimAuthId) + .attr("environments", d.environments.isEmpty() ? null : String.join(", ", d.environments)); m.meta(d.meta); List kinds = new ArrayList<>(d.config.keySet()); diff --git a/src/main/java/com/pointblue/dirxml/dev/deploy/DeployCli.java b/src/main/java/com/pointblue/dirxml/dev/deploy/DeployCli.java index 4dc6c3d..f64a8a3 100644 --- a/src/main/java/com/pointblue/dirxml/dev/deploy/DeployCli.java +++ b/src/main/java/com/pointblue/dirxml/dev/deploy/DeployCli.java @@ -60,6 +60,17 @@ public static int run(String[] argv) throws Exception { List drivers = opts.getOrDefault("driver", List.of()); switch (cmd) { + case "vault.compare-envs": { + String with = first(opts, "with"); + if (with == null || with.isEmpty()) { + System.err.println("usage: vault.compare-envs [] --env A --with B [--json]"); + return 2; + } + EnvironmentCompare c = VaultDiff.compareEnvironments(env, Environments.load().get(with), + pos.isEmpty() ? null : Paths.get(pos.get(0)), n -> System.err.println("note: " + n)); + System.out.print(json ? c.json() + "\n" : c.text()); + return c.isEmpty() ? 0 : 1; + } case "vault.diff": case "vault.verify": { if (pos.isEmpty()) { diff --git a/src/main/java/com/pointblue/dirxml/dev/deploy/Deployer.java b/src/main/java/com/pointblue/dirxml/dev/deploy/Deployer.java index e9faa03..454e45d 100644 --- a/src/main/java/com/pointblue/dirxml/dev/deploy/Deployer.java +++ b/src/main/java/com/pointblue/dirxml/dev/deploy/Deployer.java @@ -277,8 +277,15 @@ public Result run() throws IOException { DriverSet to = AsCodeReader.read(o.tree); Overrides.Applied overrides = Overrides.apply(to, env.name); String overrideNote = overrides.summary() == null ? "" : overrides.summary() + "\n"; + String notDeployedHere = overrides.refusalFor(o.drivers); + if (notDeployedHere != null) { + r.refusal = notDeployedHere; + return r; + } + // the drivers asked for, as this environment's vault names them (the request uses the tree's names) + List scope = overrides.vaultNames(o.drivers); String treeCommit = DeployLog.treeCommit(o.tree); - Secrets secrets = env.secretsFile == null ? Secrets.none() : Secrets.load(env.secretsFile); + Secrets secrets = (env.secretsFile == null ? Secrets.none() : Secrets.load(env.secretsFile)).withNames(overrides.renamed); Overrides.applySecretShimAuthIds(to, secrets); // ids the secrets file supplies win over the tree's (never shown) o.progress.accept("reading the vault"); @@ -286,7 +293,7 @@ public Result run() throws IOException { // 2. diff and plan — the other servers' own driver settings included, read through their connections DriverSet from = VaultDiff.fromVault(vault, dsDn); Servers.readOverrides(from, dsDn, vault, s -> vaultFor(vault, env, s), n -> r.skipped.add("note: " + n)); - String unknown = VaultDiff.unknownDrivers(from, to, o.drivers); + String unknown = VaultDiff.unknownDrivers(from, to, scope); if (unknown != null) { r.refusal = unknown; closeServerConnections(); @@ -297,12 +304,12 @@ public Result run() throws IOException { closeServerConnections(); return r; } - ModelDiff diff = VaultDiff.of(from, to, o.drivers); + ModelDiff diff = VaultDiff.of(from, to, scope); r.diffText = diff.text(); Map> liveNamed = new LinkedHashMap<>(); if ("missing".equals(o.secretsMode)) { for (var d : to.drivers) { - if (o.drivers.isEmpty() || o.drivers.contains(d.name)) { + if (scope.isEmpty() || scope.contains(d.name)) { liveNamed.put(d.name, vault.namedPasswords(VaultMapping.driverDn(dsDn, d.name))); } } @@ -459,7 +466,7 @@ public Result run() throws IOException { o.progress.accept("verifying: reading the vault again"); DriverSet after = VaultDiff.fromVault(vault, dsDn); Servers.readOverrides(after, dsDn, vault, s -> vaultFor(vault, env, s), n -> r.skipped.add("note: " + n)); - ModelDiff verify = VaultDiff.of(after, to, o.drivers.isEmpty() ? diff.affectedDrivers() : o.drivers); + ModelDiff verify = VaultDiff.of(after, to, scope.isEmpty() ? diff.affectedDrivers() : scope); r.verified = verify.isEmpty(); r.verifyText = verify.isEmpty() ? "" : verify.text(); // affectedDrivers() never includes a deleted driver (never restarted), so its removal @@ -502,8 +509,8 @@ public Result run() throws IOException { /** A tree read for comparison with an environment's vault: the base with that environment's overrides and secret shim auth ids applied. */ private static DriverSet readWithOverrides(Path tree, Environments.Environment env) throws IOException { DriverSet ds = AsCodeReader.read(tree); - Overrides.apply(ds, env.name); - Overrides.applySecretShimAuthIds(ds, env.secretsFile == null ? Secrets.none() : Secrets.load(env.secretsFile)); + Overrides.Applied applied = Overrides.apply(ds, env.name); + Overrides.applySecretShimAuthIds(ds, (env.secretsFile == null ? Secrets.none() : Secrets.load(env.secretsFile)).withNames(applied.renamed)); return ds; } diff --git a/src/main/java/com/pointblue/dirxml/dev/deploy/DriverEnvironments.java b/src/main/java/com/pointblue/dirxml/dev/deploy/DriverEnvironments.java new file mode 100644 index 0000000..2e5a181 --- /dev/null +++ b/src/main/java/com/pointblue/dirxml/dev/deploy/DriverEnvironments.java @@ -0,0 +1,93 @@ +package com.pointblue.dirxml.dev.deploy; + +import com.pointblue.dirxml.dev.model.Driver; +import com.pointblue.dirxml.dev.model.DriverSet; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * Drivers that only some environments get ({@code driver.xml}'s {@code environments}, empty = every + * environment): a driver not yet in production, or one that never will be, such as a test-only + * loopback. One tree still serves every environment; what differs is which drivers each receives. + * + *
    + *
  • {@link #filter}: the tree as environment X gets it, without the drivers X doesn't. A diff + * then never reports them as added, and a deploy never creates them in X. Applied by + * {@link Overrides#apply}, so diff, plan, deploy, the known-state gate and simulate all agree.
    • + *
  • {@link #keepOnImport}: reading X's vault into the tree keeps the tree's drivers that X doesn't + * get (X not holding them is expected, not a removal), carries every driver's list over, and can + * mark a driver new to the tree as X's only, so it never reaches another environment unasked.
    • + *
+ */ +public final class DriverEnvironments { + + private DriverEnvironments() { + } + + /** Removes from {@code ds}, in place, the drivers a deploy to {@code env} doesn't get; returns their names. */ + public static List filter(DriverSet ds, String env) { + List out = new ArrayList<>(); + if (env == null) { + return out; + } + for (Driver d : new ArrayList<>(ds.drivers)) { + if (!d.deploysTo(env)) { + ds.drivers.remove(d); + out.add(d.name); + } + } + return out; + } + + /** + * {@code live} was read from {@code env}'s vault and is about to become the tree; {@code base} is the + * tree on disk (null on a first import). In place, on {@code live}: + *
    + *
  • a driver of the tree that {@code env} doesn't get stays exactly as the tree has it, whether or + * not the vault holds one of that name;
    • + *
  • every other driver of the tree keeps its list and the DN the tree recorded;
    • + *
  • with {@code markNew}, a driver only the vault has gets {@code environments = env}: on a tree that + * other environments share, it then reaches them only when someone adds them.
    • + *
+ * With {@code names}, only those drivers are looked at (an update of some drivers, the rest of + * {@code live} being the tree already); null looks at all. Returns notes for the import's report. + */ + public static List keepOnImport(DriverSet live, String env, DriverSet base, boolean markNew, Collection names) { + List notes = new ArrayList<>(); + if (base == null || env == null) { + return notes; + } + for (Driver mine : base.drivers) { + if (names != null && !names.contains(mine.name)) { + continue; + } + Driver theirs = live.driver(mine.name); + if (!mine.deploysTo(env)) { + if (theirs != null) { + live.drivers.remove(theirs); + notes.add("'" + mine.name + "' is in " + env + "'s vault, but the tree doesn't deploy it to " + env + + " (environments=\"" + String.join(", ", mine.environments) + "\"): kept as the tree has it"); + } + live.drivers.add(mine); + } else if (theirs != null) { + theirs.environments.clear(); + theirs.environments.addAll(mine.environments); + if (mine.dn != null) { + theirs.dn = mine.dn; // the DN the tree recorded: another environment's set may sit elsewhere + } + } + } + if (markNew) { + for (Driver d : live.drivers) { + if ((names == null || names.contains(d.name)) && base.driver(d.name) == null && d.environments.isEmpty()) { + d.environments.add(env); + notes.add("'" + d.name + "' is new to the tree: marked as " + env + "'s only (environments=\"" + env + + "\"); add other environments to deploy it there"); + } + } + } + return notes; + } +} diff --git a/src/main/java/com/pointblue/dirxml/dev/deploy/EnvironmentCompare.java b/src/main/java/com/pointblue/dirxml/dev/deploy/EnvironmentCompare.java new file mode 100644 index 0000000..46998e5 --- /dev/null +++ b/src/main/java/com/pointblue/dirxml/dev/deploy/EnvironmentCompare.java @@ -0,0 +1,257 @@ +package com.pointblue.dirxml.dev.deploy; + +import com.pointblue.dirxml.dev.model.Driver; +import com.pointblue.dirxml.dev.model.DriverSet; +import com.pointblue.dirxml.sim.Xds; +import org.w3c.dom.Element; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * Two environments' vaults side by side ({@code docs/vault-deploy.md}, "Comparing two environments"): + * which drivers only one holds, which values differ, and what else differs once the values are set + * aside. With the tree, both vaults' drivers take the tree's names ({@code drivers/.name}), and a + * difference the tree already accounts for is marked expected: a driver the tree doesn't deploy to the + * other environment, a value one of the two environments' {@code overrides/.properties} sets. + * + *
    + *
  • values: every scalar value an override can carry ({@code drivers/.gcv.} for the + * driver's own GCVs, {@code .shim.}, {@code .ecv.}, {@code .shim-auth-server}, + * {@code .shim-auth-id}, {@code driverset.gcv.}) that both hold and that differs, keyed as the + * overrides file keys it;
    • + *
  • drift: the diff of the first against the second after the first takes the second's value + * for each of those, so what remains is what a value can't explain: policies, filters, linkage, + * definitions one side lacks, GCV resources several drivers share.
    • + *
+ */ +public final class EnvironmentCompare { + + /** A driver one environment's vault holds and the other's doesn't. */ + public static final class OnlyIn { + public final String driver; + public final String env; + /** The tree doesn't deploy it to the other environment. */ + public final boolean expected; + /** The tree has no driver of this name. */ + public final boolean notInTree; + + OnlyIn(String driver, String env, boolean expected, boolean notInTree) { + this.driver = driver; + this.env = env; + this.expected = expected; + this.notInTree = notInTree; + } + } + + /** One value both hold, differently. */ + public static final class Value { + /** The tree's name of the driver; null for the driver set's own GCVs. */ + public final String driver; + public final String key; + public final String a; + public final String b; + /** One of the two environments' overrides files sets this key. */ + public final boolean expected; + + Value(String driver, String key, String a, String b, boolean expected) { + this.driver = driver; + this.key = key; + this.a = a; + this.b = b; + this.expected = expected; + } + } + + public final String envA; + public final String envB; + public final List onlyIn = new ArrayList<>(); + public final List values = new ArrayList<>(); + public ModelDiff drift; + /** Changes whose every changed line differs only in letter case (a DN's {@code OU=services} against {@code OU=SERVICES}): eDirectory reads them as the same. Kept apart from the drift. */ + public final List caseOnly = new ArrayList<>(); + public final List notes = new ArrayList<>(); + + private EnvironmentCompare(String envA, String envB) { + this.envA = envA; + this.envB = envB; + } + + /** + * Compares {@code a} (read from {@code envA}'s vault) with {@code b} (from {@code envB}'s); both are + * changed in place (names, values). {@code tree} may be null: then names are compared as they are and + * nothing is marked expected. + */ + public static EnvironmentCompare of(DriverSet a, String envA, DriverSet b, String envB, DriverSet tree) { + EnvironmentCompare c = new EnvironmentCompare(envA, envB); + Map filesA = tree == null ? Map.of() : Objects.requireNonNullElse(tree.overrides.get(envA), Map.of()); + Map filesB = tree == null ? Map.of() : Objects.requireNonNullElse(tree.overrides.get(envB), Map.of()); + c.notes.addAll(Overrides.toTreeNames(a, filesA)); + c.notes.addAll(Overrides.toTreeNames(b, filesB)); + + for (Driver d : a.drivers) { + if (b.driver(d.name) == null) { + c.onlyIn.add(onlyIn(d.name, envA, envB, tree)); + } + } + for (Driver d : b.drivers) { + if (a.driver(d.name) == null) { + c.onlyIn.add(onlyIn(d.name, envB, envA, tree)); + } + } + + Map va = values(a); + Map vb = values(b); + for (Map.Entry e : va.entrySet()) { + if (!vb.containsKey(e.getKey()) || Objects.equals(e.getValue(), vb.get(e.getKey()))) { + continue; + } + String key = e.getKey(); + c.values.add(new Value(driverOf(key), key, e.getValue(), vb.get(key), filesA.containsKey(key) || filesB.containsKey(key))); + Overrides.Target t = Overrides.resolve(a, key); + if (t.resolved() && vb.get(key) != null) { + t.set(vb.get(key)); + } + } + + // the drivers only one side holds are listed above; what's left of the rest is drift, but for differences of letter case + ModelDiff rest = ModelDiff.of(a, b).only(ch -> ch.kind != ModelDiff.Kind.DRIVER_ADDED && ch.kind != ModelDiff.Kind.DRIVER_REMOVED + && (ch.driver == null || (a.driver(ch.driver) != null && b.driver(ch.driver) != null))); + for (ModelDiff.Change ch : rest.changes()) { + if (onlyLetterCase(ch.detail)) { + c.caseOnly.add(ch); + } + } + c.drift = rest.only(ch -> !c.caseOnly.contains(ch)); + return c; + } + + /** True when a change's detail has removed and added lines and they are the same lines but for letter case. */ + static boolean onlyLetterCase(String detail) { + if (detail == null) { + return false; + } + List removed = new ArrayList<>(); + List added = new ArrayList<>(); + for (String line : detail.split("\n")) { + String t = line.stripLeading(); + if (t.startsWith("- ")) { + removed.add(t.substring(2).trim().toLowerCase(java.util.Locale.ROOT)); + } else if (t.startsWith("+ ")) { + added.add(t.substring(2).trim().toLowerCase(java.util.Locale.ROOT)); + } + } + return !removed.isEmpty() && removed.equals(added); + } + + private static OnlyIn onlyIn(String name, String in, String other, DriverSet tree) { + Driver t = tree == null ? null : tree.driver(name); + return new OnlyIn(name, in, t != null && !t.deploysTo(other), tree != null && t == null); + } + + private static String driverOf(String key) { + if (!key.startsWith("drivers/")) { + return null; + } + String rest = key.substring("drivers/".length()); + for (String marker : List.of(".gcv.", ".shim.", ".ecv.", "." + Driver.SHIM_AUTH_SERVER, "." + Driver.SHIM_AUTH_ID)) { + int i = rest.indexOf(marker); + if (i > 0) { + return rest.substring(0, i); + } + } + return rest; + } + + /** Every scalar value an override can carry, keyed as an overrides file keys it, for the drivers of {@code ds} and the set. */ + static Map values(DriverSet ds) { + Map out = new LinkedHashMap<>(); + definitions(ds.configValues, "driverset.gcv.", out); + for (Driver d : ds.drivers) { + String p = "drivers/" + d.name + "."; + definitions(d.config.get(Driver.CONFIG_VALUES), p + "gcv.", out); + definitions(d.config.get(Driver.SHIM_CONFIG_INFO), p + "shim.", out); + definitions(d.config.get(Driver.ENGINE_CONTROL_VALUES), p + "ecv.", out); + out.put(p + Driver.SHIM_AUTH_SERVER, d.shimAuthServer); + out.put(p + Driver.SHIM_AUTH_ID, d.shimAuthId); + } + return out; + } + + private static void definitions(Element blob, String prefix, Map out) { + if (blob == null) { + return; + } + for (Element def : Xds.descendantsByName(blob, "definition")) { + String name = def.getAttribute("name"); + if (!name.isEmpty()) { + out.putIfAbsent(prefix + name, Overrides.valueOf(def)); + } + } + } + + public boolean isEmpty() { + return onlyIn.isEmpty() && values.isEmpty() && (drift == null || drift.isEmpty()) && caseOnly.isEmpty(); + } + + /** For people: drivers in one only, values, then the rest. */ + public String text() { + StringBuilder sb = new StringBuilder(); + sb.append(envA).append(" compared with ").append(envB).append('\n'); + for (String n : notes) { + sb.append("note: ").append(n).append('\n'); + } + if (isEmpty()) { + return sb.append("no differences\n").toString(); + } + for (OnlyIn o : onlyIn) { + sb.append(" only in ").append(o.env).append(": ").append(o.driver) + .append(o.expected ? " (expected: the tree doesn't deploy it to " + (o.env.equals(envA) ? envB : envA) + ")" : "") + .append(o.notInTree ? " (not in the tree)" : "").append('\n'); + } + for (Value v : values) { + sb.append(" ").append(v.key).append(": ").append(envA).append('=').append(v.a == null ? "(none)" : v.a) + .append(" ").append(envB).append('=').append(v.b == null ? "(none)" : v.b) + .append(v.expected ? " (expected: an override)" : "").append('\n'); + } + if (drift != null && !drift.isEmpty()) { + sb.append("other differences, which no value explains:\n").append(drift.text()); + } + if (!caseOnly.isEmpty()) { + sb.append(caseOnly.size()).append(" more differ only in letter case (eDirectory reads those DNs as the same)\n"); + } + return sb.toString(); + } + + public String json() { + StringBuilder sb = new StringBuilder("{\"a\":").append(q(envA)).append(",\"b\":").append(q(envB)).append(",\"onlyIn\":["); + for (int i = 0; i < onlyIn.size(); i++) { + OnlyIn o = onlyIn.get(i); + sb.append(i == 0 ? "" : ",").append("{\"driver\":").append(q(o.driver)).append(",\"env\":").append(q(o.env)) + .append(",\"expected\":").append(o.expected).append(",\"notInTree\":").append(o.notInTree).append('}'); + } + sb.append("],\"values\":["); + for (int i = 0; i < values.size(); i++) { + Value v = values.get(i); + sb.append(i == 0 ? "" : ",").append("{\"driver\":").append(q(v.driver)).append(",\"key\":").append(q(v.key)) + .append(",\"a\":").append(q(v.a)).append(",\"b\":").append(q(v.b)).append(",\"expected\":").append(v.expected).append('}'); + } + sb.append("],\"drift\":").append(drift == null ? "null" : drift.json()).append(",\"caseOnly\":["); + for (int i = 0; i < caseOnly.size(); i++) { + ModelDiff.Change ch = caseOnly.get(i); + sb.append(i == 0 ? "" : ",").append("{\"driver\":").append(q(ch.driver)).append(",\"path\":").append(q(ch.path)).append(",\"summary\":").append(q(ch.summary)).append('}'); + } + sb.append("],\"notes\":["); + for (int i = 0; i < notes.size(); i++) { + sb.append(i == 0 ? "" : ",").append(q(notes.get(i))); + } + return sb.append("]}").toString(); + } + + private static String q(String s) { + return ModelDiff.q(s); + } +} diff --git a/src/main/java/com/pointblue/dirxml/dev/deploy/ModelDiff.java b/src/main/java/com/pointblue/dirxml/dev/deploy/ModelDiff.java index 0b6869b..6d84719 100644 --- a/src/main/java/com/pointblue/dirxml/dev/deploy/ModelDiff.java +++ b/src/main/java/com/pointblue/dirxml/dev/deploy/ModelDiff.java @@ -237,6 +237,17 @@ public List changes() { return Collections.unmodifiableList(changes); } + /** The same two sides with only the changes {@code keep} accepts. */ + public ModelDiff only(java.util.function.Predicate keep) { + ModelDiff d = new ModelDiff(from, to); + for (Change c : changes) { + if (keep.test(c)) { + d.changes.add(c); + } + } + return d; + } + /** * The {@link EmptyKind} guards this diff trips: for each driver + kind ({@code "entitlements"}, * {@code "forms"}, {@code "prds"}) with one or more REMOVED changes, true when the tree side @@ -1578,7 +1589,7 @@ private static List lcsDiff(List a, List b) { } /** A JSON string; {@code null} for a null one, so a missing field never fails the whole diff. */ - private static String q(String s) { + static String q(String s) { if (s == null) { return "null"; } diff --git a/src/main/java/com/pointblue/dirxml/dev/deploy/Overrides.java b/src/main/java/com/pointblue/dirxml/dev/deploy/Overrides.java index cb7ad21..4b4f511 100644 --- a/src/main/java/com/pointblue/dirxml/dev/deploy/Overrides.java +++ b/src/main/java/com/pointblue/dirxml/dev/deploy/Overrides.java @@ -46,6 +46,8 @@ public final class Overrides { public static final String DIR = "overrides"; + /** {@code drivers/.name = }: the name this environment's vault gives the driver, when it differs from the tree's. */ + public static final String NAME = "name"; public static final String EXT = ".properties"; private Overrides() { @@ -155,13 +157,16 @@ public String value() { if (driver == null) { return null; } + if (NAME.equals(setting)) { + return driver.name; + } return Driver.SHIM_AUTH_SERVER.equals(setting) ? driver.shimAuthServer : driver.shimAuthId; } public void set(String value) { if (definition != null) { setValue(definition, value); - } else if (driver != null) { + } else if (driver != null && !NAME.equals(setting)) { // a name is applied by renaming, never set if (Driver.SHIM_AUTH_SERVER.equals(setting)) { driver.shimAuthServer = value; } else { @@ -185,6 +190,12 @@ public static Target resolve(DriverSet ds, String key) { + "drivers/." + Driver.SHIM_AUTH_SERVER + ", drivers/." + Driver.SHIM_AUTH_ID + " or driverset.gcv."); } String rest = key.substring("drivers/".length()); + // the name this environment's vault gives the driver: applied by renames(), after the values + if (isName(key)) { + String driverName = rest.substring(0, rest.length() - NAME.length() - 1); + Driver d = ds.driver(driverName); + return d == null ? new Target(null, "no driver '" + driverName + "' in the tree") : Target.ofSetting(d, NAME); + } // the driver's own settings: drivers/.shim-auth-server, drivers/.shim-auth-id for (String setting : List.of(Driver.SHIM_AUTH_SERVER, Driver.SHIM_AUTH_ID)) { if (rest.endsWith("." + setting)) { @@ -270,41 +281,144 @@ public static final class Applied { public final String env; public final List applied = new ArrayList<>(); public final List problems = new ArrayList<>(); + /** Drivers the tree doesn't deploy to this environment ({@link DriverEnvironments}), left out of the model. */ + public final List notDeployed = new ArrayList<>(); + /** The tree's name to this environment's, for drivers its vault names differently. */ + public final Map renamed = new java.util.LinkedHashMap<>(); + + /** Names a scoped request gave (the tree's) as this environment's model now has them. */ + public List vaultNames(java.util.Collection treeNames) { + if (treeNames == null) { + return null; + } + List out = new ArrayList<>(); + for (String n : treeNames) { + out.add(renamed.getOrDefault(n, n)); + } + return out; + } + + /** The refusal for a scoped request naming drivers this environment doesn't get, or null. */ + public String refusalFor(java.util.Collection treeNames) { + if (treeNames == null) { + return null; + } + List named = new ArrayList<>(); + for (String n : treeNames) { + if (notDeployed.contains(n)) { + named.add("'" + n + "'"); + } + } + return named.isEmpty() ? null : String.join(", ", named) + (named.size() == 1 ? " is" : " are") + " not deployed to '" + env + + "' (driver.xml environments). Add " + env + " to " + (named.size() == 1 ? "its" : "their") + " environments to deploy " + (named.size() == 1 ? "it" : "them") + " there."; + } Applied(String env) { this.env = env; } public String summary() { - if (applied.isEmpty() && problems.isEmpty()) { - return null; + List parts = new ArrayList<>(); + if (!applied.isEmpty() || !problems.isEmpty()) { + parts.add("overrides for '" + env + "': " + applied.size() + " value(s) applied from " + DIR + "/" + env + EXT + + (problems.isEmpty() ? "" : "; not applied: " + String.join("; ", problems))); + } + if (!notDeployed.isEmpty()) { + parts.add("not deployed to '" + env + "' (driver.xml environments): " + String.join(", ", notDeployed)); } - return "overrides for '" + env + "': " + applied.size() + " value(s) applied from " + DIR + "/" + env + EXT - + (problems.isEmpty() ? "" : "; not applied: " + String.join("; ", problems)); + return parts.isEmpty() ? null : String.join("; ", parts); } } /** - * Put environment {@code env}'s values into the model, in place: the model is a throwaway read - * of the tree that a diff or a deploy compares and writes, never written back as the tree. - * An environment without a file applies nothing. + * Put environment {@code env}'s values into the model, in place, and take out the drivers {@code env} + * doesn't get ({@link DriverEnvironments}): the model is a throwaway read of the tree that a diff or a + * deploy compares and writes, never written back as the tree. An environment without a file applies + * no values. */ public static Applied apply(DriverSet ds, String env) { + Applied a = applyValues(ds, env); + Map values = ds.overrides.get(env); + // the drivers this environment doesn't get leave the model after the values, so a value set for one is still placed + a.notDeployed.addAll(DriverEnvironments.filter(ds, env)); + // and the names this environment's vault uses come last: every key above names the tree's driver + for (Map.Entry r : renames(values).entrySet()) { + if (a.notDeployed.contains(r.getKey())) { + continue; + } + if (ds.renameDriver(r.getKey(), r.getValue())) { + a.renamed.put(r.getKey(), r.getValue()); + a.applied.add("drivers/" + r.getKey() + "." + NAME); + } else if (!r.getKey().equals(r.getValue())) { + a.problems.add("drivers/" + r.getKey() + "." + NAME + " (" + (ds.driver(r.getKey()) == null + ? "no driver '" + r.getKey() + "' in the tree" : "the tree already has a driver '" + r.getValue() + "'") + ")"); + } + } + return a; + } + + /** + * Environment {@code env}'s values alone, every driver kept under the tree's name: for the simulator, + * whose cases name the tree's drivers. + */ + public static Applied applyValues(DriverSet ds, String env) { Applied a = new Applied(env); Map values = ds.overrides.get(env); - if (values == null) { - return a; + if (values != null) { + for (Map.Entry e : values.entrySet()) { + if (isName(e.getKey())) { + continue; + } + Target t = resolve(ds, e.getKey()); + if (!t.resolved()) { + a.problems.add(e.getKey() + " (" + t.problem + ")"); + continue; + } + t.set(e.getValue()); + a.applied.add(e.getKey()); + } } - for (Map.Entry e : values.entrySet()) { - Target t = resolve(ds, e.getKey()); - if (!t.resolved()) { - a.problems.add(e.getKey() + " (" + t.problem + ")"); + return a; + } + + /** True for {@code drivers/.name}. */ + public static boolean isName(String key) { + if (!key.startsWith("drivers/") || !key.endsWith("." + NAME) || key.length() <= "drivers/".length() + NAME.length() + 1) { + return false; + } + // drivers/AD.gcv.drv.domain.dns.name is a GCV called drv.domain.dns.name, not a name + String driver = key.substring("drivers/".length(), key.length() - NAME.length() - 1); + return !driver.contains(".gcv.") && !driver.contains(".shim.") && !driver.contains(".ecv."); + } + + /** An environment file's {@code drivers/.name} lines: the tree's name to the name in that vault, in file order. */ + public static Map renames(Map values) { + Map out = new java.util.LinkedHashMap<>(); + if (values != null) { + for (Map.Entry e : values.entrySet()) { + if (isName(e.getKey()) && !e.getValue().isBlank()) { + out.put(e.getKey().substring("drivers/".length(), e.getKey().length() - NAME.length() - 1), e.getValue().trim()); + } + } + } + return out; + } + + /** + * A model read from {@code env}'s vault, its drivers given the tree's names ({@code drivers/.name = + * } read backwards), in place, so it compares with or folds into the tree. Returns notes. + */ + public static List toTreeNames(DriverSet live, Map values) { + List notes = new ArrayList<>(); + for (Map.Entry r : renames(values).entrySet()) { + if (live.driver(r.getValue()) == null) { continue; } - t.set(e.getValue()); - a.applied.add(e.getKey()); + if (!live.renameDriver(r.getValue(), r.getKey())) { + notes.add("'" + r.getValue() + "' was not given the tree's name '" + r.getKey() + "': the vault has a driver of that name too"); + } } - return a; + return notes; } /** @@ -321,6 +435,9 @@ public static List foldBack(DriverSet live, String env, DriverSet base) return notes; } for (Map.Entry e : new ArrayList<>(values.entrySet())) { + if (isName(e.getKey())) { + continue; // a name stays as the file has it: toTreeNames already used it + } Target t = resolve(live, e.getKey()); if (!t.resolved()) { notes.add(e.getKey() + ": " + t.problem + " (left as recorded)"); diff --git a/src/main/java/com/pointblue/dirxml/dev/deploy/Secrets.java b/src/main/java/com/pointblue/dirxml/dev/deploy/Secrets.java index b3d465f..28319dc 100644 --- a/src/main/java/com/pointblue/dirxml/dev/deploy/Secrets.java +++ b/src/main/java/com/pointblue/dirxml/dev/deploy/Secrets.java @@ -6,6 +6,7 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Properties; import java.util.TreeSet; @@ -87,6 +88,31 @@ public static Properties parse(String text) { return p; } + /** + * These secrets, where a driver's keys also answer to the name the environment's vault gives it + * ({@code renamed}: the tree's name to the vault's, {@link Overrides.Applied#renamed}): a secrets file + * may use either name, and the vault's wins where both are set. + */ + public Secrets withNames(Map renamed) { + if (renamed == null || renamed.isEmpty()) { + return this; + } + Properties p = new Properties(); + p.putAll(props); + for (String k : props.stringPropertyNames()) { + for (Map.Entry r : renamed.entrySet()) { + String prefix = r.getKey() + "."; + if (k.startsWith(prefix)) { + String alias = r.getValue() + "." + k.substring(prefix.length()); + if (!props.containsKey(alias)) { + p.setProperty(alias, props.getProperty(k)); + } + } + } + } + return new Secrets(p); + } + /** True if a value (literal, Env, Command or Keychain) is configured for the key. */ public boolean has(String key) { return SecretSource.has(props, key); diff --git a/src/main/java/com/pointblue/dirxml/dev/deploy/VaultDiff.java b/src/main/java/com/pointblue/dirxml/dev/deploy/VaultDiff.java index b28e981..fa91ea5 100644 --- a/src/main/java/com/pointblue/dirxml/dev/deploy/VaultDiff.java +++ b/src/main/java/com/pointblue/dirxml/dev/deploy/VaultDiff.java @@ -171,6 +171,21 @@ static com.pointblue.dirxml.sim.LdifDriverSource.Entry toSourceEntry(Vault.Entry return new com.pointblue.dirxml.sim.LdifDriverSource.Entry(e.dn, attrs); } + /** + * Two environments' vaults side by side ({@link EnvironmentCompare}): both read live, each with the other + * servers' own settings, a shim auth id either's secrets file supplies kept out (a credential is never + * shown), drivers given the tree's names. {@code tree} may be null. + */ + public static EnvironmentCompare compareEnvironments(Environments.Environment a, Environments.Environment b, Path tree, + java.util.function.Consumer notes) throws IOException { + DriverSet base = tree == null ? null : AsCodeReader.read(tree); + DriverSet liveA = readLive(a, n -> notes.accept(a.name + ": " + n)); + DriverSet liveB = readLive(b, n -> notes.accept(b.name + ": " + n)); + Overrides.stripSecretShimAuthIds(liveA, a.secretsFile == null ? Secrets.none() : Secrets.load(a.secretsFile)); + Overrides.stripSecretShimAuthIds(liveB, b.secretsFile == null ? Secrets.none() : Secrets.load(b.secretsFile)); + return EnvironmentCompare.of(liveA, a.name, liveB, b.name, base); + } + /** The diff of the live vault against the tree. */ public static ModelDiff of(Path tree, Environments.Environment env, Collection onlyDrivers) throws IOException { DriverSet to = AsCodeReader.read(tree); @@ -178,10 +193,14 @@ public static ModelDiff of(Path tree, Environments.Environment env, Collection x.equalsIgnoreCase(e.trim()))) { + d.environments.add(e.trim()); + } + } + return; + } default: } if (key.startsWith("param:") || key.startsWith("engine:")) { diff --git a/src/main/java/com/pointblue/dirxml/dev/edit/Registry.java b/src/main/java/com/pointblue/dirxml/dev/edit/Registry.java index 027e9a6..25dd46f 100644 --- a/src/main/java/com/pointblue/dirxml/dev/edit/Registry.java +++ b/src/main/java/com/pointblue/dirxml/dev/edit/Registry.java @@ -551,7 +551,7 @@ drv, opt("nds-class", "eDirectory class name"), opt("app-class", "application cl register("override.set", "set a value for one environment (overrides/.properties); the tree keeps its base value", a -> new OverrideOps.Set(a.get("env"), a.get("key"), a.get("value")), req("env", "environment name"), - req("key", "drivers/.gcv. | .shim. | .ecv. | .shim-auth-server | .shim-auth-id | driverset.gcv."), + req("key", "drivers/.gcv. | .shim. | .ecv. | .shim-auth-server | .shim-auth-id | .name (the driver's name in that vault) | driverset.gcv."), req("value", "the environment's value")); register("override.remove", "remove an environment's override, so it takes the tree's base value", a -> new OverrideOps.Remove(a.get("env"), a.get("key")), @@ -562,9 +562,9 @@ drv, opt("nds-class", "eDirectory class name"), opt("app-class", "application cl req("to", "the server to copy to: its DN, its name, or primary"), opt("driver", "one driver (default: every driver)"), opt("only", "some of gcv,shim,ecv,auth-server,auth-id (default: all)")); - register("driver.set", "set a driver setting: shim-class, shim-auth-server, shim-auth-id, param:, engine:", + register("driver.set", "set a driver setting: shim-class, shim-auth-server, shim-auth-id, environments (the environments it is deployed to, comma-separated; --clear = every environment), param:, engine:", a -> new ConfigOps.DriverSet_(a.get("driver"), a.get("key"), a.containsKey("clear") ? "" : a.get("value")), - drv, req("key", "shim-class|shim-auth-server|shim-auth-id|param:|engine:"), opt("value", "the value"), + drv, req("key", "shim-class|shim-auth-server|shim-auth-id|environments|param:|engine:"), opt("value", "the value"), opt("clear", "flag: make the setting empty (an empty --value reads as missing on the command line)")); register("driver.setting.add", "add a shim parameter (param:, into a section of the shim config) or an engine control value (engine:): its definition with display name, type and value", a -> new ConfigOps.DriverSettingAdd(a.get("driver"), a.get("key"), a.get("type"), a.get("display"), a.get("value"), a.get("section"), a.get("description")), diff --git a/src/main/java/com/pointblue/dirxml/dev/model/Artifact.java b/src/main/java/com/pointblue/dirxml/dev/model/Artifact.java index 12997ef..4803832 100644 --- a/src/main/java/com/pointblue/dirxml/dev/model/Artifact.java +++ b/src/main/java/com/pointblue/dirxml/dev/model/Artifact.java @@ -15,8 +15,8 @@ public abstract class Artifact { public final String name; public final Scope scope; - /** Owning driver name; null for library scope. */ - public final String driver; + /** Owning driver name; null for library scope. Changed only by {@link DriverSet#renameDriver}. */ + public String driver; /** Source-specific extras preserved losslessly (package ids, Designer ids, DN…). */ public final Map meta = new LinkedHashMap<>(); diff --git a/src/main/java/com/pointblue/dirxml/dev/model/Driver.java b/src/main/java/com/pointblue/dirxml/dev/model/Driver.java index a3f9b6f..3c9a7c6 100644 --- a/src/main/java/com/pointblue/dirxml/dev/model/Driver.java +++ b/src/main/java/com/pointblue/dirxml/dev/model/Driver.java @@ -25,7 +25,8 @@ public final class Driver { public static final String SHIM_AUTH_SERVER = "shim-auth-server"; public static final String SHIM_AUTH_ID = "shim-auth-id"; - public final String name; + /** The tree's name; {@link DriverSet#renameDriver} changes it on a throwaway model (an environment that names the driver differently). */ + public String name; public String dn; public String shimClass; public String shimAuthServer; @@ -105,6 +106,25 @@ public static String iconExtensionOf(byte[] b) { public final List entitlements = new ArrayList<>(); /** The driver's jobs ({@code DirXML-Job} objects under it, docs/console-gaps.md §1). */ public final List jobs = new ArrayList<>(); + /** + * The environments this driver is deployed to, by name ({@code driver.xml}'s {@code environments}); + * empty means every environment. A driver that is not yet in production, or never will be, names + * the others ({@code docs/vault-deploy.md}, "Drivers that only some environments get"). + */ + public final List environments = new ArrayList<>(); + + /** True when a deploy to {@code env} gets this driver: no list, or {@code env} in it (any case). */ + public boolean deploysTo(String env) { + if (environments.isEmpty() || env == null) { + return true; + } + for (String e : environments) { + if (e.equalsIgnoreCase(env)) { + return true; + } + } + return false; + } public Driver(String name) { this.name = Objects.requireNonNull(name, "name"); diff --git a/src/main/java/com/pointblue/dirxml/dev/model/DriverSet.java b/src/main/java/com/pointblue/dirxml/dev/model/DriverSet.java index 0161783..e835760 100644 --- a/src/main/java/com/pointblue/dirxml/dev/model/DriverSet.java +++ b/src/main/java/com/pointblue/dirxml/dev/model/DriverSet.java @@ -63,6 +63,67 @@ public Driver driver(String name) { } /** All artifacts in the set, by path (insertion order: library, then each driver). */ + /** + * Gives driver {@code from} the name {@code to}, in place, with everything that names it by path: its + * artifacts' owner, its DN's first RDN, and every link (any driver's) into {@code drivers//}. For a + * throwaway model only, such as the tree as an environment that names the driver differently gets it + * ({@code overrides/.properties}: {@code drivers/.name = }). Text that names the driver + * inside a policy or a GCV is not rewritten. False when there is no driver {@code from}, or one named + * {@code to} already. + */ + public boolean renameDriver(String from, String to) { + Driver d = driver(from); + if (d == null || from.equals(to) || driver(to) != null) { + return false; + } + d.name = to; + if (d.dn != null && d.dn.regionMatches(true, 0, "cn=", 0, 3)) { + int comma = firstUnescapedComma(d.dn); + d.dn = "cn=" + escapeRdnValue(to) + (comma < 0 ? "" : d.dn.substring(comma)); + } + List owned = new ArrayList<>(d.policies); + owned.addAll(d.resources); + owned.addAll(d.subscriber.policies); + owned.addAll(d.publisher.policies); + for (Artifact a : owned) { + a.driver = to; + } + String oldPrefix = "drivers/" + from + "/"; + String newPrefix = "drivers/" + to + "/"; + for (Driver any : drivers) { + for (PolicyLink l : any.links) { + if (l.ref.startsWith(oldPrefix)) { + l.ref = newPrefix + l.ref.substring(oldPrefix.length()); + } + } + } + return true; + } + + private static int firstUnescapedComma(String dn) { + for (int i = 0; i < dn.length(); i++) { + char c = dn.charAt(i); + if (c == '\\') { + i++; + } else if (c == ',') { + return i; + } + } + return -1; + } + + private static String escapeRdnValue(String v) { + StringBuilder b = new StringBuilder(); + for (int i = 0; i < v.length(); i++) { + char c = v.charAt(i); + if (",+\"\\<>;=".indexOf(c) >= 0 || (c == '#' && i == 0) || (c == ' ' && (i == 0 || i == v.length() - 1))) { + b.append('\\'); + } + b.append(c); + } + return b.toString(); + } + public Map index() { Map idx = new LinkedHashMap<>(); for (Artifact a : library.artifacts()) { diff --git a/src/main/java/com/pointblue/dirxml/dev/simulate/Simulate.java b/src/main/java/com/pointblue/dirxml/dev/simulate/Simulate.java index e146485..61a4bad 100644 --- a/src/main/java/com/pointblue/dirxml/dev/simulate/Simulate.java +++ b/src/main/java/com/pointblue/dirxml/dev/simulate/Simulate.java @@ -200,7 +200,7 @@ public Outcome run(Path tree, Path cases, Path against, String env, java.util.fu DriverSet other = against == null ? null : AsCodeReader.read(against); if (env != null) { for (DriverSet target : other == null ? List.of(ds) : List.of(ds, other)) { - com.pointblue.dirxml.dev.deploy.Overrides.Applied applied = com.pointblue.dirxml.dev.deploy.Overrides.apply(target, env); + com.pointblue.dirxml.dev.deploy.Overrides.Applied applied = com.pointblue.dirxml.dev.deploy.Overrides.applyValues(target, env); if (notes != null) { notes.accept(applied.summary() == null ? "overrides for '" + env + "': none (" + (target == ds ? tree : against) + " has no " + com.pointblue.dirxml.dev.deploy.Overrides.DIR + "/" + env + com.pointblue.dirxml.dev.deploy.Overrides.EXT + "; base values used)" diff --git a/src/test/java/com/pointblue/dirxml/dev/deploy/DriverEnvironmentsTest.java b/src/test/java/com/pointblue/dirxml/dev/deploy/DriverEnvironmentsTest.java new file mode 100644 index 0000000..b4a31d7 --- /dev/null +++ b/src/test/java/com/pointblue/dirxml/dev/deploy/DriverEnvironmentsTest.java @@ -0,0 +1,198 @@ +package com.pointblue.dirxml.dev.deploy; + +import com.pointblue.dirxml.dev.ascode.AsCodeReader; +import com.pointblue.dirxml.dev.ascode.AsCodeWriter; +import com.pointblue.dirxml.dev.model.Driver; +import com.pointblue.dirxml.dev.model.DriverSet; +import com.pointblue.dirxml.dev.model.Policy; +import com.pointblue.dirxml.dev.validate.ValidatorTest; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +/** + * Environments that hold different drivers, or name a driver differently: the tree says which environments a + * driver goes to ({@code driver.xml} environments) and what an environment's vault calls it + * ({@code drivers/.name} in its overrides file); compare, deploy and import follow both. + */ +public class DriverEnvironmentsTest { + + @Rule + public TemporaryFolder tmp = new TemporaryFolder(); + + private static final String DS = "cn=dvs,o=system"; + + /** The lab model plus a loopback driver only QA gets. */ + private static DriverSet withQaOnly() { + DriverSet ds = VaultMappingTest.model(); + Driver qa = new Driver("QA Loopback"); + qa.dn = "cn=QA Loopback," + DS; + qa.shimClass = "com.example.Loopback"; + qa.config.put(Driver.CONFIG_VALUES, ValidatorTest.xml("")); + qa.environments.add("QA"); + ds.drivers.add(qa); + return ds; + } + + @Test + public void theListIsKeptInDriverXml() throws Exception { + Path dir = tmp.newFolder("tree").toPath(); + AsCodeWriter.write(withQaOnly(), dir); + assertTrue(Files.readString(dir.resolve("drivers/QA Loopback/driver.xml"), StandardCharsets.UTF_8).contains("environments=\"QA\"")); + assertFalse(Files.readString(dir.resolve("drivers/AD/driver.xml"), StandardCharsets.UTF_8).contains("environments=")); + DriverSet back = AsCodeReader.read(dir); + assertEquals(List.of("QA"), back.driver("QA Loopback").environments); + assertTrue(back.driver("AD").environments.isEmpty()); + } + + @Test + public void anEnvironmentGetsOnlyItsDrivers() throws Exception { + DriverSet prod = withQaOnly(); + Overrides.Applied a = Overrides.apply(prod, "PROD"); + assertNull(prod.driver("QA Loopback")); + assertNotNull(prod.driver("AD")); + assertEquals(List.of("QA Loopback"), a.notDeployed); + assertTrue(a.summary(), a.summary().contains("not deployed to 'PROD'")); + assertNotNull(a.refusalFor(List.of("QA Loopback"))); + assertNull(a.refusalFor(List.of("AD"))); + + DriverSet qa = withQaOnly(); + assertTrue(Overrides.apply(qa, "qa").notDeployed.isEmpty()); // names match in any case + assertNotNull(qa.driver("QA Loopback")); + + // PROD's vault never had the QA driver: nothing to deploy, rather than "added" + FakeVault prodVault = VaultDiffTest.seedVault(VaultMappingTest.model()); + DriverSet to = withQaOnly(); + Overrides.apply(to, "PROD"); + ModelDiff diff = ModelDiff.of(VaultDiff.fromVault(prodVault, DS), to); + assertTrue(diff.text(), diff.isEmpty()); + } + + @Test + public void aDriverNamedDifferentlyInAnEnvironmentComparesUnderThatName() throws Exception { + DriverSet qaModel = VaultMappingTest.model(); + assertTrue(qaModel.renameDriver("AD", "AD-QA")); + FakeVault qaVault = VaultDiffTest.seedVault(qaModel); + + DriverSet to = VaultMappingTest.model(); + to.overrides.put("QA", new LinkedHashMap<>(Map.of("drivers/AD.name", "AD-QA"))); + Overrides.Applied a = Overrides.apply(to, "QA"); + assertEquals(Map.of("AD", "AD-QA"), a.renamed); + Driver renamed = to.driver("AD-QA"); + assertNotNull(renamed); + assertEquals("cn=AD-QA," + DS, renamed.dn); + assertTrue(renamed.links.stream().anyMatch(l -> l.ref.equals("drivers/AD-QA/PKG-GCVs"))); + assertEquals(List.of("AD-QA"), a.vaultNames(List.of("AD"))); + ModelDiff diff = ModelDiff.of(VaultDiff.fromVault(qaVault, DS), to); + assertTrue(diff.text(), diff.isEmpty()); + + // and back: what QA's vault holds takes the tree's name, matching the tree as it is + DriverSet live = VaultDiff.fromVault(qaVault, DS); + assertTrue(Overrides.toTreeNames(live, Map.of("drivers/AD.name", "AD-QA")).isEmpty()); + assertNotNull(live.driver("AD")); + ModelDiff back = ModelDiff.of(live, VaultMappingTest.model()); + assertTrue(back.text(), back.isEmpty()); + + // a name key is a valid key for a driver of the tree, and only for one + assertTrue(Overrides.resolve(VaultMappingTest.model(), "drivers/AD.name").resolved()); + assertFalse(Overrides.resolve(VaultMappingTest.model(), "drivers/Nope.name").resolved()); + assertFalse("a GCV whose name ends in .name", Overrides.isName("drivers/AD.gcv.drv.domain.dns.name")); + assertFalse(Overrides.isName("drivers/AD.shim.host.name")); + assertTrue(Overrides.isName("drivers/AD Driver.name")); + } + + @Test + public void anImportKeepsTheDriversThatEnvironmentDoesntGet() { + // reading PROD, which lacks the QA driver: the tree keeps it, list and all + DriverSet live = VaultMappingTest.model(); + List notes = DriverEnvironments.keepOnImport(live, "PROD", withQaOnly(), false, null); + assertTrue(notes.toString(), notes.isEmpty()); + assertEquals(List.of("QA"), live.driver("QA Loopback").environments); + + // PROD holding a driver of that name anyway: the tree's stays, and the import says so + DriverSet live2 = VaultMappingTest.model(); + Driver stray = new Driver("QA Loopback"); + stray.shimClass = "com.example.Other"; + live2.drivers.add(stray); + notes = DriverEnvironments.keepOnImport(live2, "PROD", withQaOnly(), false, null); + assertEquals("com.example.Loopback", live2.driver("QA Loopback").shimClass); + assertTrue(notes.toString(), notes.get(0).contains("kept as the tree has it")); + + // reading QA into a tree without its driver: marked as QA's only + DriverSet fromQa = withQaOnly(); + fromQa.driver("QA Loopback").environments.clear(); + notes = DriverEnvironments.keepOnImport(fromQa, "QA", VaultMappingTest.model(), true, null); + assertEquals(List.of("QA"), fromQa.driver("QA Loopback").environments); + assertTrue(notes.toString(), notes.get(0).contains("new to the tree")); + // an update of some drivers only looks at those + DriverSet some = withQaOnly(); + some.driver("QA Loopback").environments.clear(); + assertTrue(DriverEnvironments.keepOnImport(some, "QA", VaultMappingTest.model(), true, List.of("AD")).isEmpty()); + assertTrue(some.driver("QA Loopback").environments.isEmpty()); + } + + @Test + public void twoEnvironmentsSideBySide() { + DriverSet tree = withQaOnly(); + tree.overrides.put("QA", new LinkedHashMap<>(Map.of("drivers/AD.shim-auth-server", "ldap://qa"))); + + DriverSet prod = VaultMappingTest.model(); + DriverSet qa = withQaOnly(); + qa.driver("QA Loopback").environments.clear(); // a vault records no lists + qa.driver("AD").shimAuthServer = "ldap://qa"; // expected: QA's file sets it + Overrides.Target users = Overrides.resolve(qa, "drivers/AD.gcv.drv.users"); + users.set("qa-users"); // not expected: no file sets it + Policy sub = (Policy) qa.resolve("drivers/AD/subscriber/sub-ctp"); + sub.content = ValidatorTest.xml("qa only"); + + EnvironmentCompare c = EnvironmentCompare.of(prod, "PROD", qa, "QA", tree); + assertEquals(1, c.onlyIn.size()); + assertEquals("QA Loopback", c.onlyIn.get(0).driver); + assertEquals("QA", c.onlyIn.get(0).env); + assertTrue(c.onlyIn.get(0).expected); + + Map byKey = new LinkedHashMap<>(); + for (EnvironmentCompare.Value v : c.values) { + byKey.put(v.key, v); + } + assertTrue(byKey.toString(), byKey.get("drivers/AD.shim-auth-server").expected); + EnvironmentCompare.Value u = byKey.get("drivers/AD.gcv.drv.users"); + assertFalse(u.expected); + assertEquals("users", u.a); + assertEquals("qa-users", u.b); + assertEquals(2, byKey.size()); + + // the values set aside, the policy is what remains + List drift = new ArrayList<>(); + for (ModelDiff.Change ch : c.drift.changes()) { + drift.add(ch.kind + " " + ch.path); + } + assertEquals(drift.toString(), 1, drift.size()); + assertTrue(drift.toString(), drift.get(0).contains("sub-ctp")); + assertTrue(c.json(), c.json().contains("\"expected\":true")); + } + + @Test + public void aSecretAnswersToEitherName() throws Exception { + Path f = tmp.newFile("qa.secrets").toPath(); + Files.writeString(f, "AD." + Secrets.SHIM_AUTH + " = s3cret\n", StandardCharsets.UTF_8); + Secrets s = Secrets.load(f).withNames(Map.of("AD", "AD-QA")); + assertTrue(s.has(Secrets.shimAuth("AD-QA"))); + assertTrue(s.has(Secrets.shimAuth("AD"))); + assertFalse(Secrets.load(f).has(Secrets.shimAuth("AD-QA"))); + } +} diff --git a/src/test/java/com/pointblue/dirxml/dev/deploy/VaultDiffTest.java b/src/test/java/com/pointblue/dirxml/dev/deploy/VaultDiffTest.java index 708c4a5..f40434e 100644 --- a/src/test/java/com/pointblue/dirxml/dev/deploy/VaultDiffTest.java +++ b/src/test/java/com/pointblue/dirxml/dev/deploy/VaultDiffTest.java @@ -19,7 +19,7 @@ public class VaultDiffTest { private static final String DS = "cn=dvs,o=system"; // matches VaultMappingTest's fixture DNs - private static FakeVault seedVault(DriverSet ds) { + static FakeVault seedVault(DriverSet ds) { FakeVault vault = new FakeVault(); for (LdifDriverSource.Entry e : VaultMappingTest.entries(ds)) { Vault.Entry ve = new Vault.Entry(e.dn); From 957b325ba0fa1077f990cea476154d1c653b35b4 Mon Sep 17 00:00:00 2001 From: Jerry Combs Date: Tue, 20 Oct 2026 14:51:34 -0500 Subject: [PATCH 02/18] docs: taking on existing environments (design) Co-Authored-By: Claude Opus 5.5 --- docs/existing-environments.md | 140 ++++++++++++++++++++++++++++++++++ 1 file changed, 140 insertions(+) create mode 100644 docs/existing-environments.md diff --git a/docs/existing-environments.md b/docs/existing-environments.md new file mode 100644 index 0000000..45142c3 --- /dev/null +++ b/docs/existing-environments.md @@ -0,0 +1,140 @@ +# Taking on existing environments + +*Status: design, 2026-10-20. Parts 1–3 are built on the `driver-environments` branches (core and web); parts 4–7 are proposed. Decisions marked **D1…** are Jerry's.* + +The usual case is not a clean start. A customer has a production tree and one or more others (QA, +test, dev), each changed by hand over the years: drivers added in one and not the other, the same +driver under two names, hot fixes made in production and never carried back, work in QA never +promoted, host names typed into policies. The tool has to take that on without asking anyone to +clean it up first, and it has to help decide what genuinely needs resolving. + +## What a real pair of environments looks like + +A customer's QA and production driver sets, read from two LDIF exports (2026-10-20): + +| | Production | QA | +|---|---|---| +| objects in the driver set | 1,143 | 1,542 | +| drivers | 41 | 43 | +| servers | 4 | 2 | +| driver set DN | `…,ou=SERVICES,o=…` | `…,ou=services,o=…` | + +Compared driver by driver (QA against production, production as the tree): + +- **Drivers in one only:** 7 in production only, 9 in QA only. Among them one is plainly the same + driver renamed (20 of 20 policies in common, names one character apart); one production driver has + four QA counterparts (one QA certificate-revocation driver per CA where production has one); and two + drivers look alike because each pushes to the environment below (production→QA, QA→dev), which only + a person can call the same driver or not. +- **Values:** 81 scalar values differ: 69 shim parameters, 5 engine control values, 4 driver GCVs, 3 + driver-set GCVs. These are what overrides are for. +- **Everything else:** 879 differences. 52 differ only in letter case (`OU=services` against + `OU=SERVICES`; eDirectory reads them as the same) and are set aside. The other 827: + +| | count | what it suggests | +|---|---:|---| +| only in QA | 402 | QA-only objects; 350 are role-based entitlement policies | +| only in production | 57 | production has them, QA never got them | +| lines only production has | 37 | a change made in production, not carried back | +| lines only QA has | 26 | work in QA not yet promoted | +| package versions differ | 13 | the newer side is known | +| cosmetic (descriptions, comments, display names) | 6 | align freely | +| environment text inside content | 7 | a host or server name typed into a policy: belongs in a GCV | +| both sides changed | 268 | needs a person; mostly User Application objects and PRDs | + +Two more things the data shows. Jobs differ by schedule (production runs them on a schedule, QA +doesn't), which an override can't carry yet. And jobs, the driver set's server list and per-server +settings name servers by DN. Tree names and server names are globally unique, so two environments +never share a server: every environment but the base needs its servers mapped, here four production +servers onto two in QA. + +## The process + +One environment at a time, the production tree first. + +1. **Base.** Import production into `main`. `main` is what production runs; everything any other + environment has beyond it shows up as a change to look at, not as something already on its way to + production. *(built)* +2. **Connect the next environment** under Connections, and add it to the project. +3. **Set up that environment**: a guided read (*Vault → Environments → Set up QA…*, proposed) that + proposes, for review, everything it needs. Nothing is written until you apply; you commit the + result. + 1. **Match drivers.** Same name: matched. Otherwise a likely match by shim class, name and policies + in common, which you confirm (a `drivers/.name` override) or reject. Proposals are one-to-one; + one production driver is never matched to two. *(matching proposed; the name override is built)* + 2. **Membership.** A driver only that environment has is marked as its own + (`environments="QA"` in `driver.xml`); a tree driver that environment lacks is marked as not + deployed there. *(built: the marks, compare, deploy and update follow them)* + 3. **Values.** Each differing value becomes that environment's override, ticked by default (the + tree holds production's value). *(built: recorded one at a time from the Environments tab; bulk + proposed)* + 4. **Servers.** Map the tree's servers to that environment's (several to one allowed), so jobs and + per-server settings line up. Required for every environment but the base, since no two + environments share a server. *(proposed: see "Servers")* + 5. **What's left: analyse, then decide** (next section). +4. **Compare again.** What remains is what you decided to leave, and from then on only new + differences show. + +## Analysing what's left + +Each remaining difference gets a likely cause from the comparison itself, as in the table above, and +the page groups them by driver with counts, cause first, so that 827 differences read as a short list +of drivers with a few kinds of problem each. For each you choose: + +- **Production is right:** leave the tree as it is; a deploy to QA (when you choose to) brings QA in + line. Suggested for *only in production*, *lines only production has* and an older package in QA. +- **QA is ahead:** take QA's version into a branch (*Update this branch from QA…*, ticking those + files), to promote through the normal flow. Suggested for *lines only QA has* and a newer package in + QA. +- **Needs fixing:** both sides wrong or a design problem, such as a host name typed into a policy. + Opens the file; the suggestion names the fix (move the literal into a GCV, then override it). +- **Different on purpose:** this environment's content genuinely differs and no value can carry it. + Recorded as known (**D1**). +- **Decide later:** recorded as known, with who and when, so it stops hiding new differences but + stays listed. + +Resolving is then ordinary work on branches: each decision either leaves the tree, changes it on a +branch, or records the difference as known. + +## Known differences (D1) + +A file per environment beside the overrides, `overrides/.known.json`, listing differences +accepted as they are: the object's path, a fingerprint of both sides, the decision, who and when. A +compare shows them collapsed under "known" and shows anything whose fingerprint changed as new. A +deploy to that environment still writes what the tree holds; the plan says which of its changes +overwrite a known difference, so nobody flattens one by accident. + +## Servers + +Server names are globally unique, so the server DNs in the tree (the driver set's server list, each +driver's `servers//` settings, jobs' servers) are the base environment's. Every other +environment maps them in its overrides file, one line per tree server: + +``` +# overrides/QA.properties +servers.cn=VAULT01,ou=servers,o=acme = cn=vault01qa,ou=servers,o=acme +servers.cn=VAULT02,ou=servers,o=acme = cn=vault01qa,ou=servers,o=acme +servers.cn=VAULT03,ou=servers,o=acme = cn=vault02qa,ou=servers,o=acme +servers.cn=VAULT04,ou=servers,o=acme = cn=vault02qa,ou=servers,o=acme +``` + +The tree's servers act as roles. Compare, plan and deploy against QA read and write each role on the +server it maps to; an import from QA maps the other way. Two roles mapped to one server must agree +on that server's settings, or the setup flags the conflict. A server of QA that no role maps to is +reported, as is a role left unmapped (validation: `server-unmapped`). The guided setup proposes the +map by name and by which drivers run where, for you to confirm. + +## Decisions + +- **D1 — Known differences.** Record accepted differences per environment as above, or only compare + and leave triage to people's notes? +- **D2 — Objects of the driver set other than drivers.** Role-based entitlement policies (350 only in + QA), notification templates and driver-set jobs: give them the same environments list as drivers, or + treat them only through known differences? +- **D3 — Servers.** *Decided: required* (tree and server names are globally unique). Open: the map + in each environment's overrides file as above, or in the environment's connection settings, beside + the servers' addresses? +- **D4 — Job schedules and enabled state** as override keys (`drivers/.job..schedule`, + `.disabled`), since environments run jobs on different schedules. +- **D5 — The guided setup** as a page flow (*Set up QA…*) proposing steps 3.1–3.5 at once, or the + Environments tab used step by step as now. From 9f65bfe675abc60ee80dd6957cf6b6873252aa04 Mon Sep 17 00:00:00 2001 From: Jerry Combs Date: Tue, 20 Oct 2026 14:52:46 -0500 Subject: [PATCH 03/18] docs: existing environments, decisions D1-D5 Co-Authored-By: Claude Opus 5.5 --- docs/existing-environments.md | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/docs/existing-environments.md b/docs/existing-environments.md index 45142c3..e686603 100644 --- a/docs/existing-environments.md +++ b/docs/existing-environments.md @@ -1,6 +1,6 @@ # Taking on existing environments -*Status: design, 2026-10-20. Parts 1–3 are built on the `driver-environments` branches (core and web); parts 4–7 are proposed. Decisions marked **D1…** are Jerry's.* +*Status: decided 2026-10-20; being built on the `driver-environments` branches (core and web).* The usual case is not a clean start. A customer has a production tree and one or more others (QA, test, dev), each changed by hand over the years: drivers added in one and not the other, the same @@ -124,17 +124,18 @@ on that server's settings, or the setup flags the conflict. A server of QA that reported, as is a role left unmapped (validation: `server-unmapped`). The guided setup proposes the map by name and by which drivers run where, for you to confirm. -## Decisions - -- **D1 — Known differences.** Record accepted differences per environment as above, or only compare - and leave triage to people's notes? -- **D2 — Objects of the driver set other than drivers.** Role-based entitlement policies (350 only in - QA), notification templates and driver-set jobs: give them the same environments list as drivers, or - treat them only through known differences? -- **D3 — Servers.** *Decided: required* (tree and server names are globally unique). Open: the map - in each environment's overrides file as above, or in the environment's connection settings, beside - the servers' addresses? -- **D4 — Job schedules and enabled state** as override keys (`drivers/.job..schedule`, - `.disabled`), since environments run jobs on different schedules. -- **D5 — The guided setup** as a page flow (*Set up QA…*) proposing steps 3.1–3.5 at once, or the - Environments tab used step by step as now. +## Decisions (taken 2026-10-20) + +- **D1 — Known differences: recorded** per environment in `overrides/.known.json`, as above. +- **D2 — Driver-set objects get an environments list like drivers:** role-based entitlement + policies, notification templates and driver-set jobs. Compare and deploy leave them out of other + environments; an update from an environment marks the ones new to the tree as its own. +- **D3 — Servers: required, mapped in each environment's overrides file** (`servers. = + `). Fewer servers outside production is the normal case, so several tree servers mapping + to one is the rule. A driver whose settings differ between two tree servers that map to the same + server is flagged, and the person picks which that server takes + (`servers..prefer = `). +- **D4 — Job schedules and enabled state are override keys:** `drivers/.job..schedule`, + `drivers/.job..disabled`, and `driverset.job..…` for the set's jobs. +- **D5 — A guided *Set up …* flow** proposing 3.1–3.5 together, reviewed before anything + is written, then committed. From 0603ccb7d037911029c259a8d769bd73fa4dff38 Mon Sep 17 00:00:00 2001 From: Jerry Combs Date: Tue, 20 Oct 2026 14:57:20 -0500 Subject: [PATCH 04/18] WIP: server map per environment (servers. = , prefer, foldPrimary) Co-Authored-By: Claude Opus 5.5 --- docs/existing-environments.md | 6 + .../java/com/pointblue/dirxml/dev/Cli.java | 3 + .../pointblue/dirxml/dev/deploy/Deployer.java | 3 + .../dirxml/dev/deploy/EnvironmentCompare.java | 3 + .../dirxml/dev/deploy/Overrides.java | 20 +- .../dirxml/dev/deploy/ServerMap.java | 312 ++++++++++++++++++ .../pointblue/dirxml/dev/deploy/Servers.java | 36 ++ .../dirxml/dev/deploy/VaultDiff.java | 3 + .../dirxml/dev/deploy/ServerMapTest.java | 118 +++++++ 9 files changed, 501 insertions(+), 3 deletions(-) create mode 100644 src/main/java/com/pointblue/dirxml/dev/deploy/ServerMap.java create mode 100644 src/test/java/com/pointblue/dirxml/dev/deploy/ServerMapTest.java diff --git a/docs/existing-environments.md b/docs/existing-environments.md index e686603..9a878f6 100644 --- a/docs/existing-environments.md +++ b/docs/existing-environments.md @@ -93,6 +93,12 @@ of drivers with a few kinds of problem each. For each you choose: - **Decide later:** recorded as known, with who and when, so it stops hiding new differences but stays listed. +What's known about a driver helps decide. When the repository holds notes on a driver (a summary, +known defects; for example `docs/drivers/.md` from an earlier analysis), the setup shows them +beside that driver's differences. In the customer data above, production carries an extra rule that +vetoes every user event on one driver's subscriber channel, and QA doesn't: a production stopgap for a +defect the earlier analysis had found, which only that context explains. + Resolving is then ordinary work on branches: each decision either leaves the tree, changes it on a branch, or records the difference as known. diff --git a/src/main/java/com/pointblue/dirxml/dev/Cli.java b/src/main/java/com/pointblue/dirxml/dev/Cli.java index ec950c3..63c2638 100644 --- a/src/main/java/com/pointblue/dirxml/dev/Cli.java +++ b/src/main/java/com/pointblue/dirxml/dev/Cli.java @@ -121,6 +121,9 @@ public static void main(String[] args) { for (String n : com.pointblue.dirxml.dev.deploy.Overrides.toTreeNames(live, base.overrides.get(envName))) { System.err.println("note: " + n); } + for (String n : com.pointblue.dirxml.dev.deploy.ServerMap.toTree(live, base.overrides.get(envName), base)) { + System.err.println("note: " + n); + } for (String n : com.pointblue.dirxml.dev.deploy.DriverEnvironments.keepOnImport(live, envName, base, markNewDrivers, null)) { System.err.println("note: " + n); } diff --git a/src/main/java/com/pointblue/dirxml/dev/deploy/Deployer.java b/src/main/java/com/pointblue/dirxml/dev/deploy/Deployer.java index 454e45d..42d9a18 100644 --- a/src/main/java/com/pointblue/dirxml/dev/deploy/Deployer.java +++ b/src/main/java/com/pointblue/dirxml/dev/deploy/Deployer.java @@ -293,6 +293,9 @@ public Result run() throws IOException { // 2. diff and plan — the other servers' own driver settings included, read through their connections DriverSet from = VaultDiff.fromVault(vault, dsDn); Servers.readOverrides(from, dsDn, vault, s -> vaultFor(vault, env, s), n -> r.skipped.add("note: " + n)); + if (!overrides.serverMap.isEmpty()) { + Servers.foldPrimary(to, Servers.primary(vault)); // a mapped server that is this environment's primary holds the base values + } String unknown = VaultDiff.unknownDrivers(from, to, scope); if (unknown != null) { r.refusal = unknown; diff --git a/src/main/java/com/pointblue/dirxml/dev/deploy/EnvironmentCompare.java b/src/main/java/com/pointblue/dirxml/dev/deploy/EnvironmentCompare.java index 46998e5..f075495 100644 --- a/src/main/java/com/pointblue/dirxml/dev/deploy/EnvironmentCompare.java +++ b/src/main/java/com/pointblue/dirxml/dev/deploy/EnvironmentCompare.java @@ -91,6 +91,9 @@ public static EnvironmentCompare of(DriverSet a, String envA, DriverSet b, Strin Map filesB = tree == null ? Map.of() : Objects.requireNonNullElse(tree.overrides.get(envB), Map.of()); c.notes.addAll(Overrides.toTreeNames(a, filesA)); c.notes.addAll(Overrides.toTreeNames(b, filesB)); + // each side's servers as the tree names them, so per-server settings and jobs line up + c.notes.addAll(ServerMap.toTree(a, filesA, tree)); + c.notes.addAll(ServerMap.toTree(b, filesB, tree)); for (Driver d : a.drivers) { if (b.driver(d.name) == null) { diff --git a/src/main/java/com/pointblue/dirxml/dev/deploy/Overrides.java b/src/main/java/com/pointblue/dirxml/dev/deploy/Overrides.java index 4b4f511..ab2ff03 100644 --- a/src/main/java/com/pointblue/dirxml/dev/deploy/Overrides.java +++ b/src/main/java/com/pointblue/dirxml/dev/deploy/Overrides.java @@ -180,6 +180,15 @@ public static Target resolve(DriverSet ds, String key) { if (key.startsWith("?")) { return new Target(null, "not a 'key = value' line"); } + if (ServerMap.isKey(key)) { + boolean prefer = key.endsWith(ServerMap.PREFER); + String server = key.substring(ServerMap.PREFIX.length(), key.length() - (prefer ? ServerMap.PREFER.length() : 0)); + if (prefer) { + return new Target(null, null, ServerMap.PREFIX, null); // names this environment's server; checked against the map by the setup + } + return ds.servers.stream().anyMatch(s -> s.equalsIgnoreCase(server)) ? new Target(null, null, ServerMap.PREFIX, null) + : new Target(null, "the tree names no server '" + server + "' (driverset.xml)"); + } if (key.startsWith("driverset.gcv.")) { String name = key.substring("driverset.gcv.".length()); GcvOps.Home home = GcvOps.find(ds, null, name, ds.index()); @@ -285,6 +294,8 @@ public static final class Applied { public final List notDeployed = new ArrayList<>(); /** The tree's name to this environment's, for drivers its vault names differently. */ public final Map renamed = new java.util.LinkedHashMap<>(); + /** The tree's servers to this environment's ({@link ServerMap}); empty for the base environment. */ + public final Map serverMap = new java.util.LinkedHashMap<>(); /** Names a scoped request gave (the tree's) as this environment's model now has them. */ public List vaultNames(java.util.Collection treeNames) { @@ -354,6 +365,9 @@ public static Applied apply(DriverSet ds, String env) { ? "no driver '" + r.getKey() + "' in the tree" : "the tree already has a driver '" + r.getValue() + "'") + ")"); } } + // the tree's servers as this environment's (several onto one where it has fewer) + a.serverMap.putAll(ServerMap.of(values)); + a.problems.addAll(ServerMap.apply(ds, values)); return a; } @@ -366,7 +380,7 @@ public static Applied applyValues(DriverSet ds, String env) { Map values = ds.overrides.get(env); if (values != null) { for (Map.Entry e : values.entrySet()) { - if (isName(e.getKey())) { + if (isName(e.getKey()) || ServerMap.isKey(e.getKey())) { continue; } Target t = resolve(ds, e.getKey()); @@ -435,8 +449,8 @@ public static List foldBack(DriverSet live, String env, DriverSet base) return notes; } for (Map.Entry e : new ArrayList<>(values.entrySet())) { - if (isName(e.getKey())) { - continue; // a name stays as the file has it: toTreeNames already used it + if (isName(e.getKey()) || ServerMap.isKey(e.getKey())) { + continue; // a name or a server stays as the file has it: the import already used it } Target t = resolve(live, e.getKey()); if (!t.resolved()) { diff --git a/src/main/java/com/pointblue/dirxml/dev/deploy/ServerMap.java b/src/main/java/com/pointblue/dirxml/dev/deploy/ServerMap.java new file mode 100644 index 0000000..9b6d8d1 --- /dev/null +++ b/src/main/java/com/pointblue/dirxml/dev/deploy/ServerMap.java @@ -0,0 +1,312 @@ +package com.pointblue.dirxml.dev.deploy; + +import com.pointblue.dirxml.dev.model.Driver; +import com.pointblue.dirxml.dev.model.DriverSet; +import com.pointblue.dirxml.dev.model.Job; +import org.w3c.dom.Element; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +/** + * The tree's servers as an environment's ({@code docs/existing-environments.md}, "Servers"). Tree and server + * names are globally unique, so the servers the tree names (the set's server list, each driver's + * {@code servers//} settings, jobs' servers) are the base environment's; every other environment + * maps them in its overrides file, one line per tree server, several onto one where it has fewer: + * 
+ *   servers.cn=VAULT01,ou=servers,o=acme = cn=vault01qa,ou=servers,o=acme
+ *   servers.cn=VAULT02,ou=servers,o=acme = cn=vault01qa,ou=servers,o=acme
+ *   servers.cn=vault01qa,ou=servers,o=acme.prefer = cn=VAULT02,ou=servers,o=acme
+ *
+ * Where several tree servers land on one, that server takes the settings of the one listed first, or of + * the one its {@code .prefer} line names; a driver whose settings differ between them is reported. + * Reading that environment maps back: its servers take the tree's names, its settings land on the + * preferred tree server, and the other tree servers keep what the tree holds. + */ +public final class ServerMap { + + public static final String PREFIX = "servers."; + public static final String PREFER = ".prefer"; + + private ServerMap() { + } + + /** True for an overrides-file key this class reads. */ + public static boolean isKey(String key) { + return key.startsWith(PREFIX) && key.length() > PREFIX.length(); + } + + /** The tree's server (as written) to the environment's, in file order; empty when the file maps none. */ + public static Map of(Map values) { + Map out = new LinkedHashMap<>(); + if (values != null) { + for (Map.Entry e : values.entrySet()) { + if (isKey(e.getKey()) && !e.getKey().endsWith(PREFER) && !e.getValue().isBlank()) { + out.put(e.getKey().substring(PREFIX.length()), e.getValue().trim()); + } + } + } + return out; + } + + /** For each environment server, the tree server whose settings it takes: its {@code .prefer} line, else the first mapped to it. */ + public static Map preferred(Map values) { + Map out = new LinkedHashMap<>(); + for (Map.Entry m : of(values).entrySet()) { + out.putIfAbsent(key(m.getValue()), m.getKey()); + } + if (values != null) { + for (Map.Entry e : values.entrySet()) { + if (isKey(e.getKey()) && e.getKey().endsWith(PREFER) && !e.getValue().isBlank()) { + out.put(key(e.getKey().substring(PREFIX.length(), e.getKey().length() - PREFER.length())), e.getValue().trim()); + } + } + } + return out; + } + + /** + * The tree as the environment gets it, in place: every server it names becomes the environment's, the + * settings of several tree servers landing on one taken from the preferred one. Returns what couldn't be + * applied or needs a person (a tree server not mapped, two tree servers' settings differing). + */ + public static List apply(DriverSet ds, Map values) { + List problems = new ArrayList<>(); + Map map = of(values); + if (map.isEmpty()) { + return problems; + } + Map prefer = preferred(values); + for (String s : ds.servers) { + if (find(map, s) == null) { + problems.add("server " + s + " is not mapped (" + PREFIX + s + " = )"); + } + } + replaceAll(ds.servers, map); + for (Job j : ds.jobs) { + replaceAll(j.servers, map); + } + for (Driver d : ds.drivers) { + for (Job j : d.jobs) { + replaceAll(j.servers, map); + } + d.serverConfig.putAll(remap(d, d.serverConfig, map, prefer, problems, true)); + d.serverSettings.putAll(remap(d, d.serverSettings, map, prefer, problems, false)); + } + return problems; + } + + /** + * A model read from the environment's vault, in place, as the tree names its servers: the server list and + * jobs' servers take the tree's names (the tree's own list where it maps to the same), and each environment + * server's settings go to its preferred tree server, the other tree servers keeping what {@code base} holds. + * Returns notes (an environment server no tree server maps to). + */ + public static List toTree(DriverSet live, Map values, DriverSet base) { + List notes = new ArrayList<>(); + Map map = of(values); + if (map.isEmpty()) { + return notes; + } + Map prefer = preferred(values); + // the set's list: the tree's own when it maps onto exactly what the vault holds + if (base != null && sameServers(mapped(base.servers, map), live.servers)) { + live.servers.clear(); + live.servers.addAll(base.servers); + } else { + replaceBack(live.servers, prefer, notes); + } + for (Job j : live.jobs) { + Job mine = base == null ? null : findJob(base.jobs, j.name); + restoreJob(j, mine, map, prefer, notes); + } + for (Driver d : live.drivers) { + Driver mine = base == null ? null : base.driver(d.name); + for (Job j : d.jobs) { + restoreJob(j, mine == null ? null : findJob(mine.jobs, j.name), map, prefer, notes); + } + Map> config = new LinkedHashMap<>(); + Map> settings = new LinkedHashMap<>(); + for (Map.Entry> e : d.serverConfig.entrySet()) { + String tree = prefer.get(key(e.getKey())); + if (tree == null) { + notes.add("server " + e.getKey() + " of this environment is not mapped to a tree server: its settings for '" + d.name + "' are not taken"); + } else { + config.put(tree, e.getValue()); + } + } + for (Map.Entry> e : d.serverSettings.entrySet()) { + String tree = prefer.get(key(e.getKey())); + if (tree != null) { + settings.put(tree, e.getValue()); + } + } + // the tree servers that aren't preferred anywhere keep the tree's settings + if (mine != null) { + for (Map.Entry> e : mine.serverConfig.entrySet()) { + if (!containsIgnoreCase(prefer.values(), e.getKey()) && !config.containsKey(e.getKey())) { + config.put(e.getKey(), e.getValue()); + } + } + for (Map.Entry> e : mine.serverSettings.entrySet()) { + if (!containsIgnoreCase(prefer.values(), e.getKey()) && !settings.containsKey(e.getKey())) { + settings.put(e.getKey(), e.getValue()); + } + } + } + d.serverConfig.clear(); + d.serverConfig.putAll(config); + d.serverSettings.clear(); + d.serverSettings.putAll(settings); + } + return notes; + } + + // ---- helpers ---- + + /** + * One driver's per-server settings keyed by the environment's servers: each holds exactly what its preferred + * tree server holds (nothing, when that one holds the primary's values); another tree server mapped to the same + * server whose settings differ is reported. Settings of a tree server the map leaves out stay as they are. + */ + private static Map> remap(Driver d, Map> byServer, Map map, + Map prefer, List problems, boolean xml) { + Map> out = new LinkedHashMap<>(); + for (Map.Entry> e : byServer.entrySet()) { + if (find(map, e.getKey()) == null) { + out.put(e.getKey(), e.getValue()); // unmapped: reported once, for the set's list + } + } + Map> roles = new LinkedHashMap<>(); // environment server -> tree servers, in file order + for (Map.Entry m : map.entrySet()) { + roles.computeIfAbsent(m.getValue(), k -> new ArrayList<>()).add(m.getKey()); + } + for (Map.Entry> r : roles.entrySet()) { + String env = r.getKey(); + String chosen = prefer.getOrDefault(key(env), r.getValue().get(0)); + Map held = get(byServer, chosen); + for (String other : r.getValue()) { + if (!other.equalsIgnoreCase(chosen) && !same(get(byServer, other), held, xml)) { + problems.add("driver '" + d.name + "': " + chosen + " and " + other + " both map to " + env + " but hold different settings; " + + env + " takes " + chosen + "'s (" + PREFIX + env + PREFER + " = to choose another)"); + } + } + if (held != null) { + out.put(env, held); + } + } + byServer.clear(); + return out; + } + + private static Map get(Map> byServer, String server) { + for (Map.Entry> e : byServer.entrySet()) { + if (e.getKey().equalsIgnoreCase(server)) { + return e.getValue(); + } + } + return null; + } + + private static boolean same(Map a, Map b, boolean xml) { + if (a == null || b == null) { + return a == b; + } + if (!a.keySet().equals(b.keySet())) { + return false; + } + for (Map.Entry e : a.entrySet()) { + V other = b.get(e.getKey()); + boolean equal = xml ? Servers.sameXml((Element) e.getValue(), (Element) other) : java.util.Objects.equals(e.getValue(), other); + if (!equal) { + return false; + } + } + return true; + } + + private static void restoreJob(Job j, Job mine, Map map, Map prefer, List notes) { + if (mine != null && sameServers(mapped(mine.servers, map), j.servers)) { + j.servers.clear(); + j.servers.addAll(mine.servers); + } else { + replaceBack(j.servers, prefer, notes); + } + } + + private static Job findJob(List jobs, String name) { + for (Job j : jobs) { + if (j.name.equals(name)) { + return j; + } + } + return null; + } + + private static void replaceAll(List servers, Map map) { + List out = mapped(servers, map); + servers.clear(); + servers.addAll(out); + } + + private static List mapped(List servers, Map map) { + List out = new ArrayList<>(); + for (String s : servers) { + String env = find(map, s); + String v = env == null ? s : env; + if (out.stream().noneMatch(x -> x.equalsIgnoreCase(v))) { + out.add(v); + } + } + return out; + } + + private static void replaceBack(List servers, Map prefer, List notes) { + List out = new ArrayList<>(); + for (String s : servers) { + String tree = prefer.get(key(s)); + if (tree == null) { + notes.add("server " + s + " of this environment is not mapped to a tree server"); + } + String v = tree == null ? s : tree; + if (out.stream().noneMatch(x -> x.equalsIgnoreCase(v))) { + out.add(v); + } + } + servers.clear(); + servers.addAll(out); + } + + private static boolean sameServers(List a, List b) { + if (a.size() != b.size()) { + return false; + } + for (String s : a) { + if (b.stream().noneMatch(x -> x.equalsIgnoreCase(s))) { + return false; + } + } + return true; + } + + /** The environment's server for a tree server (DNs compare without regard to case), or null. */ + private static String find(Map map, String treeServer) { + for (Map.Entry e : map.entrySet()) { + if (e.getKey().equalsIgnoreCase(treeServer)) { + return e.getValue(); + } + } + return null; + } + + private static boolean containsIgnoreCase(java.util.Collection values, String s) { + return values.stream().anyMatch(v -> v.equalsIgnoreCase(s)); + } + + private static String key(String dn) { + return dn.toLowerCase(Locale.ROOT); + } +} diff --git a/src/main/java/com/pointblue/dirxml/dev/deploy/Servers.java b/src/main/java/com/pointblue/dirxml/dev/deploy/Servers.java index edf05ab..1a4e9a4 100644 --- a/src/main/java/com/pointblue/dirxml/dev/deploy/Servers.java +++ b/src/main/java/com/pointblue/dirxml/dev/deploy/Servers.java @@ -216,6 +216,42 @@ public static void readOverrides(DriverSet ds, String driverSetDn, VaultAccess p } } + /** + * The settings a tree records for {@code primary} (the server a deploy connects through) made the drivers' + * own, in place: the tree keeps per-server settings only as they differ from the primary's, so a mapped + * server that turns out to be this environment's primary ({@link ServerMap}) holds the base values. + */ + public static void foldPrimary(DriverSet ds, String primary) { + if (primary == null) { + return; + } + for (Driver d : ds.drivers) { + for (String s : new ArrayList<>(d.serverConfig.keySet())) { + if (s.equalsIgnoreCase(primary)) { + for (Map.Entry e : d.serverConfig.remove(s).entrySet()) { + if (e.getValue() == null) { + d.config.remove(e.getKey()); + } else { + d.config.put(e.getKey(), e.getValue()); + } + } + } + } + for (String s : new ArrayList<>(d.serverSettings.keySet())) { + if (s.equalsIgnoreCase(primary)) { + for (Map.Entry e : d.serverSettings.remove(s).entrySet()) { + String v = e.getValue() == null || e.getValue().isEmpty() ? null : e.getValue(); + if (Driver.SHIM_AUTH_SERVER.equals(e.getKey())) { + d.shimAuthServer = v; + } else if (Driver.SHIM_AUTH_ID.equals(e.getKey())) { + d.shimAuthId = v; + } + } + } + } + } + } + /** Servers of the set other than the primary, by the tree's record ({@code driverset.xml}). */ public static List others(DriverSet ds, String primary) { List out = new ArrayList<>(); diff --git a/src/main/java/com/pointblue/dirxml/dev/deploy/VaultDiff.java b/src/main/java/com/pointblue/dirxml/dev/deploy/VaultDiff.java index fa91ea5..0fa35a4 100644 --- a/src/main/java/com/pointblue/dirxml/dev/deploy/VaultDiff.java +++ b/src/main/java/com/pointblue/dirxml/dev/deploy/VaultDiff.java @@ -200,6 +200,9 @@ public static ModelDiff of(Path tree, Environments.Environment env, Collection" + v + + ""); + } + + /** Production as the tree has it: P1 the primary (no entry of its own), P2 differs, P3 and P4 agree. */ + private static DriverSet prod() { + DriverSet ds = VaultMappingTest.model(); + ds.servers.addAll(List.of(P1, P2, P3, P4)); + Driver ad = ds.driver("AD"); + ad.serverConfig.put(P2, new LinkedHashMap<>(Map.of(Driver.CONFIG_VALUES, gcvs("two")))); + ad.serverConfig.put(P3, new LinkedHashMap<>(Map.of(Driver.CONFIG_VALUES, gcvs("three")))); + ad.serverConfig.put(P4, new LinkedHashMap<>(Map.of(Driver.CONFIG_VALUES, gcvs("three")))); + Job job = new Job("nightly", null); + job.servers.addAll(List.of(P1, P3)); + ad.jobs.add(job); + return ds; + } + + private static Map qaFile(boolean preferP2) { + Map f = new LinkedHashMap<>(); + f.put("servers." + P1, Q1); + f.put("servers." + P2, Q1); + f.put("servers." + P3, Q2); + f.put("servers." + P4, Q2); + if (preferP2) { + f.put("servers." + Q1 + ".prefer", P2); + } + return f; + } + + private static String value(org.w3c.dom.Element gcvs) { + return Overrides.valueOf(com.pointblue.dirxml.dev.edit.GcvOps.definition(gcvs, "v")); + } + + @Test + public void fourServersOntoTwo() { + DriverSet ds = prod(); + List problems = ServerMap.apply(ds, qaFile(false)); + assertEquals(List.of(Q1, Q2), ds.servers); + Driver ad = ds.driver("AD"); + assertFalse("Q1 takes P1's: the primary's values, no entry", ad.serverConfig.containsKey(Q1)); + assertEquals("three", value(ad.serverConfig.get(Q2).get(Driver.CONFIG_VALUES))); + assertEquals(List.of(Q1, Q2), ad.jobs.get(0).servers); + assertEquals(problems.toString(), 1, problems.size()); + assertTrue(problems.get(0), problems.get(0).contains(P2) && problems.get(0).contains(".prefer")); + } + + @Test + public void thePreferredServerWinsAndThePrimaryTakesItAsTheBase() { + DriverSet ds = prod(); + ServerMap.apply(ds, qaFile(true)); + Driver ad = ds.driver("AD"); + assertEquals("two", value(ad.serverConfig.get(Q1).get(Driver.CONFIG_VALUES))); + Servers.foldPrimary(ds, Q1); + assertNull(ad.serverConfig.get(Q1)); + assertEquals("two", value(ad.config.get(Driver.CONFIG_VALUES))); + } + + @Test + public void aReadOfQaMapsBack() { + DriverSet base = prod(); + DriverSet live = VaultMappingTest.model(); // what QA's vault holds + live.servers.addAll(List.of(Q1, Q2)); + live.driver("AD").serverConfig.put(Q2, new LinkedHashMap<>(Map.of(Driver.CONFIG_VALUES, gcvs("qa-three")))); + Job job = new Job("nightly", null); + job.servers.addAll(List.of(Q1, Q2)); + live.driver("AD").jobs.add(job); + + List notes = ServerMap.toTree(live, qaFile(false), base); + assertTrue(notes.toString(), notes.isEmpty()); + assertEquals("the tree's own list", List.of(P1, P2, P3, P4), live.servers); + Driver ad = live.driver("AD"); + assertEquals("QA's Q2 lands on P3, the server it prefers", "qa-three", value(ad.serverConfig.get(P3).get(Driver.CONFIG_VALUES))); + assertEquals("P4 keeps the tree's", "three", value(ad.serverConfig.get(P4).get(Driver.CONFIG_VALUES))); + assertEquals("P2 keeps the tree's", "two", value(ad.serverConfig.get(P2).get(Driver.CONFIG_VALUES))); + assertEquals(List.of(P1, P3), ad.jobs.get(0).servers); + } + + @Test + public void theKeysAreValidOnlyForTheTreesServers() { + DriverSet ds = prod(); + assertTrue(Overrides.resolve(ds, "servers." + P3).resolved()); + assertTrue("any case", Overrides.resolve(ds, "servers." + P3.toUpperCase()).resolved()); + assertFalse(Overrides.resolve(ds, "servers.cn=nope,o=acme").resolved()); + assertTrue(Overrides.resolve(ds, "servers." + Q1 + ".prefer").resolved()); + } +} From 8f7093ea233c5fc86e33b79add31ea04f166555b Mon Sep 17 00:00:00 2001 From: Jerry Combs Date: Tue, 20 Oct 2026 14:59:21 -0500 Subject: [PATCH 05/18] WIP: a new driver's server chosen at deploy (D6) Co-Authored-By: Claude Opus 5.5 --- docs/existing-environments.md | 6 ++++ .../dirxml/dev/deploy/DeployCli.java | 9 ++++++ .../pointblue/dirxml/dev/deploy/Deployer.java | 22 ++++++++++++- .../com/pointblue/dirxml/dev/deploy/Plan.java | 18 ++++++++++- .../pointblue/dirxml/dev/model/Driver.java | 5 +++ .../pointblue/dirxml/dev/deploy/PlanTest.java | 32 +++++++++++++++++++ 6 files changed, 90 insertions(+), 2 deletions(-) diff --git a/docs/existing-environments.md b/docs/existing-environments.md index 9a878f6..f97e12d 100644 --- a/docs/existing-environments.md +++ b/docs/existing-environments.md @@ -143,5 +143,11 @@ map by name and by which drivers run where, for you to confirm. (`servers..prefer = `). - **D4 — Job schedules and enabled state are override keys:** `drivers/.job..schedule`, `drivers/.job..disabled`, and `driverset.job..…` for the set's jobs. +- **D6 — A new driver's server, chosen at deploy:** promoting a driver to an environment with several + servers asks which one runs it (default: the one the deploy connects through). That server gets the + driver's start option, manual, and every other server of the set gets disabled + (`vault.deploy --driver-server "D="`; *Runs on* in the plan). A deploy choice, not a tree value: + it matters only when the driver is created there. The step waits for the driver to replicate to that + server first. - **D5 — A guided *Set up …* flow** proposing 3.1–3.5 together, reviewed before anything is written, then committed. diff --git a/src/main/java/com/pointblue/dirxml/dev/deploy/DeployCli.java b/src/main/java/com/pointblue/dirxml/dev/deploy/DeployCli.java index f64a8a3..b0fd7e2 100644 --- a/src/main/java/com/pointblue/dirxml/dev/deploy/DeployCli.java +++ b/src/main/java/com/pointblue/dirxml/dev/deploy/DeployCli.java @@ -101,6 +101,15 @@ public static int run(String[] argv) throws Exception { o.step = opts.containsKey("step"); o.confirm = first(opts, "confirm"); o.restart = !opts.containsKey("no-restart"); + // --driver-server "D=cn=vault02,ou=servers,o=acme": the server a driver new to this environment runs on + Map runsOn = new java.util.LinkedHashMap<>(); + for (String v : opts.getOrDefault("driver-server", List.of())) { + int eq = v.indexOf('='); + if (eq > 0) { + runsOn.put(v.substring(0, eq).trim(), v.substring(eq + 1).trim()); + } + } + o.driverServers = runsOn; o.secretsMode = opts.containsKey("secrets") ? first(opts, "secrets") : "none"; o.allowMissingSecrets = opts.containsKey("allow-missing-secrets"); o.captureDrift = opts.containsKey("capture-drift"); diff --git a/src/main/java/com/pointblue/dirxml/dev/deploy/Deployer.java b/src/main/java/com/pointblue/dirxml/dev/deploy/Deployer.java index 42d9a18..f389a6e 100644 --- a/src/main/java/com/pointblue/dirxml/dev/deploy/Deployer.java +++ b/src/main/java/com/pointblue/dirxml/dev/deploy/Deployer.java @@ -68,6 +68,8 @@ public static final class Options { public boolean captureOnly; public boolean json; public List deleteDrivers = List.of(); + /** A driver new to the environment (the tree's name) to the server it runs on, when not the one the deploy connects through. */ + public Map driverServers = Map.of(); /** {@code --delete-all} kinds ({@code entitlements}, {@code forms}, {@code prds}), repeatable. */ public List deleteAllKinds = List.of(); public int restartWaitSeconds = 180; @@ -284,6 +286,12 @@ public Result run() throws IOException { } // the drivers asked for, as this environment's vault names them (the request uses the tree's names) List scope = overrides.vaultNames(o.drivers); + for (Map.Entry ds : o.driverServers.entrySet()) { + com.pointblue.dirxml.dev.model.Driver d = to.driver(overrides.renamed.getOrDefault(ds.getKey(), ds.getKey())); + if (d != null && ds.getValue() != null && !ds.getValue().isBlank()) { + d.runsOn = ds.getValue().trim(); + } + } String treeCommit = DeployLog.treeCommit(o.tree); Secrets secrets = (env.secretsFile == null ? Secrets.none() : Secrets.load(env.secretsFile)).withNames(overrides.renamed); Overrides.applySecretShimAuthIds(to, secrets); // ids the secrets file supplies win over the tree's (never shown) @@ -565,7 +573,19 @@ private void execute(VaultAccess vault, Plan.Step s, Secrets secrets, Result r) } break; case START_OPTION: - vault.setDriverStartOption(s.dn, Vault.START_MANUAL); + if (s.server != null) { + // created through the primary: wait for the replica on this server before setting its start option there + long until = System.currentTimeMillis() + 120_000; + while (!vault.exists(s.dn) && System.currentTimeMillis() < until) { + try { + Thread.sleep(2000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } + } + } + vault.setDriverStartOption(s.dn, "disabled".equals(s.attr) ? Vault.START_DISABLED : Vault.START_MANUAL); break; case NOTIFY_JOB: vault.notifyJobUpdate(s.dn); diff --git a/src/main/java/com/pointblue/dirxml/dev/deploy/Plan.java b/src/main/java/com/pointblue/dirxml/dev/deploy/Plan.java index 949002e..48197f8 100644 --- a/src/main/java/com/pointblue/dirxml/dev/deploy/Plan.java +++ b/src/main/java/com/pointblue/dirxml/dev/deploy/Plan.java @@ -479,7 +479,23 @@ public static Plan of(ModelDiff diff, DriverSet to, String dsDn, Secrets secrets ea, edn + " " + VaultMapping.OC_ENTITLEMENT + " (" + size(ea) + ")", c.path, c.driver)); } driversNeedingLinkage.add(c.driver); - driverAttrs.add(new Step(Op.START_OPTION, dn, null, null, null, dn + " start option = manual", c.path, c.driver)); + if (d.runsOn == null || d.runsOn.equalsIgnoreCase(p.primaryServer)) { + driverAttrs.add(new Step(Op.START_OPTION, dn, null, null, null, dn + " start option = manual", c.path, c.driver)); + } else { + // a new driver that runs on another server of the set: manual there, disabled everywhere else + driverAttrs.add(new Step(Op.START_OPTION, dn, "manual", null, null, dn + " start option = manual (runs on " + d.runsOn + ")", c.path, c.driver, null, d.runsOn)); + java.util.Set others = new java.util.LinkedHashSet<>(); + if (p.primaryServer != null) { + others.add(p.primaryServer); + } + others.addAll(to.servers); + for (String server : others) { + if (!server.equalsIgnoreCase(d.runsOn)) { + driverAttrs.add(new Step(Op.START_OPTION, dn, "disabled", null, null, dn + " start option = disabled", c.path, c.driver, null, + server.equalsIgnoreCase(p.primaryServer) ? null : server)); + } + } + } // secrets a new driver needs for (SecretInventory.Need need : SecretInventory.forDriver(to, d)) { if (secrets.has(need.key)) { diff --git a/src/main/java/com/pointblue/dirxml/dev/model/Driver.java b/src/main/java/com/pointblue/dirxml/dev/model/Driver.java index 3c9a7c6..2480259 100644 --- a/src/main/java/com/pointblue/dirxml/dev/model/Driver.java +++ b/src/main/java/com/pointblue/dirxml/dev/model/Driver.java @@ -112,6 +112,11 @@ public static String iconExtensionOf(byte[] b) { * the others ({@code docs/vault-deploy.md}, "Drivers that only some environments get"). */ public final List environments = new ArrayList<>(); + /** + * A deploy's choice, never written to the tree: the server a driver new to the environment runs on, when not the + * one the deploy connects through. That server gets the start option, every other server of the set "disabled". + */ + public String runsOn; /** True when a deploy to {@code env} gets this driver: no list, or {@code env} in it (any case). */ public boolean deploysTo(String env) { diff --git a/src/test/java/com/pointblue/dirxml/dev/deploy/PlanTest.java b/src/test/java/com/pointblue/dirxml/dev/deploy/PlanTest.java index 8361ebb..e15626d 100644 --- a/src/test/java/com/pointblue/dirxml/dev/deploy/PlanTest.java +++ b/src/test/java/com/pointblue/dirxml/dev/deploy/PlanTest.java @@ -24,6 +24,7 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; /** The plan builder: steps, order, grouping, linkage, secrets, restarts — pure, no vault. */ @@ -224,6 +225,37 @@ public void newDriverIsCreatedStoppedWithItsSecrets() throws IOException { assertTrue(p.text("stg", DS).contains("created stopped")); } + /** A new driver deployed to run on another server of the set: manual there, disabled on the others. */ + @Test + public void aNewDriverRunsOnTheServerChosen() throws IOException { + DriverSet from = VaultMappingTest.model(); + DriverSet to = VaultMappingTest.model(); + String p1 = "cn=VAULT01,ou=servers,o=acme"; + String p2 = "cn=VAULT02,ou=servers,o=acme"; + String p3 = "cn=VAULT03,ou=servers,o=acme"; + to.servers.addAll(List.of(p1, p2, p3)); + to.meta.put(Servers.PRIMARY_META, p1); + Driver n = new Driver("New"); + n.shimClass = "com.example.Shim"; + n.config.put(Driver.DRIVER_FILTER, ValidatorTest.xml("")); + n.runsOn = p2; + to.drivers.add(n); + + Plan p = plan(from, to, Secrets.none(), "none"); + List starts = p.steps.stream().filter(st -> st.op == Plan.Op.START_OPTION).toList(); + assertEquals(starts.toString(), 3, starts.size()); + Plan.Step on = starts.stream().filter(st -> "manual".equals(st.attr)).findFirst().orElseThrow(); + assertEquals(p2, on.server); + assertTrue(starts.stream().anyMatch(st -> "disabled".equals(st.attr) && st.server == null)); // the primary, through the deploy's own connection + assertTrue(starts.stream().anyMatch(st -> "disabled".equals(st.attr) && p3.equals(st.server))); + + n.runsOn = null; // no choice: manual on the primary, as before + Plan q = plan(from, to, Secrets.none(), "none"); + List one = q.steps.stream().filter(st -> st.op == Plan.Op.START_OPTION).toList(); + assertEquals(1, one.size()); + assertNull(one.get(0).server); + } + @Test public void forcedSecretsOnExistingDrivers() throws IOException { DriverSet from = VaultMappingTest.model(); From 808141e32e7cc44480ed38ade0aca36bcf3c315f Mon Sep 17 00:00:00 2001 From: Jerry Combs Date: Tue, 20 Oct 2026 15:01:38 -0500 Subject: [PATCH 06/18] WIP: planned environments on drivers, jobs, policies, templates; set objects filtered per environment; docs on feeds Co-Authored-By: Claude Opus 5.5 --- docs/existing-environments.md | 34 ++++++ .../dirxml/dev/ascode/AsCodeReader.java | 16 +-- .../dirxml/dev/ascode/AsCodeWriter.java | 15 ++- .../dirxml/dev/deploy/DriverEnvironments.java | 101 ++++++++++++++++++ .../dirxml/dev/deploy/Overrides.java | 6 ++ .../pointblue/dirxml/dev/edit/ConfigOps.java | 14 +-- .../pointblue/dirxml/dev/edit/Registry.java | 4 +- .../pointblue/dirxml/dev/model/Driver.java | 15 ++- .../dirxml/dev/model/EntitlementPolicy.java | 4 + .../dirxml/dev/model/EnvironmentList.java | 47 ++++++++ .../com/pointblue/dirxml/dev/model/Job.java | 4 + .../dev/model/NotificationTemplate.java | 4 + 12 files changed, 234 insertions(+), 30 deletions(-) create mode 100644 src/main/java/com/pointblue/dirxml/dev/model/EnvironmentList.java diff --git a/docs/existing-environments.md b/docs/existing-environments.md index f97e12d..f15ea92 100644 --- a/docs/existing-environments.md +++ b/docs/existing-environments.md @@ -102,6 +102,40 @@ defect the earlier analysis had found, which only that context explains. Resolving is then ordinary work on branches: each decision either leaves the tree, changes it on a branch, or records the difference as known. +## Drivers only some environments get: for now, or for good + +An item that only some environments get may be waiting (built in QA, not yet promoted to production) or +permanent (a test loopback, a feed that only exists in QA). The tree says which, beside the list: + +```xml + + +``` + +The same two attributes go on jobs, role-based entitlement policies and notification templates. A +compare against production lists the planned ones as waiting for promotion and leaves the permanent ones +quiet; promoting moves the environment from `planned` to `environments`, one reviewed change; a plan for +production names what is planned for it and not promoted yet. + +## Feeds between environments + +Keeping a lower environment's data current from production is common: an eDirectory-to-eDirectory link +whose sending end is a driver in production (for example "VaultProdToVaultQA") and whose receiving end is +a driver in QA ("VaultQAToVaultProd"); QA may feed dev the same way ("VaultQAToVaultDev"). Each end is its +own driver. Two ways to keep them: + +- **Each end its environment's, for good** (`environments="PROD"`, `environments="QA"`): the simple + default, and right when the ends differ in more than where they point. +- **One tree driver per role, named per environment**: "send to the environment below" is the same + configuration in production (VaultProdToVaultQA) and QA (VaultQAToVaultDev), aimed at a different tree. + One tree driver with `drivers/.name` per environment and the remote tree's address as each + environment's override keeps a fix to one end in step with the other. + +Names that carry environment names, and a remote address that is another environment of the project, are +how the setup recognises a feed; it proposes, and a person decides. A feed's remote address must be an +override in every environment that gets the driver (validation: `feed-remote-not-overridden`), so a +deploy never points one environment's feed at another's target. + ## Known differences (D1) A file per environment beside the overrides, `overrides/.known.json`, listing differences diff --git a/src/main/java/com/pointblue/dirxml/dev/ascode/AsCodeReader.java b/src/main/java/com/pointblue/dirxml/dev/ascode/AsCodeReader.java index b2ca2ea..6c10724 100644 --- a/src/main/java/com/pointblue/dirxml/dev/ascode/AsCodeReader.java +++ b/src/main/java/com/pointblue/dirxml/dev/ascode/AsCodeReader.java @@ -84,14 +84,8 @@ private static Driver readDriver(Path dir) throws IOException { d.shimClass = attr(m, "shim-class"); d.shimAuthServer = attr(m, "shim-auth-server"); d.shimAuthId = attr(m, "shim-auth-id"); - String environments = attr(m, "environments"); - if (environments != null) { - for (String e : environments.split(",")) { - if (!e.isBlank()) { - d.environments.add(e.trim()); - } - } - } + d.environments.addAll(com.pointblue.dirxml.dev.model.EnvironmentList.parse(attr(m, "environments"))); + d.planned.addAll(com.pointblue.dirxml.dev.model.EnvironmentList.parse(attr(m, "planned"))); readMeta(m, d.meta); for (Element c : children(m, "config")) { d.config.put(attr(c, "kind"), xml(dir.resolve(attr(c, "file")))); @@ -340,6 +334,8 @@ private static void readJobs(Element manifest, Path owner, List kinds = new ArrayList<>(d.config.keySet()); @@ -417,7 +418,9 @@ private static void writeJobs(List jobs, Pat if (j.definition != null) { writeXml(jobDir.resolve(file), j.definition); } - Manifest jm = m.child("job").attr("name", j.name).attr("file", "jobs/" + file); + Manifest jm = m.child("job").attr("name", j.name).attr("file", "jobs/" + file) + .attr("environments", com.pointblue.dirxml.dev.model.EnvironmentList.format(j.environments)) + .attr("planned", com.pointblue.dirxml.dev.model.EnvironmentList.format(j.planned)); for (String s : j.servers) { jm.child("server").attr("dn", s); } @@ -446,7 +449,9 @@ private static void writeRbePolicies(List}, {@code templates/}, {@code jobs/}, {@code drivers//jobs/}). + */ + public static List filterObjects(DriverSet ds, String env) { + List out = new ArrayList<>(); + if (env == null) { + return out; + } + drop(ds.rbePolicies, p -> p.environments, p -> "rbe-policies/" + p.name, env, out); + drop(ds.templates, t -> t.environments, t -> "templates/" + t.name, env, out); + drop(ds.jobs, j -> j.environments, j -> "jobs/" + j.name, env, out); + for (Driver d : ds.drivers) { + drop(d.jobs, j -> j.environments, j -> "drivers/" + d.name + "/jobs/" + j.name, env, out); + } + return out; + } + + private static void drop(List list, Function> envs, Function path, String env, List out) { + for (T o : new ArrayList<>(list)) { + if (!EnvironmentList.includes(envs.apply(o), env)) { + list.remove(o); + out.add(path.apply(o)); + } + } + } + /** Removes from {@code ds}, in place, the drivers a deploy to {@code env} doesn't get; returns their names. */ public static List filter(DriverSet ds, String env) { List out = new ArrayList<>(); @@ -74,11 +104,19 @@ public static List keepOnImport(DriverSet live, String env, DriverSet ba } else if (theirs != null) { theirs.environments.clear(); theirs.environments.addAll(mine.environments); + theirs.planned.clear(); + theirs.planned.addAll(mine.planned); if (mine.dn != null) { theirs.dn = mine.dn; // the DN the tree recorded: another environment's set may sit elsewhere } + keep(theirs.jobs, mine.jobs, j -> j.name, j -> j.environments, env, markNew, "'" + mine.name + "' job", notes); } } + if (names == null) { + keep(live.rbePolicies, base.rbePolicies, p -> p.name, p -> p.environments, env, markNew, "entitlement policy", notes); + keep(live.templates, base.templates, t -> t.name, t -> t.environments, env, markNew, "template", notes); + keep(live.jobs, base.jobs, j -> j.name, j -> j.environments, env, markNew, "job", notes); + } if (markNew) { for (Driver d : live.drivers) { if ((names == null || names.contains(d.name)) && base.driver(d.name) == null && d.environments.isEmpty()) { @@ -90,4 +128,67 @@ public static List keepOnImport(DriverSet live, String env, DriverSet ba } return notes; } + + /** + * One list of named objects read from {@code env}'s vault, against the tree's: those the tree doesn't deploy to + * {@code env} stay as the tree has them, the others keep their lists, and (with {@code markNew}) one only the vault + * has is marked as {@code env}'s. A run of new ones is summed up in one note. + */ + private static void keep(List live, List base, Function name, Function> envs, + String env, boolean markNew, String what, List notes) { + for (T mine : base) { + T theirs = find(live, name, name.apply(mine)); + if (theirs != null) { + planned(theirs).clear(); + planned(theirs).addAll(planned(mine)); + } + if (!EnvironmentList.includes(envs.apply(mine), env)) { + if (theirs != null) { + live.remove(theirs); + } + live.add(mine); + } else if (theirs != null) { + envs.apply(theirs).clear(); + envs.apply(theirs).addAll(envs.apply(mine)); + } + } + if (markNew) { + int marked = 0; + for (T o : live) { + if (find(base, name, name.apply(o)) == null && envs.apply(o).isEmpty()) { + envs.apply(o).add(env); + marked++; + } + } + if (marked > 0) { + notes.add(marked + " " + what + (marked == 1 ? "" : "s") + " new to the tree: marked as " + env + "'s only"); + } + } + } + + /** The object's planned list (drivers' jobs, the set's jobs, policies, templates). */ + static List planned(Object o) { + if (o instanceof com.pointblue.dirxml.dev.model.Job j) { + return j.planned; + } + if (o instanceof com.pointblue.dirxml.dev.model.EntitlementPolicy p) { + return p.planned; + } + if (o instanceof com.pointblue.dirxml.dev.model.NotificationTemplate t) { + return t.planned; + } + if (o instanceof Driver d) { + return d.planned; + } + return new ArrayList<>(); + } + + private static T find(List list, Function name, String n) { + for (T o : list) { + if (name.apply(o).equals(n)) { + return o; + } + } + return null; + } } diff --git a/src/main/java/com/pointblue/dirxml/dev/deploy/Overrides.java b/src/main/java/com/pointblue/dirxml/dev/deploy/Overrides.java index ab2ff03..0e8dbe8 100644 --- a/src/main/java/com/pointblue/dirxml/dev/deploy/Overrides.java +++ b/src/main/java/com/pointblue/dirxml/dev/deploy/Overrides.java @@ -292,6 +292,8 @@ public static final class Applied { public final List problems = new ArrayList<>(); /** Drivers the tree doesn't deploy to this environment ({@link DriverEnvironments}), left out of the model. */ public final List notDeployed = new ArrayList<>(); + /** Entitlement policies, templates and jobs the tree doesn't deploy to this environment, by tree path. */ + public final List notDeployedObjects = new ArrayList<>(); /** The tree's name to this environment's, for drivers its vault names differently. */ public final Map renamed = new java.util.LinkedHashMap<>(); /** The tree's servers to this environment's ({@link ServerMap}); empty for the base environment. */ @@ -337,6 +339,9 @@ public String summary() { if (!notDeployed.isEmpty()) { parts.add("not deployed to '" + env + "' (driver.xml environments): " + String.join(", ", notDeployed)); } + if (!notDeployedObjects.isEmpty()) { + parts.add(notDeployedObjects.size() + " other object(s) not deployed to '" + env + "' (environments)"); + } return parts.isEmpty() ? null : String.join("; ", parts); } } @@ -352,6 +357,7 @@ public static Applied apply(DriverSet ds, String env) { Map values = ds.overrides.get(env); // the drivers this environment doesn't get leave the model after the values, so a value set for one is still placed a.notDeployed.addAll(DriverEnvironments.filter(ds, env)); + a.notDeployedObjects.addAll(DriverEnvironments.filterObjects(ds, env)); // and the names this environment's vault uses come last: every key above names the tree's driver for (Map.Entry r : renames(values).entrySet()) { if (a.notDeployed.contains(r.getKey())) { diff --git a/src/main/java/com/pointblue/dirxml/dev/edit/ConfigOps.java b/src/main/java/com/pointblue/dirxml/dev/edit/ConfigOps.java index da3cf43..1a2f504 100644 --- a/src/main/java/com/pointblue/dirxml/dev/edit/ConfigOps.java +++ b/src/main/java/com/pointblue/dirxml/dev/edit/ConfigOps.java @@ -436,16 +436,16 @@ public void apply(DriverSet ds, Transaction tx) throws Refusal, IOException { case "shim-class": d.shimClass = value; return; case "shim-auth-server": d.shimAuthServer = value; return; case "shim-auth-id": d.shimAuthId = value; return; - case "environments": { + case "environments": // the environments this driver is deployed to, comma-separated; empty = every environment d.environments.clear(); - for (String e : (value == null ? "" : value).split(",")) { - if (!e.isBlank() && d.environments.stream().noneMatch(x -> x.equalsIgnoreCase(e.trim()))) { - d.environments.add(e.trim()); - } - } + d.environments.addAll(com.pointblue.dirxml.dev.model.EnvironmentList.parse(value)); + return; + case "planned": + // environments it is meant to reach but isn't promoted to yet; empty = its list is for good + d.planned.clear(); + d.planned.addAll(com.pointblue.dirxml.dev.model.EnvironmentList.parse(value)); return; - } default: } if (key.startsWith("param:") || key.startsWith("engine:")) { diff --git a/src/main/java/com/pointblue/dirxml/dev/edit/Registry.java b/src/main/java/com/pointblue/dirxml/dev/edit/Registry.java index 25dd46f..9c2b0b1 100644 --- a/src/main/java/com/pointblue/dirxml/dev/edit/Registry.java +++ b/src/main/java/com/pointblue/dirxml/dev/edit/Registry.java @@ -562,9 +562,9 @@ drv, opt("nds-class", "eDirectory class name"), opt("app-class", "application cl req("to", "the server to copy to: its DN, its name, or primary"), opt("driver", "one driver (default: every driver)"), opt("only", "some of gcv,shim,ecv,auth-server,auth-id (default: all)")); - register("driver.set", "set a driver setting: shim-class, shim-auth-server, shim-auth-id, environments (the environments it is deployed to, comma-separated; --clear = every environment), param:, engine:", + register("driver.set", "set a driver setting: shim-class, shim-auth-server, shim-auth-id, environments (the environments it is deployed to, comma-separated; --clear = every environment), planned (environments it is meant to reach but isn't promoted to yet), param:, engine:", a -> new ConfigOps.DriverSet_(a.get("driver"), a.get("key"), a.containsKey("clear") ? "" : a.get("value")), - drv, req("key", "shim-class|shim-auth-server|shim-auth-id|environments|param:|engine:"), opt("value", "the value"), + drv, req("key", "shim-class|shim-auth-server|shim-auth-id|environments|planned|param:|engine:"), opt("value", "the value"), opt("clear", "flag: make the setting empty (an empty --value reads as missing on the command line)")); register("driver.setting.add", "add a shim parameter (param:, into a section of the shim config) or an engine control value (engine:): its definition with display name, type and value", a -> new ConfigOps.DriverSettingAdd(a.get("driver"), a.get("key"), a.get("type"), a.get("display"), a.get("value"), a.get("section"), a.get("description")), diff --git a/src/main/java/com/pointblue/dirxml/dev/model/Driver.java b/src/main/java/com/pointblue/dirxml/dev/model/Driver.java index 2480259..eacee38 100644 --- a/src/main/java/com/pointblue/dirxml/dev/model/Driver.java +++ b/src/main/java/com/pointblue/dirxml/dev/model/Driver.java @@ -112,6 +112,11 @@ public static String iconExtensionOf(byte[] b) { * the others ({@code docs/vault-deploy.md}, "Drivers that only some environments get"). */ public final List environments = new ArrayList<>(); + /** + * Environments the driver is meant to reach but isn't promoted to yet ({@code driver.xml}'s {@code planned}): a + * driver not yet in production names it here; one that never will be leaves it empty. + */ + public final List planned = new ArrayList<>(); /** * A deploy's choice, never written to the tree: the server a driver new to the environment runs on, when not the * one the deploy connects through. That server gets the start option, every other server of the set "disabled". @@ -120,15 +125,7 @@ public static String iconExtensionOf(byte[] b) { /** True when a deploy to {@code env} gets this driver: no list, or {@code env} in it (any case). */ public boolean deploysTo(String env) { - if (environments.isEmpty() || env == null) { - return true; - } - for (String e : environments) { - if (e.equalsIgnoreCase(env)) { - return true; - } - } - return false; + return EnvironmentList.includes(environments, env); } public Driver(String name) { diff --git a/src/main/java/com/pointblue/dirxml/dev/model/EntitlementPolicy.java b/src/main/java/com/pointblue/dirxml/dev/model/EntitlementPolicy.java index 56f2c1d..e944862 100644 --- a/src/main/java/com/pointblue/dirxml/dev/model/EntitlementPolicy.java +++ b/src/main/java/com/pointblue/dirxml/dev/model/EntitlementPolicy.java @@ -52,6 +52,10 @@ public final class EntitlementPolicy { public Integer priority; /** DN, Designer id, package stamps, and other source-specific extras. */ public final Map meta = new LinkedHashMap<>(); + /** The environments this is deployed to; empty = every environment ({@link EnvironmentList}). */ + public final java.util.List environments = new java.util.ArrayList<>(); + /** Environments it is meant to reach but isn't promoted to yet; empty = none, its list is for good. */ + public final java.util.List planned = new java.util.ArrayList<>(); public EntitlementPolicy(String name) { this.name = name; diff --git a/src/main/java/com/pointblue/dirxml/dev/model/EnvironmentList.java b/src/main/java/com/pointblue/dirxml/dev/model/EnvironmentList.java new file mode 100644 index 0000000..a76a3b9 --- /dev/null +++ b/src/main/java/com/pointblue/dirxml/dev/model/EnvironmentList.java @@ -0,0 +1,47 @@ +package com.pointblue.dirxml.dev.model; + +import java.util.ArrayList; +import java.util.List; + +/** + * The environments an object is deployed to ({@code environments="QA, PROD"} on its manifest entry): a driver, a job, + * a role-based entitlement policy, a notification template. Empty means every environment + * ({@code docs/existing-environments.md}). + */ +public final class EnvironmentList { + + private EnvironmentList() { + } + + /** True when a deploy to {@code env} gets the object: no list, no environment named, or {@code env} in it (any case). */ + public static boolean includes(List environments, String env) { + if (environments.isEmpty() || env == null) { + return true; + } + for (String e : environments) { + if (e.equalsIgnoreCase(env)) { + return true; + } + } + return false; + } + + /** The attribute's value, or null for every environment (no attribute written). */ + public static String format(List environments) { + return environments.isEmpty() ? null : String.join(", ", environments); + } + + /** The attribute's value as a list, trimmed, duplicates (any case) dropped; null or blank = every environment. */ + public static List parse(String value) { + List out = new ArrayList<>(); + if (value != null) { + for (String e : value.split(",")) { + String t = e.trim(); + if (!t.isEmpty() && out.stream().noneMatch(x -> x.equalsIgnoreCase(t))) { + out.add(t); + } + } + } + return out; + } +} diff --git a/src/main/java/com/pointblue/dirxml/dev/model/Job.java b/src/main/java/com/pointblue/dirxml/dev/model/Job.java index b0d9af3..4354b76 100644 --- a/src/main/java/com/pointblue/dirxml/dev/model/Job.java +++ b/src/main/java/com/pointblue/dirxml/dev/model/Job.java @@ -30,6 +30,10 @@ public final class Job { public final List scopes = new ArrayList<>(); /** Trace settings ({@code trace-level}, {@code trace-file}, {@code trace-size-limit}), DN, Designer id, package stamps, and other source-specific extras. */ public final Map meta = new LinkedHashMap<>(); + /** The environments this is deployed to; empty = every environment ({@link EnvironmentList}). */ + public final java.util.List environments = new java.util.ArrayList<>(); + /** Environments it is meant to reach but isn't promoted to yet; empty = none, its list is for good. */ + public final java.util.List planned = new java.util.ArrayList<>(); public Job(String name, Element definition) { this.name = name; diff --git a/src/main/java/com/pointblue/dirxml/dev/model/NotificationTemplate.java b/src/main/java/com/pointblue/dirxml/dev/model/NotificationTemplate.java index 5c17e19..a3b9a22 100644 --- a/src/main/java/com/pointblue/dirxml/dev/model/NotificationTemplate.java +++ b/src/main/java/com/pointblue/dirxml/dev/model/NotificationTemplate.java @@ -25,6 +25,10 @@ public final class NotificationTemplate { public Element data; /** DN, package stamps, and other source-specific extras. */ public final Map meta = new LinkedHashMap<>(); + /** The environments this is deployed to; empty = every environment ({@link EnvironmentList}). */ + public final java.util.List environments = new java.util.ArrayList<>(); + /** Environments it is meant to reach but isn't promoted to yet; empty = none, its list is for good. */ + public final java.util.List planned = new java.util.ArrayList<>(); public NotificationTemplate(String name, Element data) { this.name = name; From 25f7a3c86cc2aabe09bf554971f28cbdedbc3a2b Mon Sep 17 00:00:00 2001 From: Jerry Combs Date: Tue, 20 Oct 2026 15:05:31 -0500 Subject: [PATCH 07/18] WIP: likely causes for environment drift; one-sided set objects in only-in Co-Authored-By: Claude Opus 5.5 --- .../dirxml/dev/deploy/EnvironmentCompare.java | 161 ++++++++++++++++-- 1 file changed, 149 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/pointblue/dirxml/dev/deploy/EnvironmentCompare.java b/src/main/java/com/pointblue/dirxml/dev/deploy/EnvironmentCompare.java index f075495..330b231 100644 --- a/src/main/java/com/pointblue/dirxml/dev/deploy/EnvironmentCompare.java +++ b/src/main/java/com/pointblue/dirxml/dev/deploy/EnvironmentCompare.java @@ -30,19 +30,30 @@ */ public final class EnvironmentCompare { - /** A driver one environment's vault holds and the other's doesn't. */ + /** + * Something one environment's vault holds and the other's doesn't: a driver ({@code driver} set, {@code path} + * {@code drivers/}), or a role-based entitlement policy, template or job ({@code path} {@code rbe-policies/}, + * {@code templates/}, {@code jobs/}, {@code drivers//jobs/}). + */ public static final class OnlyIn { public final String driver; + public final String path; + public final String kind; public final String env; - /** The tree doesn't deploy it to the other environment. */ + /** The tree doesn't deploy it to the other environment, for good or for now. */ public final boolean expected; - /** The tree has no driver of this name. */ + /** The tree means it to reach the other environment later ({@code planned}): waiting for promotion. */ + public final boolean planned; + /** The tree has nothing of this name. */ public final boolean notInTree; - OnlyIn(String driver, String env, boolean expected, boolean notInTree) { + OnlyIn(String driver, String path, String kind, String env, boolean expected, boolean planned, boolean notInTree) { this.driver = driver; + this.path = path; + this.kind = kind; this.env = env; this.expected = expected; + this.planned = planned; this.notInTree = notInTree; } } @@ -71,6 +82,8 @@ public static final class Value { public final List onlyIn = new ArrayList<>(); public final List values = new ArrayList<>(); public ModelDiff drift; + /** For each change of {@link #drift}, in order, its likely cause ({@link #cause}): the analysis a person starts from. */ + public final List causes = new ArrayList<>(); /** Changes whose every changed line differs only in letter case (a DN's {@code OU=services} against {@code OU=SERVICES}): eDirectory reads them as the same. Kept apart from the drift. */ public final List caseOnly = new ArrayList<>(); public final List notes = new ArrayList<>(); @@ -123,15 +136,91 @@ public static EnvironmentCompare of(DriverSet a, String envA, DriverSet b, Strin // the drivers only one side holds are listed above; what's left of the rest is drift, but for differences of letter case ModelDiff rest = ModelDiff.of(a, b).only(ch -> ch.kind != ModelDiff.Kind.DRIVER_ADDED && ch.kind != ModelDiff.Kind.DRIVER_REMOVED && (ch.driver == null || (a.driver(ch.driver) != null && b.driver(ch.driver) != null))); + List oneSided = new ArrayList<>(); for (ModelDiff.Change ch : rest.changes()) { - if (onlyLetterCase(ch.detail)) { + boolean setObject = ch.kind.isRbe() || ch.kind.isTemplate() || ch.kind.isJob(); + boolean added = ch.kind.name().endsWith("_ADDED"); + boolean removed = ch.kind.name().endsWith("_REMOVED"); + if (setObject && (added || removed)) { + // from = a, to = b: "removed" is a's only, "added" b's only + c.onlyIn.add(objectOnlyIn(ch, removed ? envA : envB, removed ? envB : envA, tree)); + oneSided.add(ch); + } else if (onlyLetterCase(ch.detail)) { c.caseOnly.add(ch); } } - c.drift = rest.only(ch -> !c.caseOnly.contains(ch)); + c.drift = rest.only(ch -> !c.caseOnly.contains(ch) && !oneSided.contains(ch)); + // words that mark environment-specific text: the environments' names and their servers' names + java.util.Set envWords = new java.util.HashSet<>(); + envWords.add(envA.toLowerCase(java.util.Locale.ROOT)); + envWords.add(envB.toLowerCase(java.util.Locale.ROOT)); + for (DriverSet side : List.of(a, b)) { + for (String server : side.servers) { + String rdn = server.split(",")[0]; + envWords.add(rdn.substring(rdn.indexOf('=') + 1).toLowerCase(java.util.Locale.ROOT)); + } + } + for (ModelDiff.Change ch : c.drift.changes()) { + c.causes.add(cause(ch, envA, envB, envWords)); + } return c; } + /** + * A change's likely cause, from the change alone: one side only ("only in A"/"only in B"), lines only one side has, + * package versions, cosmetic (descriptions, comments, display names), environment-specific text (a name of either + * environment or of one of their servers on a changed line), or "both changed" (a person reviews it). + */ + static String cause(ModelDiff.Change ch, String envA, String envB, java.util.Set envWords) { + String k = ch.kind.name(); + if (k.endsWith("_REMOVED")) { + return "only in " + envA; + } + if (k.endsWith("_ADDED")) { + return "only in " + envB; + } + if (ch.kind == ModelDiff.Kind.DRIVER_STAMPS) { + return "package versions differ"; + } + if (ch.detail == null) { + return "both changed"; + } + List fromA = new ArrayList<>(); + List fromB = new ArrayList<>(); + for (String line : ch.detail.split("\n")) { + String t = line.stripLeading(); + if (t.startsWith("- ")) { + fromA.add(t.substring(2)); + } else if (t.startsWith("+ ")) { + fromB.add(t.substring(2)); + } + } + if (fromA.isEmpty() && fromB.isEmpty()) { + return "both changed"; + } + if (fromA.isEmpty()) { + return "lines only " + envB + " has"; + } + if (fromB.isEmpty()) { + return "lines only " + envA + " has"; + } + List all = new ArrayList<>(fromA); + all.addAll(fromB); + if (all.stream().allMatch(l -> l.contains("") || l.trim().startsWith("