From 2ced8764e1057fea6ec149c7f2878055302967df Mon Sep 17 00:00:00 2001 From: Brad Sickles Date: Wed, 7 Oct 2026 08:49:08 -0400 Subject: [PATCH 1/4] feat: scope tenants per env on a shared cluster SHARED_ENVS=true (set from the cluster's shared output) names every tenant role env.tenant and binds it to an entity carrying env and tenant metadata. Tenant policies then scope KV to envs//customers/ and the broker policies to the app's own env, which the cluster function writes on the app entity. Provisioning and apps-auth gain narrow identity grants with policies refused. Unshared clusters render the 0.1.x policies byte for byte (pinned by goldens). vault-utils: tenants create/destroy take --env (required when shared, refused otherwise), tenants list shows the env, new envs list/destroy, and env prints SHARED_ENVS from the workspace output. --- cmd/vault-utils/main.go | 141 +++++-- cmd/vault-utils/main_test.go | 38 ++ internal/nsenv/nsenv_test.go | 27 +- internal/nsenv/settings.go | 23 +- internal/vaultcluster/apps_test.go | 18 +- internal/vaultcluster/config.go | 65 +++- internal/vaultcluster/config_test.go | 65 ++++ internal/vaultcluster/credentials_test.go | 8 +- internal/vaultcluster/envs.go | 62 +++ internal/vaultcluster/isolation_test.go | 26 +- internal/vaultcluster/lint.go | 16 + internal/vaultcluster/lint_test.go | 4 +- .../policies/templates/apps-auth.hcl.tpl | 42 ++ .../policies/templates/apps-reader.hcl.tpl | 9 + .../policies/templates/apps-writer.hcl.tpl | 9 + .../policies/templates/provisioning.hcl.tpl | 41 ++ .../templates/tenant-database.hcl.tpl | 7 +- .../policies/templates/tenant-reader.hcl.tpl | 18 +- .../policies/templates/tenant-writer.hcl.tpl | 24 +- internal/vaultcluster/policy.go | 43 ++- internal/vaultcluster/policy_test.go | 115 ++++++ internal/vaultcluster/shared_test.go | 364 ++++++++++++++++++ internal/vaultcluster/tenant.go | 164 ++++++-- internal/vaultcluster/tenantid.go | 19 +- internal/vaultcluster/tenantlist.go | 55 ++- .../testdata/lint/BAD-cross-env-wildcard.hcl | 11 + .../testdata/lint/GOOD-env-scoped.hcl | 17 + .../testdata/policies/shared/apps-auth.hcl | 84 ++++ .../testdata/policies/shared/apps-reader.hcl | 40 ++ .../testdata/policies/shared/apps-writer.hcl | 40 ++ .../testdata/policies/shared/operator.hcl | 140 +++++++ .../testdata/policies/shared/provisioning.hcl | 158 ++++++++ .../policies/shared/tenant-database.hcl | 36 ++ .../policies/shared/tenant-reader.hcl | 61 +++ .../policies/shared/tenant-writer.hcl | 73 ++++ .../testdata/policies/unshared/apps-auth.hcl | 44 +++ .../policies/unshared/apps-reader.hcl | 41 ++ .../policies/unshared/apps-writer.hcl | 41 ++ .../testdata/policies/unshared/operator.hcl | 140 +++++++ .../policies/unshared/provisioning.hcl | 122 ++++++ .../policies/unshared/tenant-database.hcl | 36 ++ .../policies/unshared/tenant-reader.hcl | 61 +++ .../policies/unshared/tenant-writer.hcl | 73 ++++ 43 files changed, 2479 insertions(+), 142 deletions(-) create mode 100644 cmd/vault-utils/main_test.go create mode 100644 internal/vaultcluster/config_test.go create mode 100644 internal/vaultcluster/envs.go create mode 100644 internal/vaultcluster/policy_test.go create mode 100644 internal/vaultcluster/shared_test.go create mode 100644 internal/vaultcluster/testdata/lint/BAD-cross-env-wildcard.hcl create mode 100644 internal/vaultcluster/testdata/lint/GOOD-env-scoped.hcl create mode 100644 internal/vaultcluster/testdata/policies/shared/apps-auth.hcl create mode 100644 internal/vaultcluster/testdata/policies/shared/apps-reader.hcl create mode 100644 internal/vaultcluster/testdata/policies/shared/apps-writer.hcl create mode 100644 internal/vaultcluster/testdata/policies/shared/operator.hcl create mode 100644 internal/vaultcluster/testdata/policies/shared/provisioning.hcl create mode 100644 internal/vaultcluster/testdata/policies/shared/tenant-database.hcl create mode 100644 internal/vaultcluster/testdata/policies/shared/tenant-reader.hcl create mode 100644 internal/vaultcluster/testdata/policies/shared/tenant-writer.hcl create mode 100644 internal/vaultcluster/testdata/policies/unshared/apps-auth.hcl create mode 100644 internal/vaultcluster/testdata/policies/unshared/apps-reader.hcl create mode 100644 internal/vaultcluster/testdata/policies/unshared/apps-writer.hcl create mode 100644 internal/vaultcluster/testdata/policies/unshared/operator.hcl create mode 100644 internal/vaultcluster/testdata/policies/unshared/provisioning.hcl create mode 100644 internal/vaultcluster/testdata/policies/unshared/tenant-database.hcl create mode 100644 internal/vaultcluster/testdata/policies/unshared/tenant-reader.hcl create mode 100644 internal/vaultcluster/testdata/policies/unshared/tenant-writer.hcl diff --git a/cmd/vault-utils/main.go b/cmd/vault-utils/main.go index c04d776..35bf4c6 100644 --- a/cmd/vault-utils/main.go +++ b/cmd/vault-utils/main.go @@ -33,9 +33,11 @@ func usage() { Commands: bootstrap local|aws|azure|gcp Init once, unseal, configure - tenants create - tenants list - tenants destroy --yes [--purge-secrets] + tenants create [--env ] + tenants list [--env ] + tenants destroy [--env ] --yes [--purge-secrets] + envs list Envs with tenants (shared cluster only) + envs destroy --yes [--purge-secrets] snapshot take Write a Raft snapshot snapshot list snapshot verify @@ -45,7 +47,9 @@ Commands: health serve HTTP on :8210 (200 only if this node is a Raft voter and caught up) env --org --stack --env --block Print shell settings for a Nullstone Vault cluster workspace -Without VAULT_TOKEN, tenants, snapshot take|restore, and health use the token from "vault login". +Without VAULT_TOKEN, tenants, envs, snapshot take|restore, and health use the token from "vault login". +On a shared cluster (SHARED_ENVS=true, printed by "vault-utils env") every tenant belongs to an env: --env is +required there and refused elsewhere. Local key material: BOOTSTRAP_DIR (default .bootstrap). AWS: VAULT_INIT_SECRET_ARN, VAULT_PROVISIONING_SECRET_ARN, VAULT_OPERATOR_SECRET_ARN. @@ -74,6 +78,8 @@ func run(cmd string, args []string) error { return runBootstrap(c, args) case "tenants": return runTenants(c, args) + case "envs": + return runEnvs(c, args) case "snapshot": return runSnapshot(c, args) case "health": @@ -128,54 +134,129 @@ func runBootstrap(c *vaultcluster.Client, args []string) error { } } +// tenantArgs parses plus --env , --yes, --purge-secrets. Any other flag is an error. +type tenantArgs struct { + id, env string + yes, purge bool +} + +func parseTenantArgs(args []string) (tenantArgs, error) { + var out tenantArgs + for i := 0; i < len(args); i++ { + switch a := args[i]; { + case a == "--yes": + out.yes = true + case a == "--purge-secrets": + out.purge = true + case a == "--env": + if i+1 >= len(args) { + return out, fmt.Errorf("--env needs a value") + } + i++ + out.env = args[i] + case strings.HasPrefix(a, "--env="): + out.env = strings.TrimPrefix(a, "--env=") + case strings.HasPrefix(a, "-"): + return out, fmt.Errorf("unknown flag %q", a) + default: + out.id = a + } + } + return out, nil +} + +// checkEnvFlag enforces the cluster mode before any Vault call: --env is required on a shared cluster +// (SHARED_ENVS=true) and refused on an unshared one. +func checkEnvFlag(shared bool, env string) error { + if shared && env == "" { + return fmt.Errorf("this cluster is shared across envs; pass --env ") + } + if !shared && env != "" { + return fmt.Errorf("this cluster is not shared across envs (SHARED_ENVS is not true); drop --env") + } + return nil +} + func runTenants(c *vaultcluster.Client, args []string) error { if len(args) < 1 { return fmt.Errorf("usage: vault-utils tenants create|list|destroy") } sub, rest := args[0], args[1:] + ta, err := parseTenantArgs(rest) + if err != nil { + return err + } switch sub { case "create": - id := "" - for _, a := range rest { - if !strings.HasPrefix(a, "-") { - id = a - } + if ta.id == "" { + return fmt.Errorf("usage: vault-utils tenants create [--env ]") } - if id == "" { - return fmt.Errorf("usage: vault-utils tenants create ") + if err := checkEnvFlag(c.Cfg.SharedEnvs, ta.env); err != nil { + return err } - return c.CreateTenant(id) + return c.CreateTenant(ta.env, ta.id) case "list": tenants, err := c.ListTenants() if err != nil { return err } - vaultcluster.PrintTenants(os.Stdout, tenants) - return nil - case "destroy": - yes, purge := false, false - id := "" - for _, a := range rest { - switch a { - case "--yes": - yes = true - case "--purge-secrets": - purge = true - default: - if !strings.HasPrefix(a, "-") { - id = a + if ta.env != "" { + var kept []vaultcluster.Tenant + for _, t := range tenants { + if t.Env == ta.env { + kept = append(kept, t) } } + tenants = kept } - if id == "" || !yes { - return fmt.Errorf("usage: vault-utils tenants destroy --yes [--purge-secrets]") + vaultcluster.PrintTenants(os.Stdout, tenants, c.Cfg.SharedEnvs) + return nil + case "destroy": + if ta.id == "" || !ta.yes { + return fmt.Errorf("usage: vault-utils tenants destroy [--env ] --yes [--purge-secrets]") + } + if err := checkEnvFlag(c.Cfg.SharedEnvs, ta.env); err != nil { + return err } - return c.OffboardTenant(id, purge) + return c.OffboardTenant(ta.env, ta.id, ta.purge) default: return fmt.Errorf("unknown subcommand %q (create, list, destroy)", sub) } } +func runEnvs(c *vaultcluster.Client, args []string) error { + if len(args) < 1 { + return fmt.Errorf("usage: vault-utils envs list | destroy --yes [--purge-secrets]") + } + sub, rest := args[0], args[1:] + ta, err := parseTenantArgs(rest) + if err != nil { + return err + } + switch sub { + case "list": + envs, err := c.ListEnvs() + if err != nil { + return err + } + if len(envs) == 0 { + fmt.Println("no envs") + return nil + } + for _, e := range envs { + fmt.Println(e) + } + return nil + case "destroy": + if ta.id == "" || !ta.yes { + return fmt.Errorf("usage: vault-utils envs destroy --yes [--purge-secrets]") + } + return c.DestroyEnv(ta.id, ta.purge) + default: + return fmt.Errorf("unknown subcommand %q (list, destroy)", sub) + } +} + func runSnapshot(c *vaultcluster.Client, args []string) error { if len(args) < 1 { return fmt.Errorf("usage: vault-utils snapshot take | list | verify | restore --yes | schedule") @@ -352,7 +433,7 @@ func usesLoginToken(cmd string, args []string) bool { sub = args[0] } switch cmd { - case "tenants": + case "tenants", "envs": return true case "snapshot": return sub == "take" || sub == "restore" diff --git a/cmd/vault-utils/main_test.go b/cmd/vault-utils/main_test.go new file mode 100644 index 0000000..a83ca18 --- /dev/null +++ b/cmd/vault-utils/main_test.go @@ -0,0 +1,38 @@ +package main + +import "testing" + +func TestParseTenantArgs(t *testing.T) { + got, err := parseTenantArgs([]string{"acme", "--env", "pr-12", "--yes", "--purge-secrets"}) + if err != nil { + t.Fatal(err) + } + if got != (tenantArgs{id: "acme", env: "pr-12", yes: true, purge: true}) { + t.Fatalf("got %+v", got) + } + got, err = parseTenantArgs([]string{"--env=dev", "acme"}) + if err != nil || got.env != "dev" || got.id != "acme" { + t.Fatalf("got %+v, %v", got, err) + } + if _, err := parseTenantArgs([]string{"acme", "--env"}); err == nil { + t.Fatal("--env without a value was accepted") + } + if _, err := parseTenantArgs([]string{"acme", "--force"}); err == nil { + t.Fatal("unknown flag was accepted") + } +} + +func TestCheckEnvFlag(t *testing.T) { + if err := checkEnvFlag(true, ""); err == nil { + t.Fatal("shared cluster without --env was accepted") + } + if err := checkEnvFlag(false, "dev"); err == nil { + t.Fatal("unshared cluster with --env was accepted") + } + if err := checkEnvFlag(true, "dev"); err != nil { + t.Fatal(err) + } + if err := checkEnvFlag(false, ""); err != nil { + t.Fatal(err) + } +} diff --git a/internal/nsenv/nsenv_test.go b/internal/nsenv/nsenv_test.go index 52c62bb..e1df392 100644 --- a/internal/nsenv/nsenv_test.go +++ b/internal/nsenv/nsenv_test.go @@ -43,6 +43,15 @@ func TestSettingsFromOutputs(t *testing.T) { "vault_fqdn": "vault.internal", "vault_api_port": "8443", }, want: Settings{Addr: "http://vault.internal:8443"}, wantNote: true}, {name: "not applied", outputs: map[string]any{}, wantErr: true}, + {name: "shared cluster", outputs: map[string]any{ + "vault_fqdn": "vault.internal", "vault_api_port": "8200", "vault_addr": "http://vault.internal:8200", "shared": true, + }, want: Settings{Addr: "http://vault.internal:8200", SharedEnvs: true}, wantNote: true}, + {name: "shared as string", outputs: map[string]any{ + "vault_addr": "http://vault.internal:8200", "shared": "true", + }, want: Settings{Addr: "http://vault.internal:8200", SharedEnvs: true}, wantNote: true}, + {name: "unshared cluster", outputs: map[string]any{ + "vault_addr": "http://vault.internal:8200", "shared": false, + }, want: Settings{Addr: "http://vault.internal:8200"}, wantNote: true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -72,14 +81,16 @@ func TestRender(t *testing.T) { in Settings want string }{ - {"bash", withSNI, "export VAULT_ADDR='https://vault.internal:8200'\nexport VAULT_TLS_SERVER_NAME='vault.acme.example.com'\n"}, - {"zsh", plain, "export VAULT_ADDR='http://vault.internal:8200'\nunset VAULT_TLS_SERVER_NAME\n"}, - {"fish", withSNI, "set -gx VAULT_ADDR 'https://vault.internal:8200';\nset -gx VAULT_TLS_SERVER_NAME 'vault.acme.example.com';\n"}, - {"fish", plain, "set -gx VAULT_ADDR 'http://vault.internal:8200';\nset -e VAULT_TLS_SERVER_NAME;\n"}, - {"powershell", withSNI, "$env:VAULT_ADDR = 'https://vault.internal:8200'\n$env:VAULT_TLS_SERVER_NAME = 'vault.acme.example.com'\n"}, - {"powershell", plain, "$env:VAULT_ADDR = 'http://vault.internal:8200'\nRemove-Item Env:VAULT_TLS_SERVER_NAME -ErrorAction SilentlyContinue\n"}, - {"bash", Settings{Addr: "http://a'b"}, "export VAULT_ADDR='http://a'\\''b'\nunset VAULT_TLS_SERVER_NAME\n"}, - {"powershell", Settings{Addr: "http://a'b"}, "$env:VAULT_ADDR = 'http://a''b'\nRemove-Item Env:VAULT_TLS_SERVER_NAME -ErrorAction SilentlyContinue\n"}, + {"bash", withSNI, "export VAULT_ADDR='https://vault.internal:8200'\nexport VAULT_TLS_SERVER_NAME='vault.acme.example.com'\nunset SHARED_ENVS\n"}, + {"zsh", plain, "export VAULT_ADDR='http://vault.internal:8200'\nunset VAULT_TLS_SERVER_NAME\nunset SHARED_ENVS\n"}, + {"fish", withSNI, "set -gx VAULT_ADDR 'https://vault.internal:8200';\nset -gx VAULT_TLS_SERVER_NAME 'vault.acme.example.com';\nset -e SHARED_ENVS;\n"}, + {"fish", plain, "set -gx VAULT_ADDR 'http://vault.internal:8200';\nset -e VAULT_TLS_SERVER_NAME;\nset -e SHARED_ENVS;\n"}, + {"powershell", withSNI, "$env:VAULT_ADDR = 'https://vault.internal:8200'\n$env:VAULT_TLS_SERVER_NAME = 'vault.acme.example.com'\nRemove-Item Env:SHARED_ENVS -ErrorAction SilentlyContinue\n"}, + {"powershell", plain, "$env:VAULT_ADDR = 'http://vault.internal:8200'\nRemove-Item Env:VAULT_TLS_SERVER_NAME -ErrorAction SilentlyContinue\nRemove-Item Env:SHARED_ENVS -ErrorAction SilentlyContinue\n"}, + {"bash", Settings{Addr: "http://a'b"}, "export VAULT_ADDR='http://a'\\''b'\nunset VAULT_TLS_SERVER_NAME\nunset SHARED_ENVS\n"}, + {"powershell", Settings{Addr: "http://a'b"}, "$env:VAULT_ADDR = 'http://a''b'\nRemove-Item Env:VAULT_TLS_SERVER_NAME -ErrorAction SilentlyContinue\nRemove-Item Env:SHARED_ENVS -ErrorAction SilentlyContinue\n"}, + {"bash", Settings{Addr: "http://vault.internal:8200", SharedEnvs: true}, "export VAULT_ADDR='http://vault.internal:8200'\nunset VAULT_TLS_SERVER_NAME\nexport SHARED_ENVS='true'\n"}, + {"powershell", Settings{Addr: "http://vault.internal:8200", SharedEnvs: true}, "$env:VAULT_ADDR = 'http://vault.internal:8200'\nRemove-Item Env:VAULT_TLS_SERVER_NAME -ErrorAction SilentlyContinue\n$env:SHARED_ENVS = 'true'\n"}, } for _, tt := range tests { got, err := Render(tt.shell, tt.in) diff --git a/internal/nsenv/settings.go b/internal/nsenv/settings.go index 30e71a9..2b51c98 100644 --- a/internal/nsenv/settings.go +++ b/internal/nsenv/settings.go @@ -12,10 +12,23 @@ import ( type Settings struct { Addr string TLSServerName string + // SharedEnvs is the cluster's shared output: tenants are scoped per env and vault-utils needs --env. + SharedEnvs bool // Note explains a fallback, for stderr. Note string } +// sharedOutput reads the cluster's shared output, which older clusters lack (false). +func sharedOutput(out map[string]any) bool { + switch v := out["shared"].(type) { + case bool: + return v + case string: + return strings.EqualFold(strings.TrimSpace(v), "true") + } + return false +} + // SettingsFromOutputs prefers the user-facing address. internal selects the private name (vault_addr), which // needs the user-facing name for TLS verification when the load balancer terminates TLS. // Clusters older than the vault_addr outputs fall back to fqdn + port; they use TLS exactly when user_fqdn is set. @@ -44,10 +57,11 @@ func SettingsFromOutputs(ws Workspace, internal bool) (Settings, error) { if internalAddr == "" && userAddr == "" { return Settings{}, fmt.Errorf("workspace has no vault_addr, user_vault_addr, or vault_fqdn output; has it been applied?") } + shared := sharedOutput(out) if !internal && userAddr != "" { - return Settings{Addr: userAddr}, nil + return Settings{Addr: userAddr, SharedEnvs: shared}, nil } - s := Settings{Addr: internalAddr} + s := Settings{Addr: internalAddr, SharedEnvs: shared} if !internal { s.Note = "no subdomain is connected to this cluster; using " + vaultFQDN + ", which resolves only inside the cluster network" } @@ -93,6 +107,11 @@ func Render(shell string, s Settings) (string, error) { } else { unset("VAULT_TLS_SERVER_NAME") } + if s.SharedEnvs { + set("SHARED_ENVS", "true") + } else { + unset("SHARED_ENVS") + } return b.String(), nil } diff --git a/internal/vaultcluster/apps_test.go b/internal/vaultcluster/apps_test.go index 2db376f..8e17676 100644 --- a/internal/vaultcluster/apps_test.go +++ b/internal/vaultcluster/apps_test.go @@ -23,7 +23,7 @@ func TestAppsBrokerTokens(t *testing.T) { t.Fatal(err) } for _, id := range []string{"tenant-a", "tenant-b"} { - if err := c.CreateTenant(id); err != nil { + if err := c.CreateTenant("", id); err != nil { t.Fatal(err) } } @@ -61,7 +61,7 @@ func TestAppsBrokerTokens(t *testing.T) { if err != nil { t.Fatalf("writer broker login as tenant-a: %v", err) } - if _, err := c.WithToken(tokAW).API.Logical().Write(c.Cfg.KVDataPath("tenant-a", "fixture"), map[string]any{"data": map[string]any{"v": "FAKE-a"}}); err != nil { + if _, err := c.WithToken(tokAW).API.Logical().Write(c.Cfg.KVDataPath("", "tenant-a", "fixture"), map[string]any{"data": map[string]any{"v": "FAKE-a"}}); err != nil { t.Fatal(err) } @@ -76,11 +76,11 @@ func TestAppsBrokerTokens(t *testing.T) { if id == "tenant-b" { other = "tenant-a" } - if got := status(cl, "GET", c.Cfg.KVDataPath(id, "fixture"), nil); got != 200 && got != 404 { + if got := status(cl, "GET", c.Cfg.KVDataPath("", id, "fixture"), nil); got != 200 && got != 404 { t.Fatalf("%s reading its own secret: HTTP %d", id, got) } - deny(cl, "GET", c.Cfg.KVDataPath(other, "fixture"), nil) - deny(cl, "POST", c.Cfg.KVDataPath(id, "fixture"), map[string]any{"data": map[string]any{"v": "FAKE-write"}}) + deny(cl, "GET", c.Cfg.KVDataPath("", other, "fixture"), nil) + deny(cl, "POST", c.Cfg.KVDataPath("", id, "fixture"), map[string]any{"data": map[string]any{"v": "FAKE-write"}}) deny(cl, "GET", "auth/"+readerMount+"/role/"+other+"/role-id", nil) self, err := cl.API.Auth().Token().LookupSelf() @@ -102,7 +102,7 @@ func TestAppsBrokerTokens(t *testing.T) { } // The broker token itself holds no tenant access and cannot pick a level it was not given. - deny(reader, "GET", c.Cfg.KVDataPath("tenant-a", "fixture"), nil) + deny(reader, "GET", c.Cfg.KVDataPath("", "tenant-a", "fixture"), nil) deny(reader, "GET", "auth/"+readerMount+"/role?list=true", nil) deny(reader, "GET", "auth/"+readerMount+"/role/tenant-a", nil) deny(reader, "POST", "auth/"+readerMount+"/role/tenant-c", map[string]any{"token_policies": []string{"tenant-reader"}}) @@ -111,7 +111,7 @@ func TestAppsBrokerTokens(t *testing.T) { deny(reader, "POST", "auth/token/create", map[string]any{"policies": []string{"tenant-writer"}}) deny(reader, "GET", "sys/policies/acl/tenant-reader", nil) deny(writer, "GET", "auth/"+readerMount+"/role/tenant-a/role-id", nil) - deny(writer, "GET", c.Cfg.KVDataPath("tenant-a", "fixture"), nil) + deny(writer, "GET", c.Cfg.KVDataPath("", "tenant-a", "fixture"), nil) deny(writer, "GET", c.Cfg.DatabaseMount+"/creds/tenant-tenant-a-readonly", nil) // Unknown tenants have no role, so there is nothing to log in as. @@ -153,7 +153,7 @@ func TestListTenants(t *testing.T) { t.Fatalf("empty cluster: %v %v", got, err) } for _, id := range []string{"tenant-b", "tenant-a"} { - if err := c.CreateTenant(id); err != nil { + if err := c.CreateTenant("", id); err != nil { t.Fatal(err) } } @@ -174,7 +174,7 @@ func TestListTenants(t *testing.T) { t.Fatalf("got %+v want %+v", got, want) } var out strings.Builder - PrintTenants(&out, got) + PrintTenants(&out, got, false) if !strings.Contains(out.String(), "tenant-b\t(no writer role") || !strings.HasPrefix(out.String(), "tenant-a\n") { t.Fatalf("output:\n%s", out.String()) } diff --git a/internal/vaultcluster/config.go b/internal/vaultcluster/config.go index 4785a2c..796b072 100644 --- a/internal/vaultcluster/config.go +++ b/internal/vaultcluster/config.go @@ -8,12 +8,16 @@ import ( ) type Config struct { - Addr string - Token string - KVMount string - TenantPrefix string - AuthMount string - DatabaseMount string + Addr string + Token string + KVMount string + TenantPrefix string + AuthMount string + DatabaseMount string + // SharedEnvs scopes every tenant under an env (envs/{env}/customers/{tenant}). Set only on a cluster + // that several Nullstone envs share. Off, the cluster behaves exactly as an unshared one. + SharedEnvs bool + EnvPrefix string AuditPath string AuditDevice string EnableAudit bool @@ -37,6 +41,8 @@ func ConfigFromEnv() Config { TenantPrefix: getenv("TENANT_PREFIX", "customers"), AuthMount: getenv("AUTH_MOUNT", "approle"), DatabaseMount: getenv("DATABASE_MOUNT", "database"), + SharedEnvs: getenv("SHARED_ENVS", "false") == "true", + EnvPrefix: getenv("ENV_PREFIX", "envs"), AuditPath: getenv("AUDIT_LOG_PATH", "/vault/logs/audit.log"), AuditDevice: getenv("AUDIT_DEVICE_NAME", "file"), EnableAudit: getenv("ENABLE_AUDIT", "true") == "true", @@ -76,18 +82,57 @@ func (c Config) AppsPolicy(kind string) string { return "apps-" + kind } -func (c Config) KVDataPath(tenantID, secret string) string { - p := fmt.Sprintf("%s/data/%s/%s", c.KVMount, c.TenantPrefix, tenantID) +// TenantRole is the AppRole role name for a tenant: the tenant ID, or env.tenant on a shared cluster. +// Neither an env name nor a tenant ID may contain ".", so the split is unambiguous. +func (c Config) TenantRole(env, tenantID string) string { + if env == "" { + return tenantID + } + return env + "." + tenantID +} + +// SplitTenantRole is the inverse of TenantRole. env is empty for an unscoped role. +func SplitTenantRole(role string) (env, tenantID string) { + if i := strings.IndexByte(role, '.'); i >= 0 { + return role[:i], role[i+1:] + } + return "", role +} + +// tenantPath is the KV path segment for one tenant under the mount's data/metadata families. +func (c Config) tenantPath(env, tenantID string) string { + if env == "" { + return c.TenantPrefix + "/" + tenantID + } + return c.EnvPrefix + "/" + env + "/" + c.TenantPrefix + "/" + tenantID +} + +func (c Config) KVDataPath(env, tenantID, secret string) string { + p := fmt.Sprintf("%s/data/%s", c.KVMount, c.tenantPath(env, tenantID)) if secret != "" { p += "/" + strings.TrimPrefix(secret, "/") } return p } -func (c Config) KVMetaPath(tenantID, secret string) string { - p := fmt.Sprintf("%s/metadata/%s/%s", c.KVMount, c.TenantPrefix, tenantID) +func (c Config) KVMetaPath(env, tenantID, secret string) string { + p := fmt.Sprintf("%s/metadata/%s", c.KVMount, c.tenantPath(env, tenantID)) if secret != "" { p += "/" + strings.TrimPrefix(secret, "/") } return p } + +// checkEnv enforces the mode: a shared cluster needs an env on every tenant, an unshared one refuses it. +func (c Config) checkEnv(env string) error { + if c.SharedEnvs { + if env == "" { + return fmt.Errorf("this cluster is shared across envs; an env is required") + } + return ValidateEnvName(env) + } + if env != "" { + return fmt.Errorf("this cluster is not shared across envs; env %q is not allowed", env) + } + return nil +} diff --git a/internal/vaultcluster/config_test.go b/internal/vaultcluster/config_test.go new file mode 100644 index 0000000..c579c04 --- /dev/null +++ b/internal/vaultcluster/config_test.go @@ -0,0 +1,65 @@ +package vaultcluster + +import "testing" + +func TestTenantRole(t *testing.T) { + cfg := Config{KVMount: "kv", TenantPrefix: "customers", EnvPrefix: "envs"} + if got := cfg.TenantRole("", "acme"); got != "acme" { + t.Fatalf("unscoped role = %q", got) + } + if got := cfg.TenantRole("pr-12", "acme"); got != "pr-12.acme" { + t.Fatalf("scoped role = %q", got) + } + for _, tt := range []struct{ role, env, id string }{ + {"acme", "", "acme"}, {"pr-12.acme", "pr-12", "acme"}, {"dev.a.b", "dev", "a.b"}, + } { + env, id := SplitTenantRole(tt.role) + if env != tt.env || id != tt.id { + t.Fatalf("SplitTenantRole(%q) = %q, %q", tt.role, env, id) + } + } + if got := cfg.KVDataPath("", "acme", "cfg"); got != "kv/data/customers/acme/cfg" { + t.Fatalf("unscoped data path = %q", got) + } + if got := cfg.KVDataPath("pr-12", "acme", "/cfg"); got != "kv/data/envs/pr-12/customers/acme/cfg" { + t.Fatalf("scoped data path = %q", got) + } + if got := cfg.KVMetaPath("pr-12", "acme", ""); got != "kv/metadata/envs/pr-12/customers/acme" { + t.Fatalf("scoped metadata path = %q", got) + } +} + +func TestCheckEnv(t *testing.T) { + shared, unshared := Config{SharedEnvs: true}, Config{} + if err := shared.checkEnv(""); err == nil { + t.Fatal("shared cluster accepted an empty env") + } + if err := shared.checkEnv("pr-12"); err != nil { + t.Fatal(err) + } + if err := shared.checkEnv("a.b"); err == nil { + t.Fatal("shared cluster accepted an env with a dot") + } + if err := unshared.checkEnv("pr-12"); err == nil { + t.Fatal("unshared cluster accepted an env") + } + if err := unshared.checkEnv(""); err != nil { + t.Fatal(err) + } +} + +func TestValidateEnvName(t *testing.T) { + for _, env := range []string{"dev", "prod", "pr-123", "previews-shared", "sys", "data"} { + if err := ValidateEnvName(env); err != nil { + t.Errorf("%s: %v", env, err) + } + } + for _, env := range []string{"", "a.b", "a/b", "pr-*", "PR", "x", "-x", "x-", "a b"} { + if err := ValidateEnvName(env); err == nil { + t.Errorf("%q should have been rejected", env) + } + } + if err := ValidateTenantID("envs"); err == nil { + t.Error("envs must be a reserved tenant ID") + } +} diff --git a/internal/vaultcluster/credentials_test.go b/internal/vaultcluster/credentials_test.go index 77c2ce1..790b3cd 100644 --- a/internal/vaultcluster/credentials_test.go +++ b/internal/vaultcluster/credentials_test.go @@ -14,10 +14,10 @@ func TestCredentialsMatrix(t *testing.T) { if err := c.Configure(); err != nil { t.Fatal(err) } - if err := c.CreateTenant("tenant-a"); err != nil { + if err := c.CreateTenant("", "tenant-a"); err != nil { t.Fatal(err) } - if err := c.CreateTenant("tenant-b"); err != nil { + if err := c.CreateTenant("", "tenant-b"); err != nil { t.Fatal(err) } @@ -93,7 +93,7 @@ func TestCredentialsAfterIsolation(t *testing.T) { if err := c.Configure(); err != nil { t.Fatal(err) } - if err := c.CreateTenant("tenant-a"); err != nil { + if err := c.CreateTenant("", "tenant-a"); err != nil { t.Fatal(err) } @@ -116,7 +116,7 @@ func TestCredentialsAfterIsolation(t *testing.T) { if err := prov.enableCredentialsIfNeeded(store); err != nil { t.Fatal(err) } - if err := prov.CreateTenant("tenant-a"); err != nil { + if err := prov.CreateTenant("", "tenant-a"); err != nil { t.Fatal(err) } if r, _ := prov.Do("POST", c.Cfg.DatabaseMount+"/roles/tenant-tenant-a-readwrite", map[string]any{ diff --git a/internal/vaultcluster/envs.go b/internal/vaultcluster/envs.go new file mode 100644 index 0000000..b646986 --- /dev/null +++ b/internal/vaultcluster/envs.go @@ -0,0 +1,62 @@ +package vaultcluster + +import ( + "fmt" + "log" + "sort" +) + +// ListEnvs returns every env that has at least one tenant role on a shared cluster. +func (c *Client) ListEnvs() ([]string, error) { + if !c.Cfg.SharedEnvs { + return nil, fmt.Errorf("this cluster is not shared across envs") + } + tenants, err := c.ListTenants() + if err != nil { + return nil, err + } + seen := map[string]struct{}{} + for _, t := range tenants { + if t.Env != "" { + seen[t.Env] = struct{}{} + } + } + out := make([]string, 0, len(seen)) + for env := range seen { + out = append(out, env) + } + sort.Strings(out) + return out, nil +} + +// DestroyEnv offboards every tenant of one env. With purge it also destroys the env's KV subtree, which +// needs a token that can read and delete under kv/metadata/envs/ (provisioning cannot). +func (c *Client) DestroyEnv(env string, purge bool) error { + if !c.Cfg.SharedEnvs { + return fmt.Errorf("this cluster is not shared across envs") + } + if err := ValidateEnvName(env); err != nil { + return err + } + tenants, err := c.ListTenants() + if err != nil { + return err + } + n := 0 + for _, t := range tenants { + if t.Env != env { + continue + } + if err := c.OffboardTenant(env, t.ID, false); err != nil { + return err + } + n++ + } + if purge { + if err := c.purgeKVPrefix(fmt.Sprintf("%s/metadata/%s/%s", c.Cfg.KVMount, c.Cfg.EnvPrefix, env)); err != nil { + return err + } + } + log.Printf("env %s destroyed (%d tenants)", env, n) + return nil +} diff --git a/internal/vaultcluster/isolation_test.go b/internal/vaultcluster/isolation_test.go index 608516d..1032859 100644 --- a/internal/vaultcluster/isolation_test.go +++ b/internal/vaultcluster/isolation_test.go @@ -17,10 +17,10 @@ func TestIsolationMatrix(t *testing.T) { if err := c.Configure(); err != nil { t.Fatal(err) } - if err := c.CreateTenant("tenant-a"); err != nil { + if err := c.CreateTenant("", "tenant-a"); err != nil { t.Fatal(err) } - if err := c.CreateTenant("tenant-b"); err != nil { + if err := c.CreateTenant("", "tenant-b"); err != nil { t.Fatal(err) } @@ -59,8 +59,8 @@ func TestIsolationMatrix(t *testing.T) { t.Fatalf("write %s: %v", path, err) } } - mustWrite(tokAW, c.Cfg.KVDataPath("tenant-a", "fixture"), "secret-a") - mustWrite(tokBW, c.Cfg.KVDataPath("tenant-b", "fixture"), "secret-b") + mustWrite(tokAW, c.Cfg.KVDataPath("", "tenant-a", "fixture"), "secret-a") + mustWrite(tokBW, c.Cfg.KVDataPath("", "tenant-b", "fixture"), "secret-b") assertStatus := func(token, method, path string, body any, want int) { t.Helper() @@ -78,10 +78,10 @@ func TestIsolationMatrix(t *testing.T) { assertStatus(token, method, path, body, 403) } - aData := c.Cfg.KVDataPath("tenant-a", "fixture") - bData := c.Cfg.KVDataPath("tenant-b", "fixture") - aMeta := c.Cfg.KVMetaPath("tenant-a", "fixture") - bMeta := c.Cfg.KVMetaPath("tenant-b", "fixture") + aData := c.Cfg.KVDataPath("", "tenant-a", "fixture") + bData := c.Cfg.KVDataPath("", "tenant-b", "fixture") + aMeta := c.Cfg.KVMetaPath("", "tenant-a", "fixture") + bMeta := c.Cfg.KVMetaPath("", "tenant-b", "fixture") crossWrite := map[string]any{"data": map[string]any{"value": "FAKE-cross-tenant-write"}} assertStatus(tokA, "GET", aData, nil, 200) @@ -94,7 +94,7 @@ func TestIsolationMatrix(t *testing.T) { deny(tokBW, "GET", aData, nil) deny(tokAW, "POST", bData, crossWrite) deny(tokBW, "POST", aData, crossWrite) - deny(tokAW, "POST", c.Cfg.KVDataPath("tenant-b", "newly-planted-secret"), map[string]any{ + deny(tokAW, "POST", c.Cfg.KVDataPath("", "tenant-b", "newly-planted-secret"), map[string]any{ "data": map[string]any{"value": "FAKE-planted"}, }) deny(tokAW, "DELETE", bData, nil) @@ -104,7 +104,7 @@ func TestIsolationMatrix(t *testing.T) { deny(tokB, "GET", aMeta, nil) deny(tokA, "GET", c.Cfg.KVMount+"/metadata/"+c.Cfg.TenantPrefix+"?list=true", nil) deny(tokB, "GET", c.Cfg.KVMount+"/metadata/"+c.Cfg.TenantPrefix+"?list=true", nil) - deny(tokA, "GET", c.Cfg.KVMetaPath("tenant-b", "")+"?list=true", nil) + deny(tokA, "GET", c.Cfg.KVMetaPath("", "tenant-b", "")+"?list=true", nil) deny(tokA, "GET", c.Cfg.KVMount+"/data/"+c.Cfg.TenantPrefix, nil) deny(tokA, "GET", c.Cfg.KVMount+"/data", nil) @@ -116,8 +116,8 @@ func TestIsolationMatrix(t *testing.T) { deny(tokAW, "POST", c.Cfg.KVMount+"/data/platform/root-credentials", map[string]any{ "data": map[string]any{"value": "FAKE-planted-sibling"}, }) - deny(tokA, "GET", c.Cfg.KVDataPath("tenant-a-extended", "secret"), nil) - deny(tokA, "GET", c.Cfg.KVDataPath("tenant-ax", "secret"), nil) + deny(tokA, "GET", c.Cfg.KVDataPath("", "tenant-a-extended", "secret"), nil) + deny(tokA, "GET", c.Cfg.KVDataPath("", "tenant-ax", "secret"), nil) deny(tokA, "GET", c.Cfg.KVMount+"/data/"+c.Cfg.TenantPrefix+"/*", nil) deny(tokA, "GET", "sys/mounts", nil) @@ -183,7 +183,7 @@ func TestIsolationMatrix(t *testing.T) { } } - if err := c.OffboardTenant("tenant-a", false); err != nil { + if err := c.OffboardTenant("", "tenant-a", false); err != nil { t.Fatal(err) } if _, err := c.LoginAppRole(c.Cfg.TenantMount("reader"), "tenant-a"); err == nil { diff --git a/internal/vaultcluster/lint.go b/internal/vaultcluster/lint.go index 8d95bab..132af83 100644 --- a/internal/vaultcluster/lint.go +++ b/internal/vaultcluster/lint.go @@ -109,6 +109,17 @@ func LintPolicy(policyName, src string, cfg Config) []Finding { } } + // Shared cluster: a grant under /// must name one env and one tenant. + if envPrefix := cfg.EnvPrefix; envPrefix != "" && p.caps != "deny" && strings.HasPrefix(p.path, cfg.KVMount+"/") { + segs := strings.Split(p.path, "/") + if len(segs) >= 4 && segs[2] == envPrefix { + scoped := len(segs) >= 6 && isLiteralSegment(segs[3]) && segs[4] == cfg.TenantPrefix && isLiteralSegment(segs[5]) + if !scoped { + add(fmt.Sprintf("path %q grants [%s] across envs or tenants", p.path, p.caps)) + } + } + } + switch p.path { case cfg.KVMount + "/*", cfg.KVMount + "/data/*", cfg.KVMount + "/metadata/*": if p.caps != "deny" { @@ -138,3 +149,8 @@ func LintOrError(policyName, src string, cfg Config) error { } return fmt.Errorf("%s", b.String()) } + +// isLiteralSegment is one env or tenant segment: non-empty and not a glob. ACL templates count as literal. +func isLiteralSegment(s string) bool { + return s != "" && s != "*" && s != "+" +} diff --git a/internal/vaultcluster/lint_test.go b/internal/vaultcluster/lint_test.go index 46ba750..ffc8f0c 100644 --- a/internal/vaultcluster/lint_test.go +++ b/internal/vaultcluster/lint_test.go @@ -9,7 +9,7 @@ import ( ) func TestLintFixtures(t *testing.T) { - cfg := Config{KVMount: "kv", TenantPrefix: "customers", DatabaseMount: "database", AuthMount: "approle"} + cfg := Config{KVMount: "kv", TenantPrefix: "customers", EnvPrefix: "envs", DatabaseMount: "database", AuthMount: "approle"} root := filepath.Join("testdata", "lint") entries, err := os.ReadDir(root) if err != nil { @@ -36,7 +36,7 @@ func TestLintFixtures(t *testing.T) { } func TestRenderAndLintPlatformPolicies(t *testing.T) { - cfg := Config{KVMount: "kv", TenantPrefix: "customers", DatabaseMount: "database", AuthMount: "approle"} + cfg := Config{KVMount: "kv", TenantPrefix: "customers", EnvPrefix: "envs", DatabaseMount: "database", AuthMount: "approle"} acc := TenantAccessors{Reader: "auth_approle_1a2b3c4d", Writer: "auth_approle_5e6f7a8b"} for _, name := range []string{"provisioning", "operator", "apps-auth", "apps-reader", "apps-writer"} { hcl, err := RenderPolicy(name, cfg, acc) diff --git a/internal/vaultcluster/policies/templates/apps-auth.hcl.tpl b/internal/vaultcluster/policies/templates/apps-auth.hcl.tpl index 499bde2..4eeac4a 100644 --- a/internal/vaultcluster/policies/templates/apps-auth.hcl.tpl +++ b/internal/vaultcluster/policies/templates/apps-auth.hcl.tpl @@ -42,3 +42,45 @@ path "{{.KVMount}}/*" { path "{{.DatabaseMount}}/creds/*" { capabilities = ["deny"] } +{{- if .Shared}} + +# Shared cluster: the function writes one entity per app role (metadata env) and its alias, so the broker +# policies can scope the app to its env. policies and disabled are refused. +path "identity/entity" { + capabilities = ["create", "update"] + denied_parameters = { "policies" = [], "disabled" = [] } +} + +path "identity/entity/name/*" { + capabilities = ["read", "update", "delete"] + denied_parameters = { "policies" = [], "disabled" = [] } +} + +path "identity/entity-alias" { + capabilities = ["create", "update"] +} + +path "identity/entity-alias/id/*" { + capabilities = ["read", "delete"] +} + +path "identity/lookup/entity" { + capabilities = ["update"] +} + +path "identity/entity/merge" { + capabilities = ["deny"] +} + +path "identity/group*" { + capabilities = ["deny"] +} + +path "identity/oidc/*" { + capabilities = ["deny"] +} + +path "identity/mfa/*" { + capabilities = ["deny"] +} +{{- end}} diff --git a/internal/vaultcluster/policies/templates/apps-reader.hcl.tpl b/internal/vaultcluster/policies/templates/apps-reader.hcl.tpl index 0937968..13c0ee7 100644 --- a/internal/vaultcluster/policies/templates/apps-reader.hcl.tpl +++ b/internal/vaultcluster/policies/templates/apps-reader.hcl.tpl @@ -1,5 +1,13 @@ # App broker, read level. Lets an app log in as any onboarded tenant's reader: read the role ID and mint a # one-time secret ID on {{.ReaderMount}}. The token that login yields sees one tenant; this token sees none. +{{if .Shared -}} +# Shared cluster: only roles of the app's own env (entity metadata). No parameter may be sent, so the role, +# its role-id, and its policies cannot be changed; an empty-body secret-id mint still works. +path "auth/{{.ReaderMount}}/role/{{.Env}}.*" { + capabilities = ["read", "update"] + denied_parameters = { "*" = [] } +} +{{- else -}} path "auth/{{.ReaderMount}}/role/+/role-id" { capabilities = ["read"] } @@ -7,6 +15,7 @@ path "auth/{{.ReaderMount}}/role/+/role-id" { path "auth/{{.ReaderMount}}/role/+/secret-id" { capabilities = ["update"] } +{{- end}} path "auth/token/lookup-self" { capabilities = ["read"] diff --git a/internal/vaultcluster/policies/templates/apps-writer.hcl.tpl b/internal/vaultcluster/policies/templates/apps-writer.hcl.tpl index 8622d23..0af7425 100644 --- a/internal/vaultcluster/policies/templates/apps-writer.hcl.tpl +++ b/internal/vaultcluster/policies/templates/apps-writer.hcl.tpl @@ -1,5 +1,13 @@ # App broker, write level. Lets an app log in as any onboarded tenant's writer: read the role ID and mint a # one-time secret ID on {{.WriterMount}}. The token that login yields sees one tenant; this token sees none. +{{if .Shared -}} +# Shared cluster: only roles of the app's own env (entity metadata). No parameter may be sent, so the role, +# its role-id, and its policies cannot be changed; an empty-body secret-id mint still works. +path "auth/{{.WriterMount}}/role/{{.Env}}.*" { + capabilities = ["read", "update"] + denied_parameters = { "*" = [] } +} +{{- else -}} path "auth/{{.WriterMount}}/role/+/role-id" { capabilities = ["read"] } @@ -7,6 +15,7 @@ path "auth/{{.WriterMount}}/role/+/role-id" { path "auth/{{.WriterMount}}/role/+/secret-id" { capabilities = ["update"] } +{{- end}} path "auth/token/lookup-self" { capabilities = ["read"] diff --git a/internal/vaultcluster/policies/templates/provisioning.hcl.tpl b/internal/vaultcluster/policies/templates/provisioning.hcl.tpl index ab70a5f..4e99d2f 100644 --- a/internal/vaultcluster/policies/templates/provisioning.hcl.tpl +++ b/internal/vaultcluster/policies/templates/provisioning.hcl.tpl @@ -116,7 +116,48 @@ path "sys/auth/*" { path "auth/token/create*" { capabilities = ["deny"] } +{{if .Shared}} +# Shared cluster: tenants create writes one entity per tenant role (metadata env and tenant) and its alias. +# policies and disabled are refused, so an entity can never carry more than its roles grant. +path "identity/entity" { + capabilities = ["create", "update"] + denied_parameters = { "policies" = [], "disabled" = [] } +} + +path "identity/entity/name/*" { + capabilities = ["read", "update", "delete"] + denied_parameters = { "policies" = [], "disabled" = [] } +} +path "identity/entity-alias" { + capabilities = ["create", "update"] +} + +path "identity/entity-alias/id/*" { + capabilities = ["read", "delete"] +} + +path "identity/lookup/entity" { + capabilities = ["update"] +} + +path "identity/entity/merge" { + capabilities = ["deny"] +} + +path "identity/group*" { + capabilities = ["deny"] +} + +path "identity/oidc/*" { + capabilities = ["deny"] +} + +path "identity/mfa/*" { + capabilities = ["deny"] +} +{{- else}} path "identity/*" { capabilities = ["deny"] } +{{- end}} diff --git a/internal/vaultcluster/policies/templates/tenant-database.hcl.tpl b/internal/vaultcluster/policies/templates/tenant-database.hcl.tpl index 6a28064..e92dc98 100644 --- a/internal/vaultcluster/policies/templates/tenant-database.hcl.tpl +++ b/internal/vaultcluster/policies/templates/tenant-database.hcl.tpl @@ -1,9 +1,12 @@ +{{- /* Unshared: tenant-. Shared: tenant-.. */ -}} +{{- $role := printf "tenant-%s" .WriterTenant -}} +{{- if .Shared}}{{$role = printf "tenant-%s.%s" .Env .WriterTenant}}{{end -}} # Dynamic DB creds for the tenant named by the caller's {{.WriterMount}} role only. Additive; does not widen KV access. -path "{{.DatabaseMount}}/creds/tenant-{{.WriterTenant}}-*" { +path "{{.DatabaseMount}}/creds/{{$role}}-*" { capabilities = ["read"] } -path "{{.DatabaseMount}}/roles/tenant-{{.WriterTenant}}-*" { +path "{{.DatabaseMount}}/roles/{{$role}}-*" { capabilities = ["read"] } diff --git a/internal/vaultcluster/policies/templates/tenant-reader.hcl.tpl b/internal/vaultcluster/policies/templates/tenant-reader.hcl.tpl index cd23695..d5947ab 100644 --- a/internal/vaultcluster/policies/templates/tenant-reader.hcl.tpl +++ b/internal/vaultcluster/policies/templates/tenant-reader.hcl.tpl @@ -1,30 +1,34 @@ +{{- /* Unshared: customers/. Shared: envs//customers/. */ -}} +{{- $tenant := printf "%s/%s" .TenantPrefix .ReaderTenant -}} +{{- $deny := .TenantPrefix -}} +{{- if .Shared}}{{$tenant = printf "%s/%s/%s" .EnvPrefix .Env $tenant}}{{$deny = .EnvPrefix}}{{end -}} # Read-only on the tenant named by the caller's AppRole role ({{.ReaderMount}} mount; role name = tenant ID). Static: written once at bootstrap. # KV v2 needs data/ and metadata/; a token without a {{.ReaderMount}} alias renders no tenant path. -path "{{.KVMount}}/data/{{.TenantPrefix}}/{{.ReaderTenant}}/*" { +path "{{.KVMount}}/data/{{$tenant}}/*" { capabilities = ["read"] } -path "{{.KVMount}}/metadata/{{.TenantPrefix}}/{{.ReaderTenant}}/*" { +path "{{.KVMount}}/metadata/{{$tenant}}/*" { capabilities = ["read", "list"] } -path "{{.KVMount}}/data/{{.TenantPrefix}}/*" { +path "{{.KVMount}}/data/{{$deny}}/*" { capabilities = ["deny"] } -path "{{.KVMount}}/metadata/{{.TenantPrefix}}/*" { +path "{{.KVMount}}/metadata/{{$deny}}/*" { capabilities = ["deny"] } -path "{{.KVMount}}/delete/{{.TenantPrefix}}/*" { +path "{{.KVMount}}/delete/{{$deny}}/*" { capabilities = ["deny"] } -path "{{.KVMount}}/undelete/{{.TenantPrefix}}/*" { +path "{{.KVMount}}/undelete/{{$deny}}/*" { capabilities = ["deny"] } -path "{{.KVMount}}/destroy/{{.TenantPrefix}}/*" { +path "{{.KVMount}}/destroy/{{$deny}}/*" { capabilities = ["deny"] } diff --git a/internal/vaultcluster/policies/templates/tenant-writer.hcl.tpl b/internal/vaultcluster/policies/templates/tenant-writer.hcl.tpl index 2e84c7c..bd0e7ab 100644 --- a/internal/vaultcluster/policies/templates/tenant-writer.hcl.tpl +++ b/internal/vaultcluster/policies/templates/tenant-writer.hcl.tpl @@ -1,42 +1,46 @@ +{{- /* Unshared: customers/. Shared: envs//customers/. */ -}} +{{- $tenant := printf "%s/%s" .TenantPrefix .WriterTenant -}} +{{- $deny := .TenantPrefix -}} +{{- if .Shared}}{{$tenant = printf "%s/%s/%s" .EnvPrefix .Env $tenant}}{{$deny = .EnvPrefix}}{{end -}} # Full KV lifecycle on the tenant named by the caller's AppRole role ({{.WriterMount}} mount; role name = tenant ID). Static: written once at bootstrap. # Name all five KV v2 path families. -path "{{.KVMount}}/data/{{.TenantPrefix}}/{{.WriterTenant}}/*" { +path "{{.KVMount}}/data/{{$tenant}}/*" { capabilities = ["create", "read", "update", "patch", "delete", "list"] } -path "{{.KVMount}}/metadata/{{.TenantPrefix}}/{{.WriterTenant}}/*" { +path "{{.KVMount}}/metadata/{{$tenant}}/*" { capabilities = ["create", "read", "update", "delete", "list"] } -path "{{.KVMount}}/delete/{{.TenantPrefix}}/{{.WriterTenant}}/*" { +path "{{.KVMount}}/delete/{{$tenant}}/*" { capabilities = ["update"] } -path "{{.KVMount}}/undelete/{{.TenantPrefix}}/{{.WriterTenant}}/*" { +path "{{.KVMount}}/undelete/{{$tenant}}/*" { capabilities = ["update"] } -path "{{.KVMount}}/destroy/{{.TenantPrefix}}/{{.WriterTenant}}/*" { +path "{{.KVMount}}/destroy/{{$tenant}}/*" { capabilities = ["update"] } -path "{{.KVMount}}/data/{{.TenantPrefix}}/*" { +path "{{.KVMount}}/data/{{$deny}}/*" { capabilities = ["deny"] } -path "{{.KVMount}}/metadata/{{.TenantPrefix}}/*" { +path "{{.KVMount}}/metadata/{{$deny}}/*" { capabilities = ["deny"] } -path "{{.KVMount}}/delete/{{.TenantPrefix}}/*" { +path "{{.KVMount}}/delete/{{$deny}}/*" { capabilities = ["deny"] } -path "{{.KVMount}}/undelete/{{.TenantPrefix}}/*" { +path "{{.KVMount}}/undelete/{{$deny}}/*" { capabilities = ["deny"] } -path "{{.KVMount}}/destroy/{{.TenantPrefix}}/*" { +path "{{.KVMount}}/destroy/{{$deny}}/*" { capabilities = ["deny"] } diff --git a/internal/vaultcluster/policy.go b/internal/vaultcluster/policy.go index 42b7f20..0a9d022 100644 --- a/internal/vaultcluster/policy.go +++ b/internal/vaultcluster/policy.go @@ -9,13 +9,20 @@ import ( "github.com/nullstone-modules/vault-cluster/internal/vaultcluster/policies" ) -// TenantAccessors are the mount accessors of the tenant AppRole mounts. Static tenant policies -// template the tenant ID from the login's role_name alias metadata on these mounts. +// TenantAccessors are the mount accessors of the tenant AppRole mounts. On an unshared cluster the static +// tenant policies template the tenant ID from the login's role_name alias metadata on these mounts. On a +// shared cluster they are used to attach the per-tenant entity alias instead. type TenantAccessors struct { Reader string Writer string } +// Entity metadata keys written on a shared cluster. Tenant policies template both; broker policies template env. +const ( + entityMetaEnv = "env" + entityMetaTenant = "tenant" +) + func RenderPolicy(templateName string, cfg Config, acc TenantAccessors) (string, error) { name := templateName + ".hcl.tpl" raw, err := policies.Templates.ReadFile("templates/" + name) @@ -34,24 +41,37 @@ func RenderPolicy(templateName string, cfg Config, acc TenantAccessors) (string, stmts = append(stmts, strconv.Quote(s)) } data := struct { - KVMount string - TenantPrefix string - DatabaseMount string - ReaderMount string - WriterMount string - ReaderTenant string - WriterTenant string + Shared bool + KVMount string + TenantPrefix string + EnvPrefix string + DatabaseMount string + ReaderMount string + WriterMount string + // ReaderTenant / WriterTenant name the caller's tenant. Unshared: the role_name alias on that mount. + // Shared: env and tenant from entity metadata, as one path segment pair. + ReaderTenant string + WriterTenant string + // Env is the caller's env from entity metadata (shared only). + Env string DBCreationStatements string }{ + Shared: cfg.SharedEnvs, KVMount: cfg.KVMount, TenantPrefix: cfg.TenantPrefix, + EnvPrefix: cfg.EnvPrefix, DatabaseMount: cfg.DatabaseMount, ReaderMount: cfg.TenantMount("reader"), WriterMount: cfg.TenantMount("writer"), ReaderTenant: roleNameTemplate(acc.Reader), WriterTenant: roleNameTemplate(acc.Writer), + Env: entityMetaTemplate(entityMetaEnv), DBCreationStatements: strings.Join(stmts, ", "), } + if cfg.SharedEnvs { + data.ReaderTenant = entityMetaTemplate(entityMetaTenant) + data.WriterTenant = entityMetaTemplate(entityMetaTenant) + } var b strings.Builder if err := tpl.Execute(&b, data); err != nil { return "", fmt.Errorf("render template %q: %w", templateName, err) @@ -63,3 +83,8 @@ func RenderPolicy(templateName string, cfg Config, acc TenantAccessors) (string, func roleNameTemplate(accessor string) string { return "{{identity.entity.aliases." + accessor + ".metadata.role_name}}" } + +// entityMetaTemplate is the Vault ACL template for one entity metadata key. +func entityMetaTemplate(key string) string { + return "{{identity.entity.metadata." + key + "}}" +} diff --git a/internal/vaultcluster/policy_test.go b/internal/vaultcluster/policy_test.go new file mode 100644 index 0000000..b495e62 --- /dev/null +++ b/internal/vaultcluster/policy_test.go @@ -0,0 +1,115 @@ +package vaultcluster + +import ( + "flag" + "os" + "path/filepath" + "strings" + "testing" +) + +var updateGolden = flag.Bool("update", false, "rewrite testdata/policies goldens") + +var allPolicies = []string{"provisioning", "operator", "apps-auth", "apps-reader", "apps-writer", "tenant-reader", "tenant-writer", "tenant-database"} + +var goldenAccessors = TenantAccessors{Reader: "auth_approle_1a2b3c4d", Writer: "auth_approle_5e6f7a8b"} + +func goldenConfig(shared bool) Config { + return Config{KVMount: "kv", TenantPrefix: "customers", EnvPrefix: "envs", DatabaseMount: "database", AuthMount: "approle", SharedEnvs: shared} +} + +// The unshared goldens are the 0.1.x policies. They must never change: an unshared cluster is unaffected +// by shared-env support. The shared goldens document what a shared cluster applies. +func TestRenderedPoliciesMatchGoldens(t *testing.T) { + for _, mode := range []struct { + dir string + shared bool + }{{"unshared", false}, {"shared", true}} { + cfg := goldenConfig(mode.shared) + for _, name := range allPolicies { + hcl, err := RenderPolicy(name, cfg, goldenAccessors) + if err != nil { + t.Fatal(err) + } + if err := LintOrError(name, hcl, cfg); err != nil { + t.Fatalf("%s/%s: %v", mode.dir, name, err) + } + path := filepath.Join("testdata", "policies", mode.dir, name+".hcl") + if *updateGolden { + if err := os.WriteFile(path, []byte(hcl), 0o644); err != nil { + t.Fatal(err) + } + continue + } + want, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if string(want) != hcl { + t.Errorf("%s/%s differs from golden (run with -update to accept):\n%s", mode.dir, name, diffLines(string(want), hcl)) + } + } + } +} + +func TestSharedPoliciesScopeByEntityMetadata(t *testing.T) { + cfg := goldenConfig(true) + env, tenant := entityMetaTemplate("env"), entityMetaTemplate("tenant") + for _, name := range []string{"tenant-reader", "tenant-writer"} { + hcl, _ := RenderPolicy(name, cfg, goldenAccessors) + if !strings.Contains(hcl, `"kv/data/envs/`+env+`/customers/`+tenant+`/*"`) { + t.Fatalf("%s must scope data to envs//customers/:\n%s", name, hcl) + } + if strings.Contains(hcl, "role_name") || !strings.Contains(hcl, `"kv/data/envs/*"`) { + t.Fatalf("%s must not use the role name and must deny the env prefix:\n%s", name, hcl) + } + } + db, _ := RenderPolicy("tenant-database", cfg, goldenAccessors) + if !strings.Contains(db, `"database/creds/tenant-`+env+`.`+tenant+`-*"`) { + t.Fatalf("tenant-database must scope creds by env and tenant:\n%s", db) + } + for _, name := range []string{"apps-reader", "apps-writer"} { + hcl, _ := RenderPolicy(name, cfg, goldenAccessors) + mount := "approle-reader" + if name == "apps-writer" { + mount = "approle-writer" + } + if !strings.Contains(hcl, `"auth/`+mount+`/role/`+env+`.*"`) || !strings.Contains(hcl, `denied_parameters = { "*" = [] }`) { + t.Fatalf("%s must mint only its env's roles with no parameters:\n%s", name, hcl) + } + if strings.Contains(hcl, "role/+/") { + t.Fatalf("%s must not keep the any-tenant rules:\n%s", name, hcl) + } + } + for _, name := range []string{"provisioning", "apps-auth"} { + hcl, _ := RenderPolicy(name, cfg, goldenAccessors) + if !strings.Contains(hcl, `path "identity/entity"`) || !strings.Contains(hcl, `denied_parameters = { "policies" = [], "disabled" = [] }`) { + t.Fatalf("%s must write entities without policies:\n%s", name, hcl) + } + if strings.Contains(hcl, `path "identity/*"`) { + t.Fatalf("%s must narrow the identity deny:\n%s", name, hcl) + } + } + unshared, _ := RenderPolicy("provisioning", goldenConfig(false), goldenAccessors) + if !strings.Contains(unshared, `path "identity/*"`) || strings.Contains(unshared, `path "identity/entity"`) { + t.Fatalf("unshared provisioning must keep the identity deny:\n%s", unshared) + } +} + +func diffLines(want, got string) string { + w, g := strings.Split(want, "\n"), strings.Split(got, "\n") + var b strings.Builder + for i := 0; i < len(w) || i < len(g); i++ { + var wl, gl string + if i < len(w) { + wl = w[i] + } + if i < len(g) { + gl = g[i] + } + if wl != gl { + b.WriteString("- " + wl + "\n+ " + gl + "\n") + } + } + return b.String() +} diff --git a/internal/vaultcluster/shared_test.go b/internal/vaultcluster/shared_test.go new file mode 100644 index 0000000..627b0b5 --- /dev/null +++ b/internal/vaultcluster/shared_test.go @@ -0,0 +1,364 @@ +package vaultcluster + +import ( + "strings" + "testing" + + "github.com/hashicorp/vault/api" +) + +// A shared cluster. Everything an unshared cluster promises still holds; on top, two envs never see each +// other's secrets even for the same tenant ID, and an app's broker mints only its own env's logins. + +const appsTestMount = "apps-test" + +func startSharedVault(t *testing.T) *Client { + t.Helper() + c := startVaultInmem(t) + c.Cfg.KVMount = "kv" + c.Cfg.TenantPrefix = "customers" + c.Cfg.EnvPrefix = "envs" + c.Cfg.AuthMount = "approle" + c.Cfg.DatabaseMount = "database" + c.Cfg.EnableAudit = false + c.Cfg.SharedEnvs = true + c.Cfg.TokenTTL = "15m" + c.Cfg.TokenMaxTTL = "15m" + if err := c.Configure(); err != nil { + t.Fatal(err) + } + return c +} + +// appBroker logs in as an app would through the cluster function: one auth role, one entity with env, one +// broker policy. Stand-in mount is AppRole; the aws and gcp mounts alias by role_id the same way. +func appBroker(t *testing.T, c *Client, env, kind string) *Client { + t.Helper() + auths, err := c.API.Sys().ListAuth() + if err != nil { + t.Fatal(err) + } + if _, ok := auths[appsTestMount+"/"]; !ok { + if err := c.API.Sys().EnableAuthWithOptions(appsTestMount, &api.EnableAuthOptions{Type: "approle"}); err != nil { + t.Fatal(err) + } + } + role := "api-" + kind + "-" + env + if env == "" { + role = "api-" + kind + "-noenv" + } + if _, err := c.API.Logical().Write("auth/"+appsTestMount+"/role/"+role, map[string]any{ + "token_policies": []string{c.Cfg.AppsPolicy(kind)}, + }); err != nil { + t.Fatal(err) + } + if env != "" { + rid, err := c.API.Logical().Read("auth/" + appsTestMount + "/role/" + role + "/role-id") + if err != nil { + t.Fatal(err) + } + acc, err := c.authAccessor(appsTestMount) + if err != nil { + t.Fatal(err) + } + if err := c.ensureEntityAlias(appsTestMount+"/"+role, map[string]any{entityMetaEnv: env}, rid.Data["role_id"].(string), acc); err != nil { + t.Fatal(err) + } + } + tok, err := c.LoginAppRole(appsTestMount, role) + if err != nil { + t.Fatal(err) + } + return c.WithToken(tok) +} + +func httpStatus(t *testing.T, cl *Client, method, path string, body any) int { + t.Helper() + r, err := cl.Do(method, path, body) + if err != nil && r.Status == 0 { + t.Fatalf("%s %s: %v", method, path, err) + } + return r.Status +} + +func mustDeny(t *testing.T, cl *Client, method, path string, body any) { + t.Helper() + if got := httpStatus(t, cl, method, path, body); got != 403 { + t.Fatalf("%s %s: got HTTP %d want 403", method, path, got) + } +} + +func TestIsolationMatrixShared(t *testing.T) { + c := startSharedVault(t) + if err := c.CreateTenant("", "acme"); err == nil { + t.Fatal("shared cluster created a tenant without an env") + } + for _, env := range []string{"env-a", "env-b"} { + for _, id := range []string{"acme", "beta"} { + if err := c.CreateTenant(env, id); err != nil { + t.Fatal(err) + } + } + } + // Re-running create is a no-op, not a failure. + if err := c.CreateTenant("env-a", "acme"); err != nil { + t.Fatalf("second create: %v", err) + } + + login := func(kind, env, id string) *Client { + t.Helper() + tok, err := c.LoginAppRole(c.Cfg.TenantMount(kind), c.Cfg.TenantRole(env, id)) + if err != nil { + t.Fatal(err) + } + return c.WithToken(tok) + } + aW, bW := login("writer", "env-a", "acme"), login("writer", "env-b", "acme") + aR, bR := login("reader", "env-a", "acme"), login("reader", "env-b", "acme") + betaR := login("reader", "env-a", "beta") + + write := map[string]any{"data": map[string]any{"value": "FAKE"}} + aData, bData := c.Cfg.KVDataPath("env-a", "acme", "fixture"), c.Cfg.KVDataPath("env-b", "acme", "fixture") + if got := httpStatus(t, aW, "POST", aData, write); got != 200 { + t.Fatalf("env-a writer own secret: HTTP %d", got) + } + if got := httpStatus(t, bW, "POST", bData, write); got != 200 { + t.Fatalf("env-b writer own secret: HTTP %d", got) + } + if got := httpStatus(t, aR, "GET", aData, nil); got != 200 { + t.Fatalf("env-a reader own secret: HTTP %d", got) + } + if got := httpStatus(t, bR, "GET", bData, nil); got != 200 { + t.Fatalf("env-b reader own secret: HTTP %d", got) + } + + // Same tenant ID, other env. + mustDeny(t, aR, "GET", bData, nil) + mustDeny(t, bR, "GET", aData, nil) + mustDeny(t, aW, "POST", bData, write) + mustDeny(t, aW, "DELETE", bData, nil) + mustDeny(t, aW, "DELETE", c.Cfg.KVMetaPath("env-b", "acme", "fixture"), nil) + // Other tenant, same env. + mustDeny(t, betaR, "GET", aData, nil) + mustDeny(t, aR, "GET", c.Cfg.KVDataPath("env-a", "beta", "fixture"), nil) + // Parents, wildcards, the env prefix, and the unscoped layout. + for _, p := range []string{ + "kv/data/envs/env-a/customers/acme-extended/secret", + "kv/data/envs/env-a/customers/shared-secret", + "kv/data/envs/env-a/customers/*", + "kv/data/envs/env-a/platform/x", + "kv/data/envs/*", + "kv/data/envs/env-a", + "kv/data/customers/acme/fixture", + "kv/data/customers/env-a/acme/fixture", + "kv/data/platform/root-credentials", + } { + mustDeny(t, aR, "GET", p, nil) + mustDeny(t, aW, "POST", p, write) + } + mustDeny(t, aR, "GET", "kv/metadata/envs?list=true", nil) + mustDeny(t, aR, "GET", "kv/metadata/envs/env-a?list=true", nil) + mustDeny(t, aR, "GET", "kv/metadata/envs/env-a/customers?list=true", nil) + mustDeny(t, aR, "GET", c.Cfg.KVMetaPath("env-b", "acme", "")+"?list=true", nil) + for _, traversal := range []string{ + "kv/data/envs/env-a/customers/acme/../../../env-b/customers/acme/fixture", + "kv/data/envs/env-a/..%2fenv-b/customers/acme/fixture", + } { + if got := httpStatus(t, aR, "GET", traversal, nil); got >= 200 && got < 300 { + t.Fatalf("traversal granted HTTP %d: %s", got, traversal) + } + } + // Vault lets a token read its own entity. It must not read another's or change its own metadata. + mustDeny(t, aR, "GET", "identity/entity/name/approle-reader/env-b.acme", nil) + mustDeny(t, aR, "GET", "identity/entity/name/approle-writer/env-a.acme", nil) + mustDeny(t, aW, "POST", "identity/entity/name/approle-writer/env-a.acme", map[string]any{"metadata": map[string]any{"env": "env-b"}}) + mustDeny(t, aW, "POST", "identity/entity-alias", map[string]any{"name": "x", "canonical_id": "y", "mount_accessor": "z"}) + + // Provisioning: creates and offboards, but cannot read tenant data, write policies, grant an entity + // policies, or touch groups. + provSec, err := c.API.Auth().Token().Create(&api.TokenCreateRequest{Policies: []string{"provisioning"}, NoParent: true}) + if err != nil { + t.Fatal(err) + } + prov := c.WithToken(provSec.Auth.ClientToken) + if err := prov.CreateTenant("env-c", "gamma"); err != nil { + t.Fatalf("provisioning create: %v", err) + } + if _, err := c.LoginAppRole(c.Cfg.TenantMount("reader"), "env-c.gamma"); err != nil { + t.Fatal(err) + } + mustDeny(t, prov, "GET", aData, nil) + mustDeny(t, prov, "POST", "identity/entity", map[string]any{"name": "evil", "policies": []string{"operator"}}) + mustDeny(t, prov, "POST", "identity/entity/name/approle-reader/env-a.acme", map[string]any{"policies": []string{"operator"}}) + mustDeny(t, prov, "POST", "identity/group", map[string]any{"name": "g"}) + mustDeny(t, prov, "PUT", "sys/policies/acl/tenant-reader", map[string]any{"policy": `path "kv/*" { capabilities = ["read"] }`}) + if err := prov.OffboardTenant("env-c", "gamma", false); err != nil { + t.Fatalf("provisioning offboard: %v", err) + } + if _, err := c.LoginAppRole(c.Cfg.TenantMount("reader"), "env-c.gamma"); err == nil { + t.Fatal("login after offboard succeeded") + } + if r, _ := c.Do("GET", "identity/entity/name/approle-reader/env-c.gamma", nil); r.Status != 404 { + t.Fatalf("entity survived offboard: HTTP %d", r.Status) + } + + // A login that happened before the entity existed is re-pointed at the metadata-bearing entity. + if err := c.writeAppRole("reader", "env-d.delta", []string{"tenant-reader"}); err != nil { + t.Fatal(err) + } + early, err := c.LoginAppRole(c.Cfg.TenantMount("reader"), "env-d.delta") + if err != nil { + t.Fatal(err) + } + mustDeny(t, c.WithToken(early), "GET", c.Cfg.KVDataPath("env-d", "delta", "x"), nil) + if err := c.CreateTenant("env-d", "delta"); err != nil { + t.Fatal(err) + } + late := login("reader", "env-d", "delta") + if got := httpStatus(t, late, "GET", c.Cfg.KVDataPath("env-d", "delta", "x"), nil); got != 404 { + t.Fatalf("after bind, own path should be reachable (404 = allowed, missing): HTTP %d", got) + } +} + +func TestAppsBrokerTokensShared(t *testing.T) { + c := startSharedVault(t) + for _, env := range []string{"env-a", "env-b"} { + if err := c.CreateTenant(env, "acme"); err != nil { + t.Fatal(err) + } + } + readerMount, writerMount := c.Cfg.TenantMount("reader"), c.Cfg.TenantMount("writer") + reader, writer := appBroker(t, c, "env-a", "reader"), appBroker(t, c, "env-a", "writer") + noEnv := appBroker(t, c, "", "reader") + + tok, err := reader.LoginAppRole(readerMount, "env-a.acme") + if err != nil { + t.Fatalf("broker login as its env's tenant: %v", err) + } + if got := httpStatus(t, c.WithToken(tok), "GET", c.Cfg.KVDataPath("env-a", "acme", "fixture"), nil); got != 404 { + t.Fatalf("tenant token reading own (missing) secret: HTTP %d want 404", got) + } + mustDeny(t, c.WithToken(tok), "GET", c.Cfg.KVDataPath("env-b", "acme", "fixture"), nil) + if _, err := writer.LoginAppRole(writerMount, "env-a.acme"); err != nil { + t.Fatalf("writer broker login: %v", err) + } + + // Other env: nothing. + mustDeny(t, reader, "GET", "auth/"+readerMount+"/role/env-b.acme/role-id", nil) + mustDeny(t, reader, "POST", "auth/"+readerMount+"/role/env-b.acme/secret-id", map[string]any{}) + if _, err := reader.LoginAppRole(readerMount, "env-b.acme"); err == nil { + t.Fatal("broker logged in as another env's tenant") + } + // Own env, but the role itself stays read-only: no field can be sent. + mustDeny(t, reader, "POST", "auth/"+readerMount+"/role/env-a.acme", map[string]any{"token_policies": []string{"operator"}}) + mustDeny(t, reader, "POST", "auth/"+readerMount+"/role/env-a.acme/role-id", map[string]any{"role_id": "chosen"}) + mustDeny(t, reader, "POST", "auth/"+readerMount+"/role/env-a.acme/policies", map[string]any{"token_policies": []string{"operator"}}) + mustDeny(t, reader, "POST", "auth/"+readerMount+"/role/env-a.acme/secret-id", map[string]any{"metadata": `{"env":"env-b"}`}) + mustDeny(t, reader, "DELETE", "auth/"+readerMount+"/role/env-a.acme", nil) + mustDeny(t, reader, "GET", "auth/"+readerMount+"/role?list=true", nil) + mustDeny(t, reader, "GET", "auth/"+readerMount+"/role/env-a.acme/secret-id?list=true", nil) + role, err := c.API.Logical().Read("auth/" + readerMount + "/role/env-a.acme") + if err != nil { + t.Fatal(err) + } + if p, _ := role.Data["token_policies"].([]any); len(p) != 1 || p[0] != "tenant-reader" { + t.Fatalf("role was changed by the broker: %v", role.Data) + } + // Wrong level, KV, identity, and policies stay closed. + mustDeny(t, reader, "GET", "auth/"+writerMount+"/role/env-a.acme/role-id", nil) + mustDeny(t, writer, "GET", "auth/"+readerMount+"/role/env-a.acme/role-id", nil) + mustDeny(t, reader, "GET", c.Cfg.KVDataPath("env-a", "acme", "fixture"), nil) + mustDeny(t, reader, "GET", "identity/entity/id?list=true", nil) + mustDeny(t, reader, "POST", "identity/entity", map[string]any{"name": "x", "metadata": map[string]any{"env": "env-b"}}) + mustDeny(t, reader, "GET", "sys/policies/acl/apps-reader", nil) + if got := httpStatus(t, reader, "GET", "auth/"+readerMount+"/role/env-a.nobody/role-id", nil); got != 404 { + t.Fatalf("unknown tenant in own env: HTTP %d want 404", got) + } + // No env on the entity: no role matches at all. + mustDeny(t, noEnv, "GET", "auth/"+readerMount+"/role/env-a.acme/role-id", nil) + mustDeny(t, noEnv, "GET", "auth/"+readerMount+"/role/env-b.acme/role-id", nil) +} + +func TestEnvsAndTenantListShared(t *testing.T) { + c := startSharedVault(t) + if envs, err := c.ListEnvs(); err != nil || len(envs) != 0 { + t.Fatalf("empty cluster: %v %v", envs, err) + } + for _, tc := range [][2]string{{"env-b", "acme"}, {"env-a", "beta"}, {"env-a", "acme"}} { + if err := c.CreateTenant(tc[0], tc[1]); err != nil { + t.Fatal(err) + } + } + // An unscoped role left over from before the cluster was shared shows up flagged. + if err := c.writeAppRole("reader", "legacy", []string{"tenant-reader"}); err != nil { + t.Fatal(err) + } + envs, err := c.ListEnvs() + if err != nil || strings.Join(envs, ",") != "env-a,env-b" { + t.Fatalf("envs = %v, %v", envs, err) + } + tenants, err := c.ListTenants() + if err != nil { + t.Fatal(err) + } + want := []Tenant{{ID: "legacy", Reader: true}, {Env: "env-a", ID: "acme", Reader: true, Writer: true}, {Env: "env-a", ID: "beta", Reader: true, Writer: true}, {Env: "env-b", ID: "acme", Reader: true, Writer: true}} + if len(tenants) != len(want) { + t.Fatalf("got %+v want %+v", tenants, want) + } + for i := range want { + if tenants[i] != want[i] { + t.Fatalf("got %+v want %+v", tenants, want) + } + } + var out strings.Builder + PrintTenants(&out, tenants, true) + if !strings.Contains(out.String(), "legacy\t(no writer role; re-run tenants create; not env-scoped; this cluster is shared)") || !strings.Contains(out.String(), "env-a\tacme\n") { + t.Fatalf("output:\n%s", out.String()) + } + + // Destroy one env with purge: its roles, entities, and secrets go; the other env is untouched. + tokA, err := c.LoginAppRole(c.Cfg.TenantMount("writer"), "env-a.acme") + if err != nil { + t.Fatal(err) + } + for _, p := range []string{c.Cfg.KVDataPath("env-a", "acme", "cfg"), c.Cfg.KVDataPath("env-a", "acme", "nested/deep")} { + if got := httpStatus(t, c.WithToken(tokA), "POST", p, map[string]any{"data": map[string]any{"v": "FAKE"}}); got != 200 { + t.Fatalf("seed %s: HTTP %d", p, got) + } + } + tokB, err := c.LoginAppRole(c.Cfg.TenantMount("writer"), "env-b.acme") + if err != nil { + t.Fatal(err) + } + if got := httpStatus(t, c.WithToken(tokB), "POST", c.Cfg.KVDataPath("env-b", "acme", "cfg"), map[string]any{"data": map[string]any{"v": "FAKE"}}); got != 200 { + t.Fatalf("seed env-b: HTTP %d", got) + } + if err := c.DestroyEnv("env-a", true); err != nil { + t.Fatal(err) + } + envs, _ = c.ListEnvs() + if strings.Join(envs, ",") != "env-b" { + t.Fatalf("envs after destroy = %v", envs) + } + if r, _ := c.Do("GET", "kv/metadata/envs/env-a?list=true", nil); r.Status != 404 { + t.Fatalf("env-a secrets survived purge: HTTP %d %s", r.Status, r.Body) + } + if r, _ := c.Do("GET", c.Cfg.KVDataPath("env-b", "acme", "cfg"), nil); r.Status != 200 { + t.Fatalf("env-b secret lost: HTTP %d", r.Status) + } + if r, _ := c.Do("GET", "identity/entity/name/approle-reader/env-a.acme", nil); r.Status != 404 { + t.Fatalf("env-a entity survived: HTTP %d", r.Status) + } + if _, err := c.LoginAppRole(c.Cfg.TenantMount("reader"), "env-a.acme"); err == nil { + t.Fatal("env-a login survived destroy") + } + + unshared := c.WithToken(c.Cfg.Token) + unshared.Cfg.SharedEnvs = false + if _, err := unshared.ListEnvs(); err == nil { + t.Fatal("envs list on an unshared cluster succeeded") + } + if err := unshared.DestroyEnv("env-b", false); err == nil { + t.Fatal("envs destroy on an unshared cluster succeeded") + } +} diff --git a/internal/vaultcluster/tenant.go b/internal/vaultcluster/tenant.go index 408d4b6..a51bc32 100644 --- a/internal/vaultcluster/tenant.go +++ b/internal/vaultcluster/tenant.go @@ -16,39 +16,53 @@ const ( tenantSecretIDUses = 1 ) -// CreateTenant writes the tenant's reader and writer AppRole roles (role name = tenant ID) and, with -// credentials enabled, its database roles. Tenant policies are static, so nothing here writes a policy. -// No credentials are issued: apps mint their own at login. -func (c *Client) CreateTenant(tenantID string) error { +// CreateTenant writes the tenant's reader and writer AppRole roles and, with credentials enabled, its +// database roles. Tenant policies are static, so nothing here writes a policy. No credentials are issued: +// apps mint their own at login. +// +// env is empty on an unshared cluster. On a shared cluster it is required: the roles are named env.tenant +// and each gets an entity carrying env and tenant as metadata, which the tenant policies template. +func (c *Client) CreateTenant(env, tenantID string) error { if err := ValidateTenantID(tenantID); err != nil { return err } - if err := c.writeAppRole("reader", tenantID, []string{c.Cfg.TenantPolicy("reader")}); err != nil { + if err := c.Cfg.checkEnv(env); err != nil { + return err + } + role := c.Cfg.TenantRole(env, tenantID) + if err := c.writeAppRole("reader", role, []string{c.Cfg.TenantPolicy("reader")}); err != nil { return err } writerPolicies := []string{c.Cfg.TenantPolicy("writer")} if c.Cfg.EnableCredentials { writerPolicies = append(writerPolicies, c.Cfg.TenantPolicy("database")) } - if err := c.writeAppRole("writer", tenantID, writerPolicies); err != nil { + if err := c.writeAppRole("writer", role, writerPolicies); err != nil { return err } + if c.Cfg.SharedEnvs { + for _, kind := range []string{"reader", "writer"} { + if err := c.bindTenantEntity(kind, env, tenantID); err != nil { + return err + } + } + } if c.Cfg.EnableCredentials { - if err := c.writeDBRole(tenantID, "readonly", "app_readonly"); err != nil { + if err := c.writeDBRole(role, "readonly", "app_readonly"); err != nil { return err } - if err := c.writeDBRole(tenantID, "readwrite", "app_readwrite"); err != nil { + if err := c.writeDBRole(role, "readwrite", "app_readwrite"); err != nil { return err } } - log.Printf("tenant %s onboarded", tenantID) + log.Printf("tenant %s onboarded", role) return nil } -func (c *Client) writeAppRole(kind, tenantID string, policies []string) error { - _, err := c.API.Logical().Write("auth/"+c.Cfg.TenantMount(kind)+"/role/"+tenantID, map[string]any{ +func (c *Client) writeAppRole(kind, role string, policies []string) error { + _, err := c.API.Logical().Write("auth/"+c.Cfg.TenantMount(kind)+"/role/"+role, map[string]any{ "token_policies": policies, "token_ttl": c.Cfg.TokenTTL, "token_max_ttl": c.Cfg.TokenMaxTTL, @@ -60,6 +74,97 @@ func (c *Client) writeAppRole(kind, tenantID string, policies []string) error { return err } +// tenantEntityName is the identity entity for one tenant role on one mount. +func (c *Client) tenantEntityName(kind, env, tenantID string) string { + return c.Cfg.TenantMount(kind) + "/" + c.Cfg.TenantRole(env, tenantID) +} + +// bindTenantEntity gives the tenant role on one mount an entity with env and tenant metadata, aliased by the +// role's role_id so every login on that role lands on it. Re-runs are no-ops; an entity Vault auto-created for +// an earlier login is unlinked so the metadata-bearing one wins. +func (c *Client) bindTenantEntity(kind, env, tenantID string) error { + mount := c.Cfg.TenantMount(kind) + role := c.Cfg.TenantRole(env, tenantID) + rid, err := c.API.Logical().Read("auth/" + mount + "/role/" + role + "/role-id") + if err != nil { + return err + } + if rid == nil || rid.Data["role_id"] == nil { + return fmt.Errorf("role %s/%s has no role_id", mount, role) + } + accessor, err := c.authAccessor(mount) + if err != nil { + return err + } + return c.ensureEntityAlias(c.tenantEntityName(kind, env, tenantID), + map[string]any{entityMetaEnv: env, entityMetaTenant: tenantID}, + fmt.Sprint(rid.Data["role_id"]), accessor) +} + +func (c *Client) authAccessor(mount string) (string, error) { + auths, err := c.API.Sys().ListAuth() + if err != nil { + return "", err + } + a := auths[mount+"/"] + if a == nil || a.Accessor == "" { + return "", fmt.Errorf("auth mount %s has no accessor", mount) + } + return a.Accessor, nil +} + +// ensureEntityAlias upserts the entity by name with metadata and points the (aliasName, accessor) alias at it. +func (c *Client) ensureEntityAlias(name string, metadata map[string]any, aliasName, accessor string) error { + if _, err := c.API.Logical().Write("identity/entity/name/"+name, map[string]any{"metadata": metadata}); err != nil { + return fmt.Errorf("write entity %s: %w", name, err) + } + ent, err := c.API.Logical().Read("identity/entity/name/" + name) + if err != nil { + return err + } + if ent == nil || ent.Data["id"] == nil { + return fmt.Errorf("entity %s was not created", name) + } + entityID := fmt.Sprint(ent.Data["id"]) + + existing, err := c.API.Logical().Write("identity/lookup/entity", map[string]any{ + "alias_name": aliasName, "alias_mount_accessor": accessor, + }) + if err != nil { + return fmt.Errorf("lookup alias: %w", err) + } + if existing != nil && existing.Data != nil { + if fmt.Sprint(existing.Data["id"]) == entityID { + return nil + } + for _, a := range aliasList(existing.Data["aliases"]) { + if a["name"] == aliasName && a["mount_accessor"] == accessor { + if err := c.deleteMissingOK("identity/entity-alias/id/" + fmt.Sprint(a["id"])); err != nil { + return fmt.Errorf("unlink stale alias: %w", err) + } + } + } + } + _, err = c.API.Logical().Write("identity/entity-alias", map[string]any{ + "name": aliasName, "canonical_id": entityID, "mount_accessor": accessor, + }) + if err != nil { + return fmt.Errorf("write alias for %s: %w", name, err) + } + return nil +} + +func aliasList(v any) []map[string]any { + raw, _ := v.([]any) + out := make([]map[string]any, 0, len(raw)) + for _, r := range raw { + if m, ok := r.(map[string]any); ok { + out = append(out, m) + } + } + return out +} + // dbCreationStatements are the only statements a tenant database role may run. The provisioning // policy pins creation_statements to these values, so a role cannot grant more than the app groups. func dbCreationStatements() []string { @@ -70,8 +175,8 @@ func dbCreationStatements() []string { } } -func (c *Client) writeDBRole(tenantID, suffix, group string) error { - role := fmt.Sprintf("tenant-%s-%s", tenantID, suffix) +func (c *Client) writeDBRole(tenantRole, suffix, group string) error { + role := fmt.Sprintf("tenant-%s-%s", tenantRole, suffix) stmts := dbCreationStatements() grant := fmt.Sprintf(`GRANT %s TO "{{name}}";`, group) found := false @@ -117,18 +222,27 @@ func (c *Client) LoginAppRole(mount, role string) (string, error) { return login.Auth.ClientToken, nil } -func (c *Client) OffboardTenant(tenantID string, purge bool) error { +func (c *Client) OffboardTenant(env, tenantID string, purge bool) error { if err := ValidateTenantID(tenantID); err != nil { return err } + if err := c.Cfg.checkEnv(env); err != nil { + return err + } + role := c.Cfg.TenantRole(env, tenantID) for _, kind := range []string{"reader", "writer"} { - if err := c.deleteMissingOK("auth/" + c.Cfg.TenantMount(kind) + "/role/" + tenantID); err != nil { - return fmt.Errorf("delete %s role %s: %w", kind, tenantID, err) + if c.Cfg.SharedEnvs { + if err := c.deleteMissingOK("identity/entity/name/" + c.tenantEntityName(kind, env, tenantID)); err != nil { + return fmt.Errorf("delete %s entity %s: %w", kind, role, err) + } + } + if err := c.deleteMissingOK("auth/" + c.Cfg.TenantMount(kind) + "/role/" + role); err != nil { + return fmt.Errorf("delete %s role %s: %w", kind, role, err) } } if c.Cfg.EnableCredentials { for _, suffix := range []string{"readonly", "readwrite"} { - dbRole := fmt.Sprintf("tenant-%s-%s", tenantID, suffix) + dbRole := fmt.Sprintf("tenant-%s-%s", role, suffix) _ = c.API.Sys().RevokePrefix(c.Cfg.DatabaseMount + "/creds/" + dbRole) if err := c.deleteMissingOK(c.Cfg.DatabaseMount + "/roles/" + dbRole); err != nil { return fmt.Errorf("delete database role %s: %w", dbRole, err) @@ -136,16 +250,16 @@ func (c *Client) OffboardTenant(tenantID string, purge bool) error { } } if purge { - if err := c.purgeTenantSecrets(tenantID); err != nil { + if err := c.purgeKVPrefix(c.Cfg.KVMetaPath(env, tenantID, "")); err != nil { return err } } - log.Printf("tenant %s offboarded", tenantID) + log.Printf("tenant %s offboarded", role) return nil } -func (c *Client) purgeTenantSecrets(tenantID string) error { - meta := c.Cfg.KVMetaPath(tenantID, "") +// purgeKVPrefix destroys every secret under one KV v2 metadata path, recursing into folders. +func (c *Client) purgeKVPrefix(meta string) error { r, err := c.Do("GET", meta+"?list=true", nil) if err != nil && r.Status == 0 { return err @@ -168,7 +282,13 @@ func (c *Client) purgeTenantSecrets(tenantID string) error { return err } for _, key := range wrap.Data.Keys { - path := strings.TrimSuffix(meta+"/"+strings.TrimSuffix(key, "/"), "/") + if strings.HasSuffix(key, "/") { + if err := c.purgeKVPrefix(meta + "/" + strings.TrimSuffix(key, "/")); err != nil { + return err + } + continue + } + path := meta + "/" + key if err := c.deleteMissingOK(path); err != nil { return fmt.Errorf("destroy %s: %w", path, err) } diff --git a/internal/vaultcluster/tenantid.go b/internal/vaultcluster/tenantid.go index c5474e2..13592cf 100644 --- a/internal/vaultcluster/tenantid.go +++ b/internal/vaultcluster/tenantid.go @@ -13,7 +13,7 @@ var reservedTenantIDs = map[string]struct{}{ "sys": {}, "auth": {}, "identity": {}, "cubbyhole": {}, "root": {}, "default": {}, "admin": {}, "data": {}, "metadata": {}, "delete": {}, "undelete": {}, "destroy": {}, "config": {}, "subkeys": {}, "tenant": {}, - "customers": {}, + "customers": {}, "envs": {}, } func ValidateTenantID(id string) error { @@ -56,3 +56,20 @@ func ValidateTenantID(id string) error { } return nil } + +// ValidateEnvName accepts a Nullstone env name as a path segment: the tenant ID character class, so a +// name never contains "." (the role separator), "/", or a glob. +func ValidateEnvName(env string) error { + if env == "" { + return fmt.Errorf("env name is empty") + } + if err := ValidateTenantID(env); err != nil { + if _, reserved := reservedTenantIDs[env]; !reserved { + return fmt.Errorf("env name %q: %w", env, err) + } + if !tenantIDPattern.MatchString(env) { + return fmt.Errorf("env name %q does not match %s", env, tenantIDPattern) + } + } + return nil +} diff --git a/internal/vaultcluster/tenantlist.go b/internal/vaultcluster/tenantlist.go index 76a61cd..4210229 100644 --- a/internal/vaultcluster/tenantlist.go +++ b/internal/vaultcluster/tenantlist.go @@ -10,25 +10,33 @@ import ( // Tenant is one onboarded tenant. The AppRole roles are the record of truth: a tenant exists exactly when // its roles exist. Reader or Writer is false when that mount has no role, which a failed create can leave. +// Env is set on a shared cluster (role name env.tenant) and empty otherwise. type Tenant struct { + Env string ID string Reader bool Writer bool } -// ListTenants reads the role names on both tenant mounts and merges them by tenant ID. +// Role is the AppRole role name this tenant logs in with. +func (t Tenant) Role() string { + return Config{}.TenantRole(t.Env, t.ID) +} + +// ListTenants reads the role names on both tenant mounts and merges them by role name. func (c *Client) ListTenants() ([]Tenant, error) { - byID := map[string]*Tenant{} + byRole := map[string]*Tenant{} for _, kind := range []string{"reader", "writer"} { names, err := c.listRoles(c.Cfg.TenantMount(kind)) if err != nil { return nil, err } for _, name := range names { - t := byID[name] + t := byRole[name] if t == nil { - t = &Tenant{ID: name} - byID[name] = t + env, id := SplitTenantRole(name) + t = &Tenant{Env: env, ID: id} + byRole[name] = t } if kind == "reader" { t.Reader = true @@ -37,11 +45,16 @@ func (c *Client) ListTenants() ([]Tenant, error) { } } } - out := make([]Tenant, 0, len(byID)) - for _, t := range byID { + out := make([]Tenant, 0, len(byRole)) + for _, t := range byRole { out = append(out, *t) } - sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID }) + sort.Slice(out, func(i, j int) bool { + if out[i].Env != out[j].Env { + return out[i].Env < out[j].Env + } + return out[i].ID < out[j].ID + }) return out, nil } @@ -67,20 +80,36 @@ func (c *Client) listRoles(mount string) ([]string, error) { return listed.Data.Keys, nil } -// PrintTenants writes one line per tenant and marks tenants missing a role on either mount. -func PrintTenants(w io.Writer, tenants []Tenant) { +// PrintTenants writes one line per tenant and marks tenants missing a role on either mount or whose role +// name does not fit the cluster mode. shared selects the env column. +func PrintTenants(w io.Writer, tenants []Tenant, shared bool) { if len(tenants) == 0 { fmt.Fprintln(w, "no tenants") return } for _, t := range tenants { + label := t.ID + if shared { + label = t.Env + "\t" + t.ID + } + var notes []string switch { case t.Reader && t.Writer: - fmt.Fprintln(w, t.ID) case t.Reader: - fmt.Fprintf(w, "%s\t(no writer role; re-run tenants create)\n", t.ID) + notes = append(notes, "no writer role; re-run tenants create") default: - fmt.Fprintf(w, "%s\t(no reader role; re-run tenants create)\n", t.ID) + notes = append(notes, "no reader role; re-run tenants create") + } + if shared && t.Env == "" { + notes = append(notes, "not env-scoped; this cluster is shared") + } + if !shared && t.Env != "" { + notes = append(notes, "env-scoped; this cluster is not shared") + } + if len(notes) == 0 { + fmt.Fprintln(w, label) + continue } + fmt.Fprintf(w, "%s\t(%s)\n", label, strings.Join(notes, "; ")) } } diff --git a/internal/vaultcluster/testdata/lint/BAD-cross-env-wildcard.hcl b/internal/vaultcluster/testdata/lint/BAD-cross-env-wildcard.hcl new file mode 100644 index 0000000..98928e5 --- /dev/null +++ b/internal/vaultcluster/testdata/lint/BAD-cross-env-wildcard.hcl @@ -0,0 +1,11 @@ +# MUST BE REJECTED - shared cluster: the wildcard sits under the env prefix, so it +# covers every env, or under one env, so it covers every tenant in it. +path "kv/data/envs/*" { + capabilities = ["read", "list"] +} +path "kv/data/envs/+/customers/*" { + capabilities = ["read"] +} +path "kv/metadata/envs/dev/customers/*" { + capabilities = ["read", "list"] +} diff --git a/internal/vaultcluster/testdata/lint/GOOD-env-scoped.hcl b/internal/vaultcluster/testdata/lint/GOOD-env-scoped.hcl new file mode 100644 index 0000000..348df38 --- /dev/null +++ b/internal/vaultcluster/testdata/lint/GOOD-env-scoped.hcl @@ -0,0 +1,17 @@ +# MUST PASS - shared cluster: scoped to one env and one tenant, denies the env +# prefix for everything else. +path "kv/data/envs/{{identity.entity.metadata.env}}/customers/{{identity.entity.metadata.tenant}}/*" { + capabilities = ["create", "read", "update", "delete", "list"] +} +path "kv/metadata/envs/dev/customers/tenant-a/*" { + capabilities = ["read", "list"] +} +path "kv/data/envs/*" { + capabilities = ["deny"] +} +path "kv/metadata/envs/*" { + capabilities = ["deny"] +} +path "sys/*" { + capabilities = ["deny"] +} diff --git a/internal/vaultcluster/testdata/policies/shared/apps-auth.hcl b/internal/vaultcluster/testdata/policies/shared/apps-auth.hcl new file mode 100644 index 0000000..123369f --- /dev/null +++ b/internal/vaultcluster/testdata/policies/shared/apps-auth.hcl @@ -0,0 +1,84 @@ +# Writes app auth roles through the cluster function, on the aws and gcp mounts. Cannot read tenant secrets. +path "sys/auth" { + capabilities = ["read"] +} + +path "sys/auth/aws" { + capabilities = ["create", "read", "update", "sudo"] +} + +path "sys/auth/gcp" { + capabilities = ["create", "read", "update", "sudo"] +} + +path "auth/aws/role" { + capabilities = ["list"] +} + +path "auth/aws/role/*" { + capabilities = ["create", "read", "update", "delete", "list"] +} + +path "auth/gcp/role" { + capabilities = ["list"] +} + +path "auth/gcp/role/*" { + capabilities = ["create", "read", "update", "delete", "list"] +} + +path "auth/token/lookup-self" { + capabilities = ["read"] +} + +path "auth/token/renew-self" { + capabilities = ["update"] +} + +path "kv/*" { + capabilities = ["deny"] +} + +path "database/creds/*" { + capabilities = ["deny"] +} + +# Shared cluster: the function writes one entity per app role (metadata env) and its alias, so the broker +# policies can scope the app to its env. policies and disabled are refused. +path "identity/entity" { + capabilities = ["create", "update"] + denied_parameters = { "policies" = [], "disabled" = [] } +} + +path "identity/entity/name/*" { + capabilities = ["read", "update", "delete"] + denied_parameters = { "policies" = [], "disabled" = [] } +} + +path "identity/entity-alias" { + capabilities = ["create", "update"] +} + +path "identity/entity-alias/id/*" { + capabilities = ["read", "delete"] +} + +path "identity/lookup/entity" { + capabilities = ["update"] +} + +path "identity/entity/merge" { + capabilities = ["deny"] +} + +path "identity/group*" { + capabilities = ["deny"] +} + +path "identity/oidc/*" { + capabilities = ["deny"] +} + +path "identity/mfa/*" { + capabilities = ["deny"] +} diff --git a/internal/vaultcluster/testdata/policies/shared/apps-reader.hcl b/internal/vaultcluster/testdata/policies/shared/apps-reader.hcl new file mode 100644 index 0000000..3c492d5 --- /dev/null +++ b/internal/vaultcluster/testdata/policies/shared/apps-reader.hcl @@ -0,0 +1,40 @@ +# App broker, read level. Lets an app log in as any onboarded tenant's reader: read the role ID and mint a +# one-time secret ID on approle-reader. The token that login yields sees one tenant; this token sees none. +# Shared cluster: only roles of the app's own env (entity metadata). No parameter may be sent, so the role, +# its role-id, and its policies cannot be changed; an empty-body secret-id mint still works. +path "auth/approle-reader/role/{{identity.entity.metadata.env}}.*" { + capabilities = ["read", "update"] + denied_parameters = { "*" = [] } +} + +path "auth/token/lookup-self" { + capabilities = ["read"] +} + +path "auth/token/renew-self" { + capabilities = ["update"] +} + +path "auth/approle-writer/*" { + capabilities = ["deny"] +} + +path "kv/*" { + capabilities = ["deny"] +} + +path "database/*" { + capabilities = ["deny"] +} + +path "sys/*" { + capabilities = ["deny"] +} + +path "auth/token/create*" { + capabilities = ["deny"] +} + +path "identity/*" { + capabilities = ["deny"] +} diff --git a/internal/vaultcluster/testdata/policies/shared/apps-writer.hcl b/internal/vaultcluster/testdata/policies/shared/apps-writer.hcl new file mode 100644 index 0000000..f543a7e --- /dev/null +++ b/internal/vaultcluster/testdata/policies/shared/apps-writer.hcl @@ -0,0 +1,40 @@ +# App broker, write level. Lets an app log in as any onboarded tenant's writer: read the role ID and mint a +# one-time secret ID on approle-writer. The token that login yields sees one tenant; this token sees none. +# Shared cluster: only roles of the app's own env (entity metadata). No parameter may be sent, so the role, +# its role-id, and its policies cannot be changed; an empty-body secret-id mint still works. +path "auth/approle-writer/role/{{identity.entity.metadata.env}}.*" { + capabilities = ["read", "update"] + denied_parameters = { "*" = [] } +} + +path "auth/token/lookup-self" { + capabilities = ["read"] +} + +path "auth/token/renew-self" { + capabilities = ["update"] +} + +path "auth/approle-reader/*" { + capabilities = ["deny"] +} + +path "kv/*" { + capabilities = ["deny"] +} + +path "database/*" { + capabilities = ["deny"] +} + +path "sys/*" { + capabilities = ["deny"] +} + +path "auth/token/create*" { + capabilities = ["deny"] +} + +path "identity/*" { + capabilities = ["deny"] +} diff --git a/internal/vaultcluster/testdata/policies/shared/operator.hcl b/internal/vaultcluster/testdata/policies/shared/operator.hcl new file mode 100644 index 0000000..0625271 --- /dev/null +++ b/internal/vaultcluster/testdata/policies/shared/operator.hcl @@ -0,0 +1,140 @@ +# Local operator: health and config read. Can start generate-root (recovery keys still required). Not a tenant secret reader. No restore. +path "sys/health" { + capabilities = ["read", "sudo"] +} + +path "sys/seal-status" { + capabilities = ["read"] +} + +path "sys/leader" { + capabilities = ["read"] +} + +path "sys/mounts" { + capabilities = ["read", "sudo"] +} + +path "sys/mounts/database" { + capabilities = ["create", "update", "sudo"] +} + +path "sys/mounts/*" { + capabilities = ["read"] +} + +path "sys/auth" { + capabilities = ["read"] +} + +path "sys/policies/acl" { + capabilities = ["list"] +} + +path "sys/policies/acl/*" { + capabilities = ["read"] +} + +path "sys/audit" { + capabilities = ["read", "sudo"] +} + +path "auth/approle-reader/role" { + capabilities = ["list"] +} + +path "auth/approle-reader/role/*" { + capabilities = ["read"] +} + +path "auth/approle-writer/role" { + capabilities = ["list"] +} + +path "auth/approle-writer/role/*" { + capabilities = ["read"] +} + +path "database/roles" { + capabilities = ["list"] +} + +path "database/roles/*" { + capabilities = ["read"] +} + +path "database/config/*" { + capabilities = ["create", "update", "read"] +} + +path "sys/leases/lookup" { + capabilities = ["update"] +} + +path "sys/leases/lookup/*" { + capabilities = ["list", "read"] +} + +path "sys/leases/revoke" { + capabilities = ["update"] +} + +path "sys/leases/revoke-prefix/*" { + capabilities = ["update", "sudo"] +} + +path "sys/generate-root" { + capabilities = ["create", "update", "read", "delete", "sudo"] +} + +path "sys/generate-root/*" { + capabilities = ["create", "update", "read", "delete", "sudo"] +} + +path "sys/storage/raft/snapshot" { + capabilities = ["read"] +} + +path "sys/storage/raft/configuration" { + capabilities = ["read"] +} + +path "sys/storage/raft/autopilot/state" { + capabilities = ["read"] +} + +path "auth/token/lookup-self" { + capabilities = ["read"] +} + +path "auth/token/renew-self" { + capabilities = ["update"] +} + +path "auth/token/revoke-self" { + capabilities = ["update"] +} + +path "kv/data/*" { + capabilities = ["deny"] +} + +path "kv/metadata/*" { + capabilities = ["deny"] +} + +path "database/creds/*" { + capabilities = ["deny"] +} + +path "sys/policies/acl/operator" { + capabilities = ["deny"] +} + +path "sys/audit/*" { + capabilities = ["deny"] +} + +path "sys/storage/raft/snapshot-force" { + capabilities = ["deny"] +} diff --git a/internal/vaultcluster/testdata/policies/shared/provisioning.hcl b/internal/vaultcluster/testdata/policies/shared/provisioning.hcl new file mode 100644 index 0000000..138b1c6 --- /dev/null +++ b/internal/vaultcluster/testdata/policies/shared/provisioning.hcl @@ -0,0 +1,158 @@ +# Onboard/offboard tenants. Writes AppRole and database roles only; tenant policies are static (bootstrap). +# allowed_parameters pins every role to the tenant policies and fixed SQL, so nothing written here +# can carry a platform policy. Can mint any tenant's credentials; cannot read tenant secrets itself. +path "auth/approle-reader/role" { + capabilities = ["list"] +} + +path "auth/approle-reader/role/*" { + capabilities = ["create", "read", "update", "delete", "list"] + allowed_parameters = { + "token_policies" = ["tenant-reader"] + "token_ttl" = [] + "token_max_ttl" = [] + "token_type" = ["service"] + "secret_id_ttl" = [] + "secret_id_num_uses" = [] + "bind_secret_id" = [] + } +} + +path "auth/approle-reader/role/+/role-id" { + capabilities = ["read"] +} + +path "auth/approle-reader/role/+/secret-id" { + capabilities = ["update"] +} + +path "auth/approle-writer/role" { + capabilities = ["list"] +} + +path "auth/approle-writer/role/*" { + capabilities = ["create", "read", "update", "delete", "list"] + allowed_parameters = { + "token_policies" = ["tenant-writer", "tenant-database"] + "token_ttl" = [] + "token_max_ttl" = [] + "token_type" = ["service"] + "secret_id_ttl" = [] + "secret_id_num_uses" = [] + "bind_secret_id" = [] + } +} + +path "auth/approle-writer/role/+/role-id" { + capabilities = ["read"] +} + +path "auth/approle-writer/role/+/secret-id" { + capabilities = ["update"] +} + +path "database/roles/tenant-*" { + capabilities = ["create", "read", "update", "delete", "list"] + allowed_parameters = { + "db_name" = [] + "creation_statements" = ["CREATE ROLE \"{{name}}\" WITH LOGIN PASSWORD '{{password}}' VALID UNTIL '{{expiration}}';", "GRANT app_readonly TO \"{{name}}\";", "GRANT app_readwrite TO \"{{name}}\";"] + "default_ttl" = [] + "max_ttl" = [] + } +} + +path "sys/mounts" { + capabilities = ["read"] +} + +path "sys/auth" { + capabilities = ["read"] +} + +path "auth/token/lookup-self" { + capabilities = ["read"] +} + +path "auth/token/renew-self" { + capabilities = ["update"] +} + +path "kv/*" { + capabilities = ["deny"] +} + +path "sys/policies/*" { + capabilities = ["deny"] +} + +path "sys/policy/*" { + capabilities = ["deny"] +} + +path "sys/audit" { + capabilities = ["deny"] +} + +path "sys/audit/*" { + capabilities = ["deny"] +} + +path "sys/audit-hash/*" { + capabilities = ["deny"] +} + +path "database/creds/*" { + capabilities = ["deny"] +} + +path "sys/mounts/*" { + capabilities = ["deny"] +} + +path "sys/auth/*" { + capabilities = ["deny"] +} + +path "auth/token/create*" { + capabilities = ["deny"] +} + +# Shared cluster: tenants create writes one entity per tenant role (metadata env and tenant) and its alias. +# policies and disabled are refused, so an entity can never carry more than its roles grant. +path "identity/entity" { + capabilities = ["create", "update"] + denied_parameters = { "policies" = [], "disabled" = [] } +} + +path "identity/entity/name/*" { + capabilities = ["read", "update", "delete"] + denied_parameters = { "policies" = [], "disabled" = [] } +} + +path "identity/entity-alias" { + capabilities = ["create", "update"] +} + +path "identity/entity-alias/id/*" { + capabilities = ["read", "delete"] +} + +path "identity/lookup/entity" { + capabilities = ["update"] +} + +path "identity/entity/merge" { + capabilities = ["deny"] +} + +path "identity/group*" { + capabilities = ["deny"] +} + +path "identity/oidc/*" { + capabilities = ["deny"] +} + +path "identity/mfa/*" { + capabilities = ["deny"] +} diff --git a/internal/vaultcluster/testdata/policies/shared/tenant-database.hcl b/internal/vaultcluster/testdata/policies/shared/tenant-database.hcl new file mode 100644 index 0000000..0c54eaf --- /dev/null +++ b/internal/vaultcluster/testdata/policies/shared/tenant-database.hcl @@ -0,0 +1,36 @@ +# Dynamic DB creds for the tenant named by the caller's approle-writer role only. Additive; does not widen KV access. +path "database/creds/tenant-{{identity.entity.metadata.env}}.{{identity.entity.metadata.tenant}}-*" { + capabilities = ["read"] +} + +path "database/roles/tenant-{{identity.entity.metadata.env}}.{{identity.entity.metadata.tenant}}-*" { + capabilities = ["read"] +} + +path "sys/leases/renew" { + capabilities = ["update"] +} + +path "sys/leases/revoke" { + capabilities = ["update"] +} + +path "database/creds/*" { + capabilities = ["deny"] +} + +path "database/roles/*" { + capabilities = ["deny"] +} + +path "database/config/*" { + capabilities = ["deny"] +} + +path "database/static-creds/*" { + capabilities = ["deny"] +} + +path "database/rotate-root/*" { + capabilities = ["deny"] +} diff --git a/internal/vaultcluster/testdata/policies/shared/tenant-reader.hcl b/internal/vaultcluster/testdata/policies/shared/tenant-reader.hcl new file mode 100644 index 0000000..6a16c80 --- /dev/null +++ b/internal/vaultcluster/testdata/policies/shared/tenant-reader.hcl @@ -0,0 +1,61 @@ +# Read-only on the tenant named by the caller's AppRole role (approle-reader mount; role name = tenant ID). Static: written once at bootstrap. +# KV v2 needs data/ and metadata/; a token without a approle-reader alias renders no tenant path. +path "kv/data/envs/{{identity.entity.metadata.env}}/customers/{{identity.entity.metadata.tenant}}/*" { + capabilities = ["read"] +} + +path "kv/metadata/envs/{{identity.entity.metadata.env}}/customers/{{identity.entity.metadata.tenant}}/*" { + capabilities = ["read", "list"] +} + +path "kv/data/envs/*" { + capabilities = ["deny"] +} + +path "kv/metadata/envs/*" { + capabilities = ["deny"] +} + +path "kv/delete/envs/*" { + capabilities = ["deny"] +} + +path "kv/undelete/envs/*" { + capabilities = ["deny"] +} + +path "kv/destroy/envs/*" { + capabilities = ["deny"] +} + +path "auth/token/lookup-self" { + capabilities = ["read"] +} + +path "auth/token/renew-self" { + capabilities = ["update"] +} + +path "sys/*" { + capabilities = ["deny"] +} + +path "auth/token/create*" { + capabilities = ["deny"] +} + +path "auth/approle-reader/role/*" { + capabilities = ["deny"] +} + +path "auth/approle-writer/role/*" { + capabilities = ["deny"] +} + +path "identity/*" { + capabilities = ["deny"] +} + +path "kv/config" { + capabilities = ["deny"] +} diff --git a/internal/vaultcluster/testdata/policies/shared/tenant-writer.hcl b/internal/vaultcluster/testdata/policies/shared/tenant-writer.hcl new file mode 100644 index 0000000..2752bd6 --- /dev/null +++ b/internal/vaultcluster/testdata/policies/shared/tenant-writer.hcl @@ -0,0 +1,73 @@ +# Full KV lifecycle on the tenant named by the caller's AppRole role (approle-writer mount; role name = tenant ID). Static: written once at bootstrap. +# Name all five KV v2 path families. +path "kv/data/envs/{{identity.entity.metadata.env}}/customers/{{identity.entity.metadata.tenant}}/*" { + capabilities = ["create", "read", "update", "patch", "delete", "list"] +} + +path "kv/metadata/envs/{{identity.entity.metadata.env}}/customers/{{identity.entity.metadata.tenant}}/*" { + capabilities = ["create", "read", "update", "delete", "list"] +} + +path "kv/delete/envs/{{identity.entity.metadata.env}}/customers/{{identity.entity.metadata.tenant}}/*" { + capabilities = ["update"] +} + +path "kv/undelete/envs/{{identity.entity.metadata.env}}/customers/{{identity.entity.metadata.tenant}}/*" { + capabilities = ["update"] +} + +path "kv/destroy/envs/{{identity.entity.metadata.env}}/customers/{{identity.entity.metadata.tenant}}/*" { + capabilities = ["update"] +} + +path "kv/data/envs/*" { + capabilities = ["deny"] +} + +path "kv/metadata/envs/*" { + capabilities = ["deny"] +} + +path "kv/delete/envs/*" { + capabilities = ["deny"] +} + +path "kv/undelete/envs/*" { + capabilities = ["deny"] +} + +path "kv/destroy/envs/*" { + capabilities = ["deny"] +} + +path "auth/token/lookup-self" { + capabilities = ["read"] +} + +path "auth/token/renew-self" { + capabilities = ["update"] +} + +path "sys/*" { + capabilities = ["deny"] +} + +path "auth/token/create*" { + capabilities = ["deny"] +} + +path "auth/approle-reader/role/*" { + capabilities = ["deny"] +} + +path "auth/approle-writer/role/*" { + capabilities = ["deny"] +} + +path "identity/*" { + capabilities = ["deny"] +} + +path "kv/config" { + capabilities = ["deny"] +} diff --git a/internal/vaultcluster/testdata/policies/unshared/apps-auth.hcl b/internal/vaultcluster/testdata/policies/unshared/apps-auth.hcl new file mode 100644 index 0000000..64b840d --- /dev/null +++ b/internal/vaultcluster/testdata/policies/unshared/apps-auth.hcl @@ -0,0 +1,44 @@ +# Writes app auth roles through the cluster function, on the aws and gcp mounts. Cannot read tenant secrets. +path "sys/auth" { + capabilities = ["read"] +} + +path "sys/auth/aws" { + capabilities = ["create", "read", "update", "sudo"] +} + +path "sys/auth/gcp" { + capabilities = ["create", "read", "update", "sudo"] +} + +path "auth/aws/role" { + capabilities = ["list"] +} + +path "auth/aws/role/*" { + capabilities = ["create", "read", "update", "delete", "list"] +} + +path "auth/gcp/role" { + capabilities = ["list"] +} + +path "auth/gcp/role/*" { + capabilities = ["create", "read", "update", "delete", "list"] +} + +path "auth/token/lookup-self" { + capabilities = ["read"] +} + +path "auth/token/renew-self" { + capabilities = ["update"] +} + +path "kv/*" { + capabilities = ["deny"] +} + +path "database/creds/*" { + capabilities = ["deny"] +} diff --git a/internal/vaultcluster/testdata/policies/unshared/apps-reader.hcl b/internal/vaultcluster/testdata/policies/unshared/apps-reader.hcl new file mode 100644 index 0000000..3e4f674 --- /dev/null +++ b/internal/vaultcluster/testdata/policies/unshared/apps-reader.hcl @@ -0,0 +1,41 @@ +# App broker, read level. Lets an app log in as any onboarded tenant's reader: read the role ID and mint a +# one-time secret ID on approle-reader. The token that login yields sees one tenant; this token sees none. +path "auth/approle-reader/role/+/role-id" { + capabilities = ["read"] +} + +path "auth/approle-reader/role/+/secret-id" { + capabilities = ["update"] +} + +path "auth/token/lookup-self" { + capabilities = ["read"] +} + +path "auth/token/renew-self" { + capabilities = ["update"] +} + +path "auth/approle-writer/*" { + capabilities = ["deny"] +} + +path "kv/*" { + capabilities = ["deny"] +} + +path "database/*" { + capabilities = ["deny"] +} + +path "sys/*" { + capabilities = ["deny"] +} + +path "auth/token/create*" { + capabilities = ["deny"] +} + +path "identity/*" { + capabilities = ["deny"] +} diff --git a/internal/vaultcluster/testdata/policies/unshared/apps-writer.hcl b/internal/vaultcluster/testdata/policies/unshared/apps-writer.hcl new file mode 100644 index 0000000..2ebb71a --- /dev/null +++ b/internal/vaultcluster/testdata/policies/unshared/apps-writer.hcl @@ -0,0 +1,41 @@ +# App broker, write level. Lets an app log in as any onboarded tenant's writer: read the role ID and mint a +# one-time secret ID on approle-writer. The token that login yields sees one tenant; this token sees none. +path "auth/approle-writer/role/+/role-id" { + capabilities = ["read"] +} + +path "auth/approle-writer/role/+/secret-id" { + capabilities = ["update"] +} + +path "auth/token/lookup-self" { + capabilities = ["read"] +} + +path "auth/token/renew-self" { + capabilities = ["update"] +} + +path "auth/approle-reader/*" { + capabilities = ["deny"] +} + +path "kv/*" { + capabilities = ["deny"] +} + +path "database/*" { + capabilities = ["deny"] +} + +path "sys/*" { + capabilities = ["deny"] +} + +path "auth/token/create*" { + capabilities = ["deny"] +} + +path "identity/*" { + capabilities = ["deny"] +} diff --git a/internal/vaultcluster/testdata/policies/unshared/operator.hcl b/internal/vaultcluster/testdata/policies/unshared/operator.hcl new file mode 100644 index 0000000..0625271 --- /dev/null +++ b/internal/vaultcluster/testdata/policies/unshared/operator.hcl @@ -0,0 +1,140 @@ +# Local operator: health and config read. Can start generate-root (recovery keys still required). Not a tenant secret reader. No restore. +path "sys/health" { + capabilities = ["read", "sudo"] +} + +path "sys/seal-status" { + capabilities = ["read"] +} + +path "sys/leader" { + capabilities = ["read"] +} + +path "sys/mounts" { + capabilities = ["read", "sudo"] +} + +path "sys/mounts/database" { + capabilities = ["create", "update", "sudo"] +} + +path "sys/mounts/*" { + capabilities = ["read"] +} + +path "sys/auth" { + capabilities = ["read"] +} + +path "sys/policies/acl" { + capabilities = ["list"] +} + +path "sys/policies/acl/*" { + capabilities = ["read"] +} + +path "sys/audit" { + capabilities = ["read", "sudo"] +} + +path "auth/approle-reader/role" { + capabilities = ["list"] +} + +path "auth/approle-reader/role/*" { + capabilities = ["read"] +} + +path "auth/approle-writer/role" { + capabilities = ["list"] +} + +path "auth/approle-writer/role/*" { + capabilities = ["read"] +} + +path "database/roles" { + capabilities = ["list"] +} + +path "database/roles/*" { + capabilities = ["read"] +} + +path "database/config/*" { + capabilities = ["create", "update", "read"] +} + +path "sys/leases/lookup" { + capabilities = ["update"] +} + +path "sys/leases/lookup/*" { + capabilities = ["list", "read"] +} + +path "sys/leases/revoke" { + capabilities = ["update"] +} + +path "sys/leases/revoke-prefix/*" { + capabilities = ["update", "sudo"] +} + +path "sys/generate-root" { + capabilities = ["create", "update", "read", "delete", "sudo"] +} + +path "sys/generate-root/*" { + capabilities = ["create", "update", "read", "delete", "sudo"] +} + +path "sys/storage/raft/snapshot" { + capabilities = ["read"] +} + +path "sys/storage/raft/configuration" { + capabilities = ["read"] +} + +path "sys/storage/raft/autopilot/state" { + capabilities = ["read"] +} + +path "auth/token/lookup-self" { + capabilities = ["read"] +} + +path "auth/token/renew-self" { + capabilities = ["update"] +} + +path "auth/token/revoke-self" { + capabilities = ["update"] +} + +path "kv/data/*" { + capabilities = ["deny"] +} + +path "kv/metadata/*" { + capabilities = ["deny"] +} + +path "database/creds/*" { + capabilities = ["deny"] +} + +path "sys/policies/acl/operator" { + capabilities = ["deny"] +} + +path "sys/audit/*" { + capabilities = ["deny"] +} + +path "sys/storage/raft/snapshot-force" { + capabilities = ["deny"] +} diff --git a/internal/vaultcluster/testdata/policies/unshared/provisioning.hcl b/internal/vaultcluster/testdata/policies/unshared/provisioning.hcl new file mode 100644 index 0000000..2377550 --- /dev/null +++ b/internal/vaultcluster/testdata/policies/unshared/provisioning.hcl @@ -0,0 +1,122 @@ +# Onboard/offboard tenants. Writes AppRole and database roles only; tenant policies are static (bootstrap). +# allowed_parameters pins every role to the tenant policies and fixed SQL, so nothing written here +# can carry a platform policy. Can mint any tenant's credentials; cannot read tenant secrets itself. +path "auth/approle-reader/role" { + capabilities = ["list"] +} + +path "auth/approle-reader/role/*" { + capabilities = ["create", "read", "update", "delete", "list"] + allowed_parameters = { + "token_policies" = ["tenant-reader"] + "token_ttl" = [] + "token_max_ttl" = [] + "token_type" = ["service"] + "secret_id_ttl" = [] + "secret_id_num_uses" = [] + "bind_secret_id" = [] + } +} + +path "auth/approle-reader/role/+/role-id" { + capabilities = ["read"] +} + +path "auth/approle-reader/role/+/secret-id" { + capabilities = ["update"] +} + +path "auth/approle-writer/role" { + capabilities = ["list"] +} + +path "auth/approle-writer/role/*" { + capabilities = ["create", "read", "update", "delete", "list"] + allowed_parameters = { + "token_policies" = ["tenant-writer", "tenant-database"] + "token_ttl" = [] + "token_max_ttl" = [] + "token_type" = ["service"] + "secret_id_ttl" = [] + "secret_id_num_uses" = [] + "bind_secret_id" = [] + } +} + +path "auth/approle-writer/role/+/role-id" { + capabilities = ["read"] +} + +path "auth/approle-writer/role/+/secret-id" { + capabilities = ["update"] +} + +path "database/roles/tenant-*" { + capabilities = ["create", "read", "update", "delete", "list"] + allowed_parameters = { + "db_name" = [] + "creation_statements" = ["CREATE ROLE \"{{name}}\" WITH LOGIN PASSWORD '{{password}}' VALID UNTIL '{{expiration}}';", "GRANT app_readonly TO \"{{name}}\";", "GRANT app_readwrite TO \"{{name}}\";"] + "default_ttl" = [] + "max_ttl" = [] + } +} + +path "sys/mounts" { + capabilities = ["read"] +} + +path "sys/auth" { + capabilities = ["read"] +} + +path "auth/token/lookup-self" { + capabilities = ["read"] +} + +path "auth/token/renew-self" { + capabilities = ["update"] +} + +path "kv/*" { + capabilities = ["deny"] +} + +path "sys/policies/*" { + capabilities = ["deny"] +} + +path "sys/policy/*" { + capabilities = ["deny"] +} + +path "sys/audit" { + capabilities = ["deny"] +} + +path "sys/audit/*" { + capabilities = ["deny"] +} + +path "sys/audit-hash/*" { + capabilities = ["deny"] +} + +path "database/creds/*" { + capabilities = ["deny"] +} + +path "sys/mounts/*" { + capabilities = ["deny"] +} + +path "sys/auth/*" { + capabilities = ["deny"] +} + +path "auth/token/create*" { + capabilities = ["deny"] +} + +path "identity/*" { + capabilities = ["deny"] +} diff --git a/internal/vaultcluster/testdata/policies/unshared/tenant-database.hcl b/internal/vaultcluster/testdata/policies/unshared/tenant-database.hcl new file mode 100644 index 0000000..681fe52 --- /dev/null +++ b/internal/vaultcluster/testdata/policies/unshared/tenant-database.hcl @@ -0,0 +1,36 @@ +# Dynamic DB creds for the tenant named by the caller's approle-writer role only. Additive; does not widen KV access. +path "database/creds/tenant-{{identity.entity.aliases.auth_approle_5e6f7a8b.metadata.role_name}}-*" { + capabilities = ["read"] +} + +path "database/roles/tenant-{{identity.entity.aliases.auth_approle_5e6f7a8b.metadata.role_name}}-*" { + capabilities = ["read"] +} + +path "sys/leases/renew" { + capabilities = ["update"] +} + +path "sys/leases/revoke" { + capabilities = ["update"] +} + +path "database/creds/*" { + capabilities = ["deny"] +} + +path "database/roles/*" { + capabilities = ["deny"] +} + +path "database/config/*" { + capabilities = ["deny"] +} + +path "database/static-creds/*" { + capabilities = ["deny"] +} + +path "database/rotate-root/*" { + capabilities = ["deny"] +} diff --git a/internal/vaultcluster/testdata/policies/unshared/tenant-reader.hcl b/internal/vaultcluster/testdata/policies/unshared/tenant-reader.hcl new file mode 100644 index 0000000..1e3de7a --- /dev/null +++ b/internal/vaultcluster/testdata/policies/unshared/tenant-reader.hcl @@ -0,0 +1,61 @@ +# Read-only on the tenant named by the caller's AppRole role (approle-reader mount; role name = tenant ID). Static: written once at bootstrap. +# KV v2 needs data/ and metadata/; a token without a approle-reader alias renders no tenant path. +path "kv/data/customers/{{identity.entity.aliases.auth_approle_1a2b3c4d.metadata.role_name}}/*" { + capabilities = ["read"] +} + +path "kv/metadata/customers/{{identity.entity.aliases.auth_approle_1a2b3c4d.metadata.role_name}}/*" { + capabilities = ["read", "list"] +} + +path "kv/data/customers/*" { + capabilities = ["deny"] +} + +path "kv/metadata/customers/*" { + capabilities = ["deny"] +} + +path "kv/delete/customers/*" { + capabilities = ["deny"] +} + +path "kv/undelete/customers/*" { + capabilities = ["deny"] +} + +path "kv/destroy/customers/*" { + capabilities = ["deny"] +} + +path "auth/token/lookup-self" { + capabilities = ["read"] +} + +path "auth/token/renew-self" { + capabilities = ["update"] +} + +path "sys/*" { + capabilities = ["deny"] +} + +path "auth/token/create*" { + capabilities = ["deny"] +} + +path "auth/approle-reader/role/*" { + capabilities = ["deny"] +} + +path "auth/approle-writer/role/*" { + capabilities = ["deny"] +} + +path "identity/*" { + capabilities = ["deny"] +} + +path "kv/config" { + capabilities = ["deny"] +} diff --git a/internal/vaultcluster/testdata/policies/unshared/tenant-writer.hcl b/internal/vaultcluster/testdata/policies/unshared/tenant-writer.hcl new file mode 100644 index 0000000..1b3d305 --- /dev/null +++ b/internal/vaultcluster/testdata/policies/unshared/tenant-writer.hcl @@ -0,0 +1,73 @@ +# Full KV lifecycle on the tenant named by the caller's AppRole role (approle-writer mount; role name = tenant ID). Static: written once at bootstrap. +# Name all five KV v2 path families. +path "kv/data/customers/{{identity.entity.aliases.auth_approle_5e6f7a8b.metadata.role_name}}/*" { + capabilities = ["create", "read", "update", "patch", "delete", "list"] +} + +path "kv/metadata/customers/{{identity.entity.aliases.auth_approle_5e6f7a8b.metadata.role_name}}/*" { + capabilities = ["create", "read", "update", "delete", "list"] +} + +path "kv/delete/customers/{{identity.entity.aliases.auth_approle_5e6f7a8b.metadata.role_name}}/*" { + capabilities = ["update"] +} + +path "kv/undelete/customers/{{identity.entity.aliases.auth_approle_5e6f7a8b.metadata.role_name}}/*" { + capabilities = ["update"] +} + +path "kv/destroy/customers/{{identity.entity.aliases.auth_approle_5e6f7a8b.metadata.role_name}}/*" { + capabilities = ["update"] +} + +path "kv/data/customers/*" { + capabilities = ["deny"] +} + +path "kv/metadata/customers/*" { + capabilities = ["deny"] +} + +path "kv/delete/customers/*" { + capabilities = ["deny"] +} + +path "kv/undelete/customers/*" { + capabilities = ["deny"] +} + +path "kv/destroy/customers/*" { + capabilities = ["deny"] +} + +path "auth/token/lookup-self" { + capabilities = ["read"] +} + +path "auth/token/renew-self" { + capabilities = ["update"] +} + +path "sys/*" { + capabilities = ["deny"] +} + +path "auth/token/create*" { + capabilities = ["deny"] +} + +path "auth/approle-reader/role/*" { + capabilities = ["deny"] +} + +path "auth/approle-writer/role/*" { + capabilities = ["deny"] +} + +path "identity/*" { + capabilities = ["deny"] +} + +path "kv/config" { + capabilities = ["deny"] +} From 449c04eb8174402c2472a2924c610d9616eb3923 Mon Sep 17 00:00:00 2001 From: Brad Sickles Date: Wed, 7 Oct 2026 08:52:18 -0400 Subject: [PATCH 2/4] feat(aws): shared cluster in previews-shared scopes secrets per env aws-ec2-vault-cluster 0.2.0: output shared (env_name == previews-shared), SHARED_ENVS in user-data, vault-admin ~> 0.3.0. aws-vault-access 0.3.0: passes the app env to the function and injects VAULT_ENV when the cluster is shared; empty otherwise. --- CHANGELOG.md | 8 +++ README.md | 60 ++++++++++++++-- aws/aws-ec2-vault-cluster/admin.tf | 2 +- aws/aws-ec2-vault-cluster/nullstone.tf | 7 ++ aws/aws-ec2-vault-cluster/outputs.tf | 5 ++ aws/aws-ec2-vault-cluster/outputs.tftest.hcl | 25 +++++++ .../templates/user-data.sh.tpl | 1 + aws/aws-ec2-vault-cluster/user-data.tf | 1 + aws/aws-vault-access/auth.tf | 4 +- aws/aws-vault-access/outputs.tf | 4 ++ aws/aws-vault-access/vault.tf | 6 ++ aws/aws-vault-access/vault.tftest.hcl | 68 ++++++++++++++++++- 12 files changed, 181 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cabead7..04d18d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +# 0.2.0 + +Modules: `aws-ec2-vault-cluster` 0.2.0, `aws-vault-access` 0.3.0. Needs `vault-admin` 0.3.0 and a fresh node AMI. Nothing changes for a cluster outside `previews-shared`. + +* A cluster launched in the `previews-shared` env is shared: output `shared = true`, and every tenant belongs to an env. Tenant roles are `.`, secrets live under `kv/data/envs//customers//`, and an app's broker can only log in as its own env's tenants. The cluster function binds each app role to its env through an identity entity; provisioning does the same for tenant roles. +* `aws-vault-access` passes the app's env to the function on a shared cluster and injects `VAULT_ENV` (empty on an unshared cluster). +* `vault-utils tenants create|destroy` take `--env` (required on a shared cluster, refused elsewhere). `tenants list` shows the env. New `envs list` and `envs destroy --yes [--purge-secrets]`. `vault-utils env` prints `SHARED_ENVS=true` for a shared cluster. + # 0.1.2 * Node AMI serves the Vault web UI at `$VAULT_ADDR/ui`. Needs a fresh AMI. diff --git a/README.md b/README.md index 127cf41..6a65783 100644 --- a/README.md +++ b/README.md @@ -161,6 +161,9 @@ Run from `local/` unless noted. Destructive commands require `--yes`. | `docker compose run --rm -e VAULT_TOKEN=... bootstrap tenants list` | no | List tenants; flags one missing a reader or writer role | | `docker compose run --rm -e VAULT_TOKEN=... bootstrap tenants destroy --yes` | yes (access) | Revoke access; secrets kept | | `docker compose run --rm -e VAULT_TOKEN=... bootstrap tenants destroy --yes --purge-secrets` | yes | Also destroy secret versions | +| `vault-utils tenants create --env ` | no | Onboard a tenant for one env (shared cluster; see [Shared cluster](#shared-cluster)) | +| `vault-utils envs list` | no | Envs with tenants (shared cluster) | +| `vault-utils envs destroy --yes [--purge-secrets]` | yes | Offboard every tenant of an env; purge also destroys its secrets | | `docker compose run --rm bootstrap snapshot take` | no | Raft snapshot plus SHA-256 | | `docker compose run --rm bootstrap snapshot restore --yes` | yes | Replaces all Vault state | | `go test -short ./...` | no | Unit tests (tenant ID, policy lint, render, compose lint) — repo root | @@ -187,7 +190,34 @@ kv/metadata/customers/{tenant_id}/* | Tenant policies | `tenant-reader`, `tenant-writer`, `tenant-database` | Static, written at bootstrap. The tenant comes from the login role name (`{{identity.entity.aliases..metadata.role_name}}`), so nothing is written per tenant. | | Broker policies | `apps-reader`, `apps-writer` | Read `role-id` and mint `secret-id` for any onboarded tenant on one mount. No KV, database, or sys access. | -Tenant IDs: `^[a-z0-9]([a-z0-9-]{1,30}[a-z0-9])$` (3-32 characters). Rejected: `/`, `..`, `*`, `sys`, `data`, `root`, and similar reserved names. +Tenant IDs: `^[a-z0-9]([a-z0-9-]{1,30}[a-z0-9])$` (3-32 characters). Rejected: `/`, `..`, `*`, `sys`, `data`, `root`, `envs`, and similar reserved names. + +### Shared cluster + +A cluster launched in the Nullstone env `previews-shared` serves every preview env. Its output `shared` is true, its nodes run with `SHARED_ENVS=true`, and every tenant belongs to one env: + +``` +kv/data/envs/{env}/customers/{tenant_id}/* +kv/metadata/envs/{env}/customers/{tenant_id}/* +``` + +| Piece | Shared cluster | +|---|---| +| Tenant role name | `{env}.{tenant_id}` on the same two mounts. Env names use the tenant ID character class, so the `.` is unambiguous. | +| Tenant policies | Env and tenant come from entity metadata (`{{identity.entity.metadata.env}}`, `...tenant`). `tenants create --env` writes the roles, one entity per role with that metadata, and an alias on the role's `role_id`. Nothing per env. | +| Broker policies | `auth/approle-/role/{{identity.entity.metadata.env}}.*` with every parameter denied: an app mints logins only for its own env and cannot change a role. The env comes from the entity the cluster function writes on the app's auth role (see [App access](#app-access)). | +| Provisioning, apps-auth | May create entities and aliases; `policies` and `disabled` are refused, groups and OIDC are denied. | + +A cluster anywhere else is unshared and renders the 0.1.x policies unchanged. Converting a cluster between the two is not supported: launch a new one. + +```bash +vault-utils tenants create acme-corp --env pr-123 +vault-utils tenants list +vault-utils envs list +vault-utils envs destroy pr-123 --yes --purge-secrets +``` + +`--env` is required on a shared cluster and refused on an unshared one; `vault-utils env` exports `SHARED_ENVS` so the CLI knows which it is talking to. `envs destroy` offboards every tenant of the env; `--purge-secrets` also destroys `kv/metadata/envs/` and needs break-glass. ```bash export VAULT_ADDR=http://127.0.0.1:8200 @@ -429,20 +459,40 @@ On boot, `vault-configure.service` runs after cloud-init, writes `/etc/vault.d/c Connect `vault` to the cluster. The app module must expose `security_group_id` and its IAM role name. -`role_name` is optional. If empty, the Vault role is `-`. `access` is `reader` (default) or `writer`; the capability binds the matching broker policy (`apps-reader` or `apps-writer`) and nothing else. The app receives `VAULT_ADDR`, `VAULT_ROLE`, `VAULT_TLS_SERVER_NAME` (empty unless the NLB terminates TLS), and `VAULT_TENANT_MOUNT`. During apply the capability calls the cluster function with `method = "aws"` and the app IAM role ARN; the function binds only that principal to that role. The same function serves GCP service accounts (`method = "gcp"`) for a future GCP cluster module. +`role_name` is optional. If empty, the Vault role is `-`. `access` is `reader` (default) or `writer`; the capability binds the matching broker policy (`apps-reader` or `apps-writer`) and nothing else. The app receives `VAULT_ADDR`, `VAULT_ROLE`, `VAULT_TLS_SERVER_NAME` (empty unless the NLB terminates TLS), `VAULT_TENANT_MOUNT`, and `VAULT_ENV`. During apply the capability calls the cluster function with `method = "aws"` and the app IAM role ARN; the function binds only that principal to that role. The same function serves GCP service accounts (`method = "gcp"`) for a future GCP cluster module. + +On a shared cluster (output `shared`), the capability also sends the app's Nullstone env name. The function then gives the auth role an identity entity with that env, which the broker policies template, and `VAULT_ENV` carries it to the app. On an unshared cluster `VAULT_ENV` is empty and the function never touches identity. Leave `role_name` empty on a shared cluster: a fixed name collides across preview envs and the function refuses the second binding. -The app does not receive a Vault token. At startup it logs in with its IAM role, then for every request it logs in again as one tenant: +The app does not receive a Vault token. At startup it logs in with its IAM role, then for every request it logs in again as one tenant. The role is `$TENANT`, or `$VAULT_ENV.$TENANT` when `VAULT_ENV` is set; the KV path is `customers/$TENANT/...`, or `envs/$VAULT_ENV/customers/$TENANT/...`: ```bash vault login -method=aws role="$VAULT_ROLE" -ROLE_ID=$(vault read -field=role_id auth/$VAULT_TENANT_MOUNT/role/$TENANT/role-id) -SECRET_ID=$(vault write -f -field=secret_id auth/$VAULT_TENANT_MOUNT/role/$TENANT/secret-id) +ROLE="${VAULT_ENV:+$VAULT_ENV.}$TENANT" +ROLE_ID=$(vault read -field=role_id auth/$VAULT_TENANT_MOUNT/role/$ROLE/role-id) +SECRET_ID=$(vault write -f -field=secret_id auth/$VAULT_TENANT_MOUNT/role/$ROLE/secret-id) TENANT_TOKEN=$(vault write -field=token auth/$VAULT_TENANT_MOUNT/login role_id=$ROLE_ID secret_id=$SECRET_ID) ``` The broker token reads no secrets. The tenant token is good for that tenant only, for 15 minutes. A login as any other role is denied. The operator token is not injected. +### Shared cluster in Nullstone + +Launch the cluster block in the `previews-shared` env. In `.nullstone/previews.yml`, point each app's capability at it; everywhere else the app uses its own env's cluster: + +```yaml +version: "0.1" + +apps: + api: + capabilities: + vault: + connections: + vault: previews-shared.vault +``` + +Create the preview env's tenants with `vault-utils tenants create --env ` and remove them with `vault-utils envs destroy --yes --purge-secrets` when the env goes away. + ## Security - Host ports bind to `127.0.0.1` only diff --git a/aws/aws-ec2-vault-cluster/admin.tf b/aws/aws-ec2-vault-cluster/admin.tf index 8652551..8c650e4 100644 --- a/aws/aws-ec2-vault-cluster/admin.tf +++ b/aws/aws-ec2-vault-cluster/admin.tf @@ -1,6 +1,6 @@ module "vault_admin" { source = "api.nullstone.io/nullstone/aws-vault-admin/aws" - version = "~> 0.2.2" + version = "~> 0.3.0" name = "${local.resource_name}-apps-auth" tags = local.tags diff --git a/aws/aws-ec2-vault-cluster/nullstone.tf b/aws/aws-ec2-vault-cluster/nullstone.tf index a66bc88..43f8c77 100644 --- a/aws/aws-ec2-vault-cluster/nullstone.tf +++ b/aws/aws-ec2-vault-cluster/nullstone.tf @@ -28,3 +28,10 @@ locals { block_name = data.ns_workspace.this.block_name resource_name = "${data.ns_workspace.this.block_ref}-${random_string.resource_suffix.result}" } + +# A cluster launched in the previews-shared env serves every preview env and keeps their secrets apart +# (envs//customers/). Anywhere else it is one env's cluster with the unscoped layout. +locals { + shared_env_name = "previews-shared" + shared = data.ns_workspace.this.env_name == local.shared_env_name +} diff --git a/aws/aws-ec2-vault-cluster/outputs.tf b/aws/aws-ec2-vault-cluster/outputs.tf index 7659b50..60458c0 100644 --- a/aws/aws-ec2-vault-cluster/outputs.tf +++ b/aws/aws-ec2-vault-cluster/outputs.tf @@ -102,3 +102,8 @@ output "public_urls" { value = local.public_urls description = "list(string) ||| A list of URLs accessible to the public" } + +output "shared" { + value = local.shared + description = "bool ||| True when this cluster runs in the previews-shared env and scopes every tenant's secrets under envs//." +} diff --git a/aws/aws-ec2-vault-cluster/outputs.tftest.hcl b/aws/aws-ec2-vault-cluster/outputs.tftest.hcl index ccc7b1c..e0a767c 100644 --- a/aws/aws-ec2-vault-cluster/outputs.tftest.hcl +++ b/aws/aws-ec2-vault-cluster/outputs.tftest.hcl @@ -121,3 +121,28 @@ run "uses_https_with_a_subdomain" { error_message = "With TLS, vault.internal needs the user-facing name for certificate verification." } } + +run "is_not_shared_outside_previews_shared" { + command = plan + + assert { + condition = output.shared == false && strcontains(base64decode(aws_launch_template.this.user_data), "SHARED_ENVS=false") + error_message = "A cluster in any env but previews-shared is unshared and tells its nodes so." + } +} + +run "is_shared_in_previews_shared" { + command = plan + + override_data { + target = data.ns_workspace.this + values = { + env_name = "previews-shared" + } + } + + assert { + condition = output.shared == true && strcontains(base64decode(aws_launch_template.this.user_data), "SHARED_ENVS=true") + error_message = "A cluster in previews-shared is shared and its nodes scope tenants per env." + } +} diff --git a/aws/aws-ec2-vault-cluster/templates/user-data.sh.tpl b/aws/aws-ec2-vault-cluster/templates/user-data.sh.tpl index da9b916..56bb850 100644 --- a/aws/aws-ec2-vault-cluster/templates/user-data.sh.tpl +++ b/aws/aws-ec2-vault-cluster/templates/user-data.sh.tpl @@ -13,6 +13,7 @@ VAULT_APPS_AUTH_SECRET_ARN=${apps_auth_secret_arn} SNAPSHOT_BUCKET=${snapshot_bucket_name} SNAPSHOT_PREFIX=${snapshot_prefix} BACKUP_SCHEDULE="${backup_schedule}" +SHARED_ENVS=${shared} AUDIT_LOG_PATH=/opt/vault/audit/audit.log AWS_REGION=${aws_region} UNSEAL_KMS_KEY_ARN=${unseal_kms_key_arn} diff --git a/aws/aws-ec2-vault-cluster/user-data.tf b/aws/aws-ec2-vault-cluster/user-data.tf index 109a4c3..3c12351 100644 --- a/aws/aws-ec2-vault-cluster/user-data.tf +++ b/aws/aws-ec2-vault-cluster/user-data.tf @@ -11,6 +11,7 @@ locals { snapshot_bucket_name = local.snapshot_bucket_name snapshot_prefix = local.snapshot_prefix backup_schedule = var.backup_schedule + shared = local.shared vault_api_addr = "https://${trimsuffix(aws_route53_record.vault.fqdn, ".")}:${local.vault_api_port}" }) } diff --git a/aws/aws-vault-access/auth.tf b/aws/aws-vault-access/auth.tf index ae78e85..84556d8 100644 --- a/aws/aws-vault-access/auth.tf +++ b/aws/aws-vault-access/auth.tf @@ -2,7 +2,8 @@ data "aws_iam_role" "app" { name = var.app_metadata["role_name"] } -# The cluster function writes one Vault auth role for this app's IAM role on the aws auth mount. +# The cluster function writes one Vault auth role for this app's IAM role on the aws auth mount. On a shared +# cluster it also binds the role to this env, so the app can only log in as this env's tenants. resource "aws_lambda_invocation" "vault_role" { function_name = local.admin_function_name lifecycle_scope = "CRUD" @@ -14,6 +15,7 @@ resource "aws_lambda_invocation" "vault_role" { method = "aws" principal = data.aws_iam_role.app.arn policies = ["apps-${var.access}"] + env = local.env } }) } diff --git a/aws/aws-vault-access/outputs.tf b/aws/aws-vault-access/outputs.tf index 221b100..8e91fe1 100644 --- a/aws/aws-vault-access/outputs.tf +++ b/aws/aws-vault-access/outputs.tf @@ -15,6 +15,10 @@ output "env" { { name = "VAULT_TENANT_MOUNT" value = local.tenant_mount + }, + { + name = "VAULT_ENV" + value = local.env } ] } diff --git a/aws/aws-vault-access/vault.tf b/aws/aws-vault-access/vault.tf index a5407dc..996553d 100644 --- a/aws/aws-vault-access/vault.tf +++ b/aws/aws-vault-access/vault.tf @@ -11,3 +11,9 @@ locals { vault_addr = data.ns_connection.vault.outputs.vault_addr tls_server_name = try(data.ns_connection.vault.outputs.tls_server_name, "") } + +# shared is true for a cluster in previews-shared (0.2.0+). Older clusters have no such output: not shared. +locals { + shared = try(tobool(data.ns_connection.vault.outputs.shared), false) + env = local.shared ? data.ns_workspace.this.env_name : "" +} diff --git a/aws/aws-vault-access/vault.tftest.hcl b/aws/aws-vault-access/vault.tftest.hcl index 306dafc..23c879f 100644 --- a/aws/aws-vault-access/vault.tftest.hcl +++ b/aws/aws-vault-access/vault.tftest.hcl @@ -1,4 +1,14 @@ -mock_provider "ns" {} +mock_provider "ns" { + mock_data "ns_workspace" { + defaults = { + stack_name = "core" + env_name = "pr-12" + block_ref = "api" + block_name = "api" + aws_tags = {} + } + } +} mock_provider "aws" {} variables { @@ -33,8 +43,8 @@ run "injects_the_capability_role" { } assert { - condition = output.env == [{ name = "VAULT_ADDR", value = "http://vault.internal:8200" }, { name = "VAULT_ROLE", value = "billing" }, { name = "VAULT_TLS_SERVER_NAME", value = "" }, { name = "VAULT_TENANT_MOUNT", value = "approle-reader" }] - error_message = "The app must receive VAULT_ADDR, the capability role, an empty TLS name without TLS, and its tenant mount." + condition = output.env == [{ name = "VAULT_ADDR", value = "http://vault.internal:8200" }, { name = "VAULT_ROLE", value = "billing" }, { name = "VAULT_TLS_SERVER_NAME", value = "" }, { name = "VAULT_TENANT_MOUNT", value = "approle-reader" }, { name = "VAULT_ENV", value = "" }] + error_message = "The app must receive VAULT_ADDR, the capability role, an empty TLS name without TLS, its tenant mount, and an empty VAULT_ENV on a cluster without a shared output." } assert { @@ -47,12 +57,63 @@ run "injects_the_capability_role" { error_message = "A reader app gets only the apps-reader broker policy." } + assert { + condition = jsondecode(aws_lambda_invocation.vault_role.input).data.env == "" + error_message = "Without a shared output the function gets no env." + } + assert { condition = aws_security_group_rule.app_to_vault.from_port == 8200 && aws_security_group_rule.vault_from_app.source_security_group_id == "sg-app" error_message = "Security group rules must use the cluster API port and the app security group." } } +run "shared_cluster_binds_the_env" { + command = plan + + override_data { + target = data.ns_connection.vault + values = { + outputs = { + vault_fqdn = "vault.internal" + vault_addr = "http://vault.internal:8200" + nlb_security_group_id = "sg-nlb" + vault_api_port = "8200" + admin_function_name = "vault-apps-auth" + shared = "true" + } + } + } + + assert { + condition = jsondecode(aws_lambda_invocation.vault_role.input).data.env == "pr-12" && output.env[4] == { name = "VAULT_ENV", value = "pr-12" } + error_message = "On a shared cluster the function binds the app to its env and the app receives VAULT_ENV." + } +} + +run "unshared_cluster_passes_no_env" { + command = plan + + override_data { + target = data.ns_connection.vault + values = { + outputs = { + vault_fqdn = "vault.internal" + vault_addr = "http://vault.internal:8200" + nlb_security_group_id = "sg-nlb" + vault_api_port = "8200" + admin_function_name = "vault-apps-auth" + shared = "false" + } + } + } + + assert { + condition = jsondecode(aws_lambda_invocation.vault_role.input).data.env == "" && output.env[4] == { name = "VAULT_ENV", value = "" } + error_message = "An unshared cluster must behave as before: no env for the function, empty VAULT_ENV." + } +} + run "uses_the_cluster_port" { command = plan @@ -174,6 +235,7 @@ run "uses_the_cluster_addr_and_tls_name" { { name = "VAULT_ROLE", value = "billing" }, { name = "VAULT_TLS_SERVER_NAME", value = "vault.acme.example.com" }, { name = "VAULT_TENANT_MOUNT", value = "approle-reader" }, + { name = "VAULT_ENV", value = "" }, ] error_message = "A TLS cluster must give the app its https address and the certificate name." } From 39bccb08305dbed978a6a520f2312fa413cf7365 Mon Sep 17 00:00:00 2001 From: Brad Sickles Date: Wed, 7 Oct 2026 09:13:28 -0400 Subject: [PATCH 3/4] docs(aws): add READMEs for the cluster and access modules Cluster: where each tenant's secrets live, how tenant tokens are scoped, and what apps and humans can do in both modes. Access: the two-step login, token lifetimes, and gotchas. .gitignore now keeps module READMEs. --- .gitignore | 1 + aws/aws-ec2-vault-cluster/README.md | 81 +++++++++++++++++++++++++++++ aws/aws-vault-access/README.md | 77 +++++++++++++++++++++++++++ 3 files changed, 159 insertions(+) create mode 100644 aws/aws-ec2-vault-cluster/README.md create mode 100644 aws/aws-vault-access/README.md diff --git a/.gitignore b/.gitignore index bf2b9d1..03c667b 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ *.md !/README.md !/CHANGELOG.md +!/aws/*/README.md docs/ runbooks/ examples/ diff --git a/aws/aws-ec2-vault-cluster/README.md b/aws/aws-ec2-vault-cluster/README.md new file mode 100644 index 0000000..c7cc445 --- /dev/null +++ b/aws/aws-ec2-vault-cluster/README.md @@ -0,0 +1,81 @@ +# aws-ec2-vault-cluster + +Self-hosted Vault CE on an EC2 Auto Scaling Group: Raft storage, KMS auto-unseal, internal NLB on `vault.internal`, S3 snapshots, platform tokens in Secrets Manager. Apps connect through `nullstone/aws-vault-access`. Operations, backup, and break-glass are in the [repo README](../../README.md). + +## Connections + +| Name | Contract | Purpose | +|---|---|---| +| `network` | `network/aws/vpc` | VPC, private subnets, internal zone | +| `snapshots_bucket` | `datastore/aws/s3` | Raft snapshots and the init claim | +| `unseal_key` | `datastore/aws/kms` | Auto-unseal key (not the bucket key) | +| `subdomain` (optional) | `subdomain/aws/route53` | User-facing name; the NLB terminates TLS only with this | + +## Variables + +| Name | Default | Purpose | +|---|---|---| +| `cluster_size` | `1` | Odd number of nodes | +| `instance_type` | `t3.micro` | | +| `ami`, `ami_owner` | latest `nullstone-vault` AMI from `522657839841` | Set `ami_owner = "self"` when this account bakes it | +| `backup_schedule` | empty | Cron for S3 snapshots; empty disables | +| `protect_platform_secrets` | `true` | `prevent_destroy` on the token secrets; set false before destroying the workspace | + +## Shared cluster + +Launched in the Nullstone env `previews-shared`, the cluster is shared: output `shared` is true and the nodes run with `SHARED_ENVS=true`. Every tenant then belongs to an env. Anywhere else the cluster is unshared and nothing below changes from 0.1.x. A cluster cannot switch modes; launch a new one. + +## Where a tenant's secrets live + +| Cluster | KV path | AppRole role name | Database roles | +|---|---|---|---| +| Unshared | `kv/data/customers/{tenant}/*` | `{tenant}` | `tenant-{tenant}-readonly`, `-readwrite` | +| Shared | `kv/data/envs/{env}/customers/{tenant}/*` | `{env}.{tenant}` | `tenant-{env}.{tenant}-readonly`, `-readwrite` | + +KV v2: `metadata/`, `delete/`, `undelete/`, and `destroy/` mirror the `data/` tree. Tenant IDs and env names match `^[a-z0-9]([a-z0-9-]{1,30}[a-z0-9])$`; neither contains `.`, so the role name splits unambiguously. + +## How a tenant token is scoped + +The tenant policies (`tenant-reader`, `tenant-writer`, `tenant-database`) are static, written once at bootstrap, and never name a tenant. A Vault ACL template fills the tenant in from the identity of the token that logged in: + +- Unshared: the AppRole role name (`{{identity.entity.aliases..metadata.role_name}}`), which only provisioning can set. +- Shared: entity metadata `env` and `tenant` (`{{identity.entity.metadata.env}}`), written by provisioning when it creates the tenant's roles and aliased on each role's `role_id`. + +Apps cannot reach the identity API, so neither value can be forged. A token whose entity lacks the metadata matches no path. Cross-tenant, cross-env, parent, wildcard, and traversal requests return 403; a 200 there is a breach. + +## Apps + +An app never holds a token that spans tenants. + +1. It logs in with its IAM role (`vault login -method=aws role=$VAULT_ROLE`) and gets a broker token holding `apps-reader` or `apps-writer`. That token reads no secrets. +2. For each request it mints a one-time secret-id for one tenant's role on `$VAULT_TENANT_MOUNT` (`approle-reader` or `approle-writer`) and logs in again. The result is a 15 minute token for that tenant only. + +On a shared cluster the broker policy is templated on the app entity's `env`, which the cluster function writes when `aws-vault-access` binds the app: `auth/approle-/role/{{identity.entity.metadata.env}}.*` with every parameter denied. The app can mint logins only for its own env's tenants and cannot alter a role. `VAULT_ENV` tells the app its env; it is empty on an unshared cluster, where the role is `$TENANT` and the path `customers/$TENANT`. + +## Humans + +There is no per-person login. A human uses a platform token from Secrets Manager, or the token `vault login` saved, with `vault-utils` (`vault-utils env --org --stack --env --block` prints `VAULT_ADDR`, `VAULT_TLS_SERVER_NAME`, and `SHARED_ENVS`): + +| Token | Can | Cannot | +|---|---|---| +| Provisioning (`provisioning_secret_arn`) | `tenants create / list / destroy`, `envs list / destroy`; on a shared cluster also the tenant entities. Mint any tenant's login. | Read tenant KV with its own token, write policies, give an entity policies, touch groups or OIDC | +| Operator (`operator_secret_arn`) | Health, mounts, snapshots, start generate-root | Read tenant KV, restore | +| Break-glass root | Restore, purge, read tenant data in an incident | Kept: minted from operator plus recovery keys, two people, revoked after one action | + +The function's `apps-auth` token writes app auth roles and, on a shared cluster, app entities. It cannot read tenant secrets. + +Per-tenant CLI on a shared cluster takes `--env`; on an unshared one `--env` is refused. + +## Outputs + +`vault_addr`, `user_vault_addr`, `tls_server_name`, `vault_fqdn`, `user_fqdn`, `vault_api_port`, `nlb_security_group_id`, `admin_function_name`, `shared`, `operator_secret_arn`, `provisioning_secret_arn`, `db_hostname`, `db_port`, `db_endpoint`, `private_urls`, `public_urls`, plus node identifiers (`role_name`, `instance_profile_name`, `security_group_id`, `autoscaling_group_name`, `ami_id`). + +## Testing + +```bash +tofu fmt -check +tofu init -backend=false +tofu test +``` + +`tofu test` mocks every provider. A real plan needs a Nullstone workspace and AWS credentials. diff --git a/aws/aws-vault-access/README.md b/aws/aws-vault-access/README.md new file mode 100644 index 0000000..9d07a81 --- /dev/null +++ b/aws/aws-vault-access/README.md @@ -0,0 +1,77 @@ +# aws-vault-access + +Capability that connects an app to `nullstone/aws-ec2-vault-cluster`. It opens the security groups, binds the app's IAM role to one Vault auth role, and injects the settings the app needs. The app receives no token. + +## What it does at apply + +1. Adds egress on the app security group and ingress on the cluster NLB security group for the Vault API port. The app module must expose `security_group_id` and its IAM role name in `app_metadata`. +2. Calls the cluster function with the app IAM role ARN. The function writes `auth/aws/role/` bound to exactly that principal, granting `apps-reader` or `apps-writer` and nothing else. On a shared cluster it also records the app's Nullstone env on the role's identity entity. +3. Injects environment variables: + +| Variable | Value | +|---|---| +| `VAULT_ADDR` | `http(s)://vault.internal:8200` | +| `VAULT_TLS_SERVER_NAME` | Name to verify in the certificate when `VAULT_ADDR` is https; empty otherwise. The certificate names the user-facing host, not `vault.internal`. | +| `VAULT_ROLE` | The auth role to log in with | +| `VAULT_TENANT_MOUNT` | `approle-reader` or `approle-writer`, from `access` | +| `VAULT_ENV` | The app's env on a shared cluster; empty otherwise | + +## Variables + +| Name | Default | Purpose | +|---|---|---| +| `access` | `reader` | `reader`: read tenant KV. `writer`: full KV lifecycle plus dynamic database credentials. | +| `role_name` | empty | Vault auth role name. Empty gives `-`, unique per workspace. Leave it empty on a shared cluster: a fixed name collides across preview envs and the function refuses the second binding. | + +## Gaining access from the app + +Two logins. The first is once per process, the second is once per tenant request. + +**1. Broker login with the IAM role.** Vault verifies a signed `sts:GetCallerIdentity` from the app's AWS credentials (ECS task role, Lambda execution role, or EC2 instance profile). This yields a broker token that can mint tenant logins and read nothing. + +```bash +vault login -method=aws role="$VAULT_ROLE" +``` + +With an SDK, call `auth/aws/login` with `role` and the signed STS request; every Vault client library has an AWS IAM auth helper. + +**2. Tenant login per request.** Read the role ID, mint a one-time secret ID, log in. The role is `$TENANT`, or `$VAULT_ENV.$TENANT` on a shared cluster. The resulting token sees one tenant for 15 minutes. + +```bash +ROLE="${VAULT_ENV:+$VAULT_ENV.}$TENANT" +ROLE_ID=$(vault read -field=role_id "auth/$VAULT_TENANT_MOUNT/role/$ROLE/role-id") +SECRET_ID=$(vault write -f -field=secret_id "auth/$VAULT_TENANT_MOUNT/role/$ROLE/secret-id") +TENANT_TOKEN=$(vault write -field=token "auth/$VAULT_TENANT_MOUNT/login" role_id="$ROLE_ID" secret_id="$SECRET_ID") +``` + +**3. Use the tenant token.** Paths are `customers/$TENANT/...`, or `envs/$VAULT_ENV/customers/$TENANT/...` on a shared cluster. + +```bash +PREFIX="${VAULT_ENV:+envs/$VAULT_ENV/}customers/$TENANT" +VAULT_TOKEN=$TENANT_TOKEN vault kv get -mount=kv "$PREFIX/app-config" +# writer only: +VAULT_TOKEN=$TENANT_TOKEN vault kv put -mount=kv "$PREFIX/app-config" api_key=... +VAULT_TOKEN=$TENANT_TOKEN vault read "database/creds/tenant-$ROLE-readwrite" +``` + +## Token lifetimes + +| Token | TTL | Renewable | Do | +|---|---|---|---| +| Broker (from the IAM login) | Vault default, 32 days unless the mount is tuned | Yes | Cache it for the process. Renew with `auth/token/renew-self` before expiry, or log in again. Treat a 403 on `role-id` or `secret-id` as expired and log in again. | +| Secret ID | 60 seconds, one use | No | Mint right before login. Never cache or share it. | +| Tenant token | 15 minutes, hard maximum | Renew keeps it under 15 minutes total | Cache per tenant for up to 15 minutes if you like, then log in again. Never hold one past a request on behalf of a different tenant. | +| Database credentials (writer) | Lease from the cluster (`DatabaseTTL`) | Yes, up to the max | Renew with `sys/leases/renew` while in use; revoke with `sys/leases/revoke` when done. They outlive the tenant token that issued them until their lease ends. | + +## Gotchas + +- **The tenant must exist.** `tenants create ` (with `--env ` on a shared cluster) writes the roles. Until then `role-id` returns 404. There is no self-service onboarding from the app. +- **One tenant per token.** Reading another tenant's path, a parent path, a wildcard, or the other env returns 403. That is the contract, not a misconfiguration. Do not work around it by logging in as a broad tenant. +- **Shared cluster paths differ.** When `VAULT_ENV` is non-empty, both the role name and the KV prefix change. Branch on the variable rather than hard-coding either form. +- **Reader versus writer is the mount.** A `reader` app cannot log in on `approle-writer` even for its own tenant; `access = "writer"` changes both the broker policy and `VAULT_TENANT_MOUNT`. +- **TLS on `vault.internal`.** When `VAULT_ADDR` is https, set the TLS server name to `VAULT_TLS_SERVER_NAME` (the Vault CLI and SDKs read the variable). Without it the certificate fails to verify because it names the user-facing host. +- **IAM login region.** The STS request is signed for a region. Log in from the cluster's region, or pass `region=` to the login (CLI: `-region`, SDKs: the auth helper's region option). A signature error on login is almost always this. +- **Recreating the app IAM role.** The auth role binds the principal's unique ID. If the app's IAM role is destroyed and recreated with the same name, logins fail until the capability re-applies and the function rewrites the binding. +- **Task role, not execution role.** On ECS the signing credentials must be the task role that `app_metadata.role_name` names. The execution role is not bound. +- **No token in the environment.** Nothing injects `VAULT_TOKEN`. An app that expects one at startup must perform the IAM login itself. +- **Do not log secret IDs or tokens.** Vault HMACs them in its audit log; the app's logs should contain neither. From ddf48f6355089a7f8afb85cfde6e90c5e86ce63d Mon Sep 17 00:00:00 2001 From: Brad Sickles Date: Wed, 7 Oct 2026 09:20:36 -0400 Subject: [PATCH 4/4] feat(aws): gate shared mode on the env type, not the env name shared = data.ns_env.this.type == "PreviewsSharedEnv". --- CHANGELOG.md | 4 +-- README.md | 4 +-- aws/aws-ec2-vault-cluster/README.md | 2 +- aws/aws-ec2-vault-cluster/nullstone.tf | 12 ++++--- aws/aws-ec2-vault-cluster/outputs.tf | 2 +- aws/aws-ec2-vault-cluster/outputs.tftest.hcl | 35 ++++++++++++++++---- aws/aws-vault-access/vault.tf | 2 +- 7 files changed, 44 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 04d18d4..5b73829 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,8 @@ # 0.2.0 -Modules: `aws-ec2-vault-cluster` 0.2.0, `aws-vault-access` 0.3.0. Needs `vault-admin` 0.3.0 and a fresh node AMI. Nothing changes for a cluster outside `previews-shared`. +Modules: `aws-ec2-vault-cluster` 0.2.0, `aws-vault-access` 0.3.0. Needs `vault-admin` 0.3.0 and a fresh node AMI. Nothing changes for a cluster outside the shared previews env. -* A cluster launched in the `previews-shared` env is shared: output `shared = true`, and every tenant belongs to an env. Tenant roles are `.`, secrets live under `kv/data/envs//customers//`, and an app's broker can only log in as its own env's tenants. The cluster function binds each app role to its env through an identity entity; provisioning does the same for tenant roles. +* A cluster launched in the shared previews env (Nullstone env type `PreviewsSharedEnv`) is shared: output `shared = true`, and every tenant belongs to an env. Tenant roles are `.`, secrets live under `kv/data/envs//customers//`, and an app's broker can only log in as its own env's tenants. The cluster function binds each app role to its env through an identity entity; provisioning does the same for tenant roles. * `aws-vault-access` passes the app's env to the function on a shared cluster and injects `VAULT_ENV` (empty on an unshared cluster). * `vault-utils tenants create|destroy` take `--env` (required on a shared cluster, refused elsewhere). `tenants list` shows the env. New `envs list` and `envs destroy --yes [--purge-secrets]`. `vault-utils env` prints `SHARED_ENVS=true` for a shared cluster. diff --git a/README.md b/README.md index 6a65783..eb5dae3 100644 --- a/README.md +++ b/README.md @@ -194,7 +194,7 @@ Tenant IDs: `^[a-z0-9]([a-z0-9-]{1,30}[a-z0-9])$` (3-32 characters). Rejected: ` ### Shared cluster -A cluster launched in the Nullstone env `previews-shared` serves every preview env. Its output `shared` is true, its nodes run with `SHARED_ENVS=true`, and every tenant belongs to one env: +A cluster launched in the stack's shared previews env (Nullstone env type `PreviewsSharedEnv`) serves every preview env. Its output `shared` is true, its nodes run with `SHARED_ENVS=true`, and every tenant belongs to one env: ``` kv/data/envs/{env}/customers/{tenant_id}/* @@ -478,7 +478,7 @@ The broker token reads no secrets. The tenant token is good for that tenant only ### Shared cluster in Nullstone -Launch the cluster block in the `previews-shared` env. In `.nullstone/previews.yml`, point each app's capability at it; everywhere else the app uses its own env's cluster: +Launch the cluster block in the stack's shared previews env (named `previews-shared` below). In `.nullstone/previews.yml`, point each app's capability at it; everywhere else the app uses its own env's cluster: ```yaml version: "0.1" diff --git a/aws/aws-ec2-vault-cluster/README.md b/aws/aws-ec2-vault-cluster/README.md index c7cc445..31f8b56 100644 --- a/aws/aws-ec2-vault-cluster/README.md +++ b/aws/aws-ec2-vault-cluster/README.md @@ -23,7 +23,7 @@ Self-hosted Vault CE on an EC2 Auto Scaling Group: Raft storage, KMS auto-unseal ## Shared cluster -Launched in the Nullstone env `previews-shared`, the cluster is shared: output `shared` is true and the nodes run with `SHARED_ENVS=true`. Every tenant then belongs to an env. Anywhere else the cluster is unshared and nothing below changes from 0.1.x. A cluster cannot switch modes; launch a new one. +Launched in the stack's shared previews env (Nullstone env type `PreviewsSharedEnv`), the cluster is shared: output `shared` is true and the nodes run with `SHARED_ENVS=true`. Every tenant then belongs to an env. Anywhere else the cluster is unshared and nothing below changes from 0.1.x. A cluster cannot switch modes; launch a new one. ## Where a tenant's secrets live diff --git a/aws/aws-ec2-vault-cluster/nullstone.tf b/aws/aws-ec2-vault-cluster/nullstone.tf index 43f8c77..7ed55d3 100644 --- a/aws/aws-ec2-vault-cluster/nullstone.tf +++ b/aws/aws-ec2-vault-cluster/nullstone.tf @@ -29,9 +29,13 @@ locals { resource_name = "${data.ns_workspace.this.block_ref}-${random_string.resource_suffix.result}" } -# A cluster launched in the previews-shared env serves every preview env and keeps their secrets apart -# (envs//customers/). Anywhere else it is one env's cluster with the unscoped layout. +data "ns_env" "this" { + stack_id = data.ns_workspace.this.stack_id + env_id = data.ns_workspace.this.env_id +} + +# A cluster launched in the stack's shared previews env serves every preview env and keeps their secrets +# apart (envs//customers/). Anywhere else it is one env's cluster with the unscoped layout. locals { - shared_env_name = "previews-shared" - shared = data.ns_workspace.this.env_name == local.shared_env_name + shared = data.ns_env.this.type == "PreviewsSharedEnv" } diff --git a/aws/aws-ec2-vault-cluster/outputs.tf b/aws/aws-ec2-vault-cluster/outputs.tf index 60458c0..c1230ab 100644 --- a/aws/aws-ec2-vault-cluster/outputs.tf +++ b/aws/aws-ec2-vault-cluster/outputs.tf @@ -105,5 +105,5 @@ output "public_urls" { output "shared" { value = local.shared - description = "bool ||| True when this cluster runs in the previews-shared env and scopes every tenant's secrets under envs//." + description = "bool ||| True when this cluster runs in the shared previews env (Nullstone env type PreviewsSharedEnv) and scopes every tenant's secrets under envs//." } diff --git a/aws/aws-ec2-vault-cluster/outputs.tftest.hcl b/aws/aws-ec2-vault-cluster/outputs.tftest.hcl index e0a767c..213a7b0 100644 --- a/aws/aws-ec2-vault-cluster/outputs.tftest.hcl +++ b/aws/aws-ec2-vault-cluster/outputs.tftest.hcl @@ -11,6 +11,13 @@ mock_provider "ns" { } } + mock_data "ns_env" { + defaults = { + name = "prod" + type = "PipelineEnv" + } + } + mock_data "ns_connection" { defaults = { outputs = { @@ -122,27 +129,43 @@ run "uses_https_with_a_subdomain" { } } -run "is_not_shared_outside_previews_shared" { +run "is_not_shared_in_a_pipeline_env" { command = plan assert { condition = output.shared == false && strcontains(base64decode(aws_launch_template.this.user_data), "SHARED_ENVS=false") - error_message = "A cluster in any env but previews-shared is unshared and tells its nodes so." + error_message = "A cluster in any env but the shared previews env is unshared and tells its nodes so." + } +} + +run "is_not_shared_in_a_preview_env" { + command = plan + + override_data { + target = data.ns_env.this + values = { + type = "PreviewEnv" + } + } + + assert { + condition = output.shared == false + error_message = "A preview env's own cluster is unshared." } } -run "is_shared_in_previews_shared" { +run "is_shared_in_the_shared_previews_env" { command = plan override_data { - target = data.ns_workspace.this + target = data.ns_env.this values = { - env_name = "previews-shared" + type = "PreviewsSharedEnv" } } assert { condition = output.shared == true && strcontains(base64decode(aws_launch_template.this.user_data), "SHARED_ENVS=true") - error_message = "A cluster in previews-shared is shared and its nodes scope tenants per env." + error_message = "A cluster in the shared previews env is shared and its nodes scope tenants per env." } } diff --git a/aws/aws-vault-access/vault.tf b/aws/aws-vault-access/vault.tf index 996553d..ecd3617 100644 --- a/aws/aws-vault-access/vault.tf +++ b/aws/aws-vault-access/vault.tf @@ -12,7 +12,7 @@ locals { tls_server_name = try(data.ns_connection.vault.outputs.tls_server_name, "") } -# shared is true for a cluster in previews-shared (0.2.0+). Older clusters have no such output: not shared. +# shared is true for a cluster in the shared previews env (0.2.0+). Older clusters have no such output: not shared. locals { shared = try(tobool(data.ns_connection.vault.outputs.shared), false) env = local.shared ? data.ns_workspace.this.env_name : ""