diff --git a/CHANGELOG.md b/CHANGELOG.md index fc3043c..7d631bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,21 @@ +# 0.4.0 + +Module: `aws-ec2-vault-cluster` 0.4.0. Needs a fresh node AMI. The operator policy gains `sys/step-down` and `sys/storage/raft/remove-peer`; a cluster bootstrapped before this release must rewrite it with a break-glass root before the leave step works. A launch-template change rolls the nodes. + +* A rolling instance refresh is safe on a one-node cluster. A launch lifecycle hook (`vault-join`) holds a new node until `vault-utils lifecycle` reports it an unsealed, caught-up Raft voter; a node that never joins is abandoned after 30 minutes, which fails the refresh and rolls it back with the old node untouched. A terminate hook (`vault-leave`) holds a departing node until it has stepped down and removed itself from the peer set, so the survivor keeps quorum. Before this, the refresh counted a new node healthy on EC2 status alone and terminated the only copy of the data. + # 0.3.1 Module: `aws-ec2-vault-cluster` 0.3.1. Launch-template change; nodes roll. * Nodes advertise `api_addr` as the URL clients use: `user_vault_addr` when a subdomain is connected, else `vault_addr`. It was always `https://vault.internal:8200`, so a standby redirect sent browsers to an origin the NLB certificate does not name and the web UI failed with "The request failed and the interceptors did not return an alternative response". +# 0.3.0 + +Modules: `aws-ec2-vault-cluster` 0.3.0, `aws-vault-access` 0.3.0. No runtime change. + +* A `v*` tag push publishes both modules at that version (`.github/workflows/publish.yml`). The cluster module packages `templates/*`. + # 0.2.0 Modules: `aws-ec2-vault-cluster` 0.2.0, `aws-vault-access` 0.3.0. Needs `vault-admin` 0.3.0 and a fresh node AMI. Nothing changes for a cluster outside the shared previews env. diff --git a/README.md b/README.md index eb5dae3..33c708e 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ Implemented: - Auto-init (first start) and one-shot Shamir unseal via `vault-utils` - Isolation tests in Go (`go test`); credentials tests in Go (`TestCredentialsMatrix`) - `bootstrap aws` (KMS auto-unseal, Secrets Manager tokens), health on 8210, S3 snapshots and S3 restore -- AWS AMI, user-data, internal NLB (TLS 8200, health 8210), `vault.internal`, optional user-facing subdomain SNI, ASG (`cluster_size`), rolling instance refresh on launch-template change +- AWS AMI, user-data, internal NLB (TLS 8200, health 8210), `vault.internal`, optional user-facing subdomain SNI, ASG (`cluster_size`), rolling instance refresh on launch-template change gated by lifecycle hooks (join before the old node leaves; leave the peer set before termination) Not implemented: diff --git a/aws/aws-ec2-vault-cluster/README.md b/aws/aws-ec2-vault-cluster/README.md index 43eeaef..ba2238b 100644 --- a/aws/aws-ec2-vault-cluster/README.md +++ b/aws/aws-ec2-vault-cluster/README.md @@ -68,6 +68,12 @@ Per-tenant CLI on a shared cluster takes `--env`; on an unshared one `--env` is Nodes advertise `api_addr` as `user_vault_addr` when a subdomain is connected, else `vault_addr`. Redirects from a standby therefore land on the origin clients already use. Clients on `vault.internal` set `VAULT_TLS_SERVER_NAME` to `tls_server_name`. +## Upgrades + +A launch-template change (AMI, instance type, user-data) starts a rolling instance refresh. Two lifecycle hooks make it safe at any `cluster_size`, including 1. `vault-join` holds a new instance in `Pending:Wait` until `vault-utils lifecycle` sees it unsealed, a Raft voter, and caught up; a node that never joins is abandoned after 30 minutes and the refresh rolls back. `vault-leave` holds a departing instance in `Terminating:Wait` until it has stepped down and removed itself from the peer set; it terminates either way. Watch a node with `journalctl -u vault-lifecycle`. + +The ASG health check stays `EC2`. An `ELB` check would replace a one-node cluster whenever Vault was briefly unhealthy, and the data lives on that instance. + ## Outputs `vault_addr`, `user_vault_addr`, `tls_server_name`, `vault_fqdn`, `user_fqdn`, `vault_api_port`, `nlb_security_group_id`, `admin_function_name`, `shared`, `operator_secret_arn`, `provisioning_secret_arn`, `db_hostname`, `db_port`, `db_endpoint`, `private_urls`, `public_urls`, plus node identifiers (`role_name`, `instance_profile_name`, `security_group_id`, `autoscaling_group_name`, `ami_id`). diff --git a/aws/aws-ec2-vault-cluster/asg.tf b/aws/aws-ec2-vault-cluster/asg.tf index 7599619..005beb4 100644 --- a/aws/aws-ec2-vault-cluster/asg.tf +++ b/aws/aws-ec2-vault-cluster/asg.tf @@ -22,6 +22,8 @@ resource "aws_autoscaling_group" "this" { strategy = "Rolling" preferences { + # Health after the launch hook continues; the hook, not warmup, is what proves the node joined. + instance_warmup = 60 min_healthy_percentage = 100 max_healthy_percentage = local.asg_max_healthy_percentage auto_rollback = true @@ -41,3 +43,23 @@ resource "aws_autoscaling_group" "this" { propagate_at_launch = true } } + +# A new instance waits here until vault-utils lifecycle reports it an unsealed, caught-up Raft voter. One that +# never joins is abandoned, which fails the instance refresh and rolls it back with the old node untouched. +resource "aws_autoscaling_lifecycle_hook" "join" { + name = "vault-join" + autoscaling_group_name = aws_autoscaling_group.this.name + lifecycle_transition = "autoscaling:EC2_INSTANCE_LAUNCHING" + default_result = "ABANDON" + heartbeat_timeout = 1800 +} + +# A departing instance waits here until it has left the Raft peer set, so the remaining nodes keep quorum. +# It terminates either way; vault-utils logs the attempt. +resource "aws_autoscaling_lifecycle_hook" "leave" { + name = "vault-leave" + autoscaling_group_name = aws_autoscaling_group.this.name + lifecycle_transition = "autoscaling:EC2_INSTANCE_TERMINATING" + default_result = "CONTINUE" + heartbeat_timeout = 300 +} diff --git a/aws/aws-ec2-vault-cluster/iam.tf b/aws/aws-ec2-vault-cluster/iam.tf index 9d7df9b..c49f501 100644 --- a/aws/aws-ec2-vault-cluster/iam.tf +++ b/aws/aws-ec2-vault-cluster/iam.tf @@ -94,6 +94,26 @@ data "aws_iam_policy_document" "this" { resources = ["*"] } + statement { + sid = "LifecycleState" + effect = "Allow" + actions = ["autoscaling:DescribeAutoScalingInstances"] + resources = ["*"] + } + + # The group name is unknown until it exists, so the hook permission is scoped by the cluster tag instead. + statement { + sid = "LifecycleHooks" + effect = "Allow" + actions = ["autoscaling:CompleteLifecycleAction"] + resources = ["*"] + condition { + test = "StringEquals" + variable = "autoscaling:ResourceTag/${local.vault_cluster_tag_key}" + values = [local.vault_cluster_tag_value] + } + } + statement { sid = "VaultAwsAuth" effect = "Allow" diff --git a/aws/aws-ec2-vault-cluster/outputs.tftest.hcl b/aws/aws-ec2-vault-cluster/outputs.tftest.hcl index ac4f8c7..1b23356 100644 --- a/aws/aws-ec2-vault-cluster/outputs.tftest.hcl +++ b/aws/aws-ec2-vault-cluster/outputs.tftest.hcl @@ -146,6 +146,11 @@ run "is_not_shared_in_a_pipeline_env" { condition = output.shared == false && strcontains(base64decode(aws_launch_template.this.user_data), "SHARED_ENVS=false") error_message = "A cluster in any env but the shared previews env is unshared and tells its nodes so." } + + assert { + condition = aws_autoscaling_lifecycle_hook.join.default_result == "ABANDON" && aws_autoscaling_lifecycle_hook.leave.default_result == "CONTINUE" + error_message = "A node that never joins is abandoned so the refresh rolls back; a departing node terminates after trying to leave." + } } run "is_not_shared_in_a_preview_env" { diff --git a/cmd/vault-utils/main.go b/cmd/vault-utils/main.go index 35bf4c6..8adfb9b 100644 --- a/cmd/vault-utils/main.go +++ b/cmd/vault-utils/main.go @@ -1,6 +1,8 @@ package main import ( + "context" + "errors" "fmt" "log" "os" @@ -9,6 +11,8 @@ import ( "strings" "time" + "github.com/hashicorp/vault/api" + "github.com/nullstone-modules/vault-cluster/internal/aws/asg" "github.com/nullstone-modules/vault-cluster/internal/aws/s3" "github.com/nullstone-modules/vault-cluster/internal/aws/secretsmanager" "github.com/nullstone-modules/vault-cluster/internal/vaultcluster" @@ -45,6 +49,7 @@ Commands: snapshot schedule Cron loop (BACKUP_SCHEDULE; empty disables) health Print seal status health serve HTTP on :8210 (200 only if this node is a Raft voter and caught up) + lifecycle ASG node: continue the launch hook once a caught-up voter; leave Raft before termination env --org --stack --env --block Print shell settings for a Nullstone Vault cluster workspace Without VAULT_TOKEN, tenants, envs, snapshot take|restore, and health use the token from "vault login". @@ -87,6 +92,8 @@ func run(cmd string, args []string) error { return runHealthServe(c) } return c.Health() + case "lifecycle": + return runLifecycle(c) default: usage() return fmt.Errorf("unknown command %q", cmd) @@ -502,3 +509,49 @@ func getenv(k, def string) string { } return def } + +func runLifecycle(c *vaultcluster.Client) error { + nodeID := os.Getenv("VAULT_RAFT_NODE_ID") + if nodeID == "" { + return fmt.Errorf("VAULT_RAFT_NODE_ID is required for lifecycle") + } + hooks, err := asg.New(nodeID) + if err != nil { + return err + } + log.Printf("lifecycle watching instance %s", nodeID) + return vaultcluster.RunLifecycle(context.Background(), hooks, &lifecycleNode{c: c, nodeID: nodeID}, vaultcluster.LifecycleOptions{Logf: log.Printf}) +} + +// lifecycleNode loads the operator token lazily: on a fresh cluster it does not exist until bootstrap +// finishes, and on a rebuilt one Secrets Manager may still hold the previous cluster's token. +type lifecycleNode struct { + c *vaultcluster.Client + nodeID string + renewing bool +} + +func (n *lifecycleNode) withToken(op func() error) error { + if err := useOperatorToken(n.c); err != nil { + return err + } + if !n.renewing { + n.renewing = true + go n.c.RenewToken(tokenRenewInterval, nil) + } + err := op() + var re *api.ResponseError + if errors.As(err, &re) && re.StatusCode == 403 { + n.c.Cfg.Token = "" + n.c.API.ClearToken() + } + return err +} + +func (n *lifecycleNode) Ready() error { + return n.withToken(func() error { return n.c.NodeHealthOK(n.nodeID) }) +} + +func (n *lifecycleNode) Leave() error { + return n.withToken(func() error { return n.c.LeaveRaft(n.nodeID) }) +} diff --git a/go.mod b/go.mod index 26bc227..51fea44 100644 --- a/go.mod +++ b/go.mod @@ -3,11 +3,12 @@ module github.com/nullstone-modules/vault-cluster go 1.26.5 require ( - github.com/aws/aws-sdk-go-v2 v1.41.2 + github.com/aws/aws-sdk-go-v2 v1.47.1 github.com/aws/aws-sdk-go-v2/config v1.29.14 + github.com/aws/aws-sdk-go-v2/service/autoscaling v1.78.2 github.com/aws/aws-sdk-go-v2/service/s3 v1.79.3 github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.35.6 - github.com/aws/smithy-go v1.24.1 + github.com/aws/smithy-go v1.28.1 github.com/hashicorp/vault/api v1.16.0 github.com/robfig/cron/v3 v3.0.1 gopkg.in/nullstone-io/go-api-client.v0 v0.0.0-20260929173059-6bbecdaf8ea3 @@ -20,8 +21,8 @@ require ( github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.5 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.19.10 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.18 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.18 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.18 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.18 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.5 // indirect diff --git a/go.sum b/go.sum index 570935b..bc363a0 100644 --- a/go.sum +++ b/go.sum @@ -5,8 +5,8 @@ github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmms github.com/apparentlymart/go-textseg/v17 v17.0.1 h1:bpMXRgQ5cEoRNuQke1a80/Nl6w3G5eoIbWo9f3gXkAs= github.com/apparentlymart/go-textseg/v17 v17.0.1/go.mod h1:fa8X4jgGeevslICIY6LcdjkSecWnXmYd9Lk34z/VxZs= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/aws/aws-sdk-go-v2 v1.41.2 h1:LuT2rzqNQsauaGkPK/7813XxcZ3o3yePY0Iy891T2ls= -github.com/aws/aws-sdk-go-v2 v1.41.2/go.mod h1:IvvlAZQXvTXznUPfRVfryiG1fbzE2NGK6m9u39YQ+S4= +github.com/aws/aws-sdk-go-v2 v1.47.1 h1:uOIZnp4PK3ZhKI0dNrJrhTEsLxbpXHTAJlwoS1pvAtw= +github.com/aws/aws-sdk-go-v2 v1.47.1/go.mod h1:bttEH6JqnUL8LepvDVfdrds/fZ5bCIxzpe3abyUrhDU= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.5 h1:zWFmPmgw4sveAYi1mRqG+E/g0461cJ5M4bJ8/nc6d3Q= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.5/go.mod h1:nVUlMLVV8ycXSb7mSkcNu9e3v/1TJq2RTlrPwhYWr5c= github.com/aws/aws-sdk-go-v2/config v1.29.14 h1:f+eEi/2cKCg9pqKBoAIwRGzVb70MRKqWX4dg1BDcSJM= @@ -15,14 +15,16 @@ github.com/aws/aws-sdk-go-v2/credentials v1.19.10 h1:EEhmEUFCE1Yhl7vDhNOI5OCL/iK github.com/aws/aws-sdk-go-v2/credentials v1.19.10/go.mod h1:RnnlFCAlxQCkN2Q379B67USkBMu1PipEEiibzYN5UTE= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.18 h1:Ii4s+Sq3yDfaMLpjrJsqD6SmG/Wq/P5L/hw2qa78UAY= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.18/go.mod h1:6x81qnY++ovptLE6nWQeWrpXxbnlIex+4H4eYYGcqfc= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.18 h1:F43zk1vemYIqPAwhjTjYIz0irU2EY7sOb/F5eJ3HuyM= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.18/go.mod h1:w1jdlZXrGKaJcNoL+Nnrj+k5wlpGXqnNrKoP22HvAug= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.18 h1:xCeWVjj0ki0l3nruoyP2slHsGArMxeiiaoPN5QZH6YQ= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.18/go.mod h1:r/eLGuGCBw6l36ZRWiw6PaZwPXb6YOj+i/7MizNl5/k= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4 h1:CLq4+8UHCI+ZZYl/EuJxXovaIVN2xeeT8JV+dsApQ5E= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4/go.mod h1:Wv4q5sAM04xAMkoOedxLx2inVf6K5FdxYp+A61L+q/0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4 h1:dD4MR81I7YkpEBRk6UP9rocC2QnT3qVuXwzlYTtfGEs= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4/go.mod h1:EcXV1kAFd5XwSkDHlj94gnF3q5CkJyYiIJfH8N0VmrE= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo= github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.18 h1:eZioDaZGJ0tMM4gzmkNIO2aAoQd+je7Ug7TkvAzlmkU= github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.18/go.mod h1:CCXwUKAJdoWr6/NcxZ+zsiPr6oH/Q5aTooRGYieAyj4= +github.com/aws/aws-sdk-go-v2/service/autoscaling v1.78.2 h1:3zCxKK7jLLtQLxpauXY56hmAZGhyYSR1+XQ9QviouPg= +github.com/aws/aws-sdk-go-v2/service/autoscaling v1.78.2/go.mod h1:4roDw8gYFhAVo1b2ckuzEa0QPtpRXgU4o+dn44IvNF0= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.5 h1:CeY9LUdur+Dxoeldqoun6y4WtJ3RQtzk0JMP2gfUay0= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.5/go.mod h1:AZLZf2fMaahW5s/wMRciu1sYbdsikT/UHwbUjOdEVTc= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.10 h1:fJvQ5mIBVfKtiyx0AHY6HeWcRX5LGANLpq8SVR+Uazs= @@ -41,8 +43,8 @@ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.15 h1:edCcNp9eGIUDUCrzoCu1jWA github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.15/go.mod h1:lyRQKED9xWfgkYC/wmmYfv7iVIM68Z5OQ88ZdcV1QbU= github.com/aws/aws-sdk-go-v2/service/sts v1.41.7 h1:NITQpgo9A5NrDZ57uOWj+abvXSb83BbyggcUBVksN7c= github.com/aws/aws-sdk-go-v2/service/sts v1.41.7/go.mod h1:sks5UWBhEuWYDPdwlnRFn1w7xWdH29Jcpe+/PJQefEs= -github.com/aws/smithy-go v1.24.1 h1:VbyeNfmYkWoxMVpGUAbQumkODcYmfMRfZ8yQiH30SK0= -github.com/aws/smithy-go v1.24.1/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0= +github.com/aws/smithy-go v1.28.1 h1:R/nXH00c8qcfCzQVELtRw+eLQWtzv+VAIEFJ1/xxXlQ= +github.com/aws/smithy-go v1.28.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= diff --git a/internal/aws/asg/asg.go b/internal/aws/asg/asg.go new file mode 100644 index 0000000..3861073 --- /dev/null +++ b/internal/aws/asg/asg.go @@ -0,0 +1,66 @@ +// Package asg answers lifecycle hooks on the node's own Auto Scaling group. +package asg + +import ( + "context" + "fmt" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/autoscaling" + + "github.com/nullstone-modules/vault-cluster/internal/vaultcluster" +) + +type Client struct { + inner *autoscaling.Client + instanceID string + group string +} + +// New talks about one instance, which is this node: the Raft node ID is the instance ID. +func New(instanceID string) (*Client, error) { + if instanceID == "" { + return nil, fmt.Errorf("VAULT_RAFT_NODE_ID (the instance ID) is required") + } + cfg, err := config.LoadDefaultConfig(context.Background()) + if err != nil { + return nil, fmt.Errorf("AWS credentials: %w", err) + } + return &Client{inner: autoscaling.NewFromConfig(cfg), instanceID: instanceID}, nil +} + +// State is the instance's lifecycle state, e.g. Pending:Wait, InService, Terminating:Wait. +func (c *Client) State(ctx context.Context) (string, error) { + out, err := c.inner.DescribeAutoScalingInstances(ctx, &autoscaling.DescribeAutoScalingInstancesInput{ + InstanceIds: []string{c.instanceID}, + }) + if err != nil { + return "", fmt.Errorf("describe auto scaling instance %s: %w", c.instanceID, err) + } + if len(out.AutoScalingInstances) == 0 { + return "", vaultcluster.ErrNotInGroup + } + inst := out.AutoScalingInstances[0] + c.group = aws.ToString(inst.AutoScalingGroupName) + return aws.ToString(inst.LifecycleState), nil +} + +// Complete continues the named hook for this instance. +func (c *Client) Complete(ctx context.Context, hook string) error { + if c.group == "" { + if _, err := c.State(ctx); err != nil { + return err + } + } + _, err := c.inner.CompleteLifecycleAction(ctx, &autoscaling.CompleteLifecycleActionInput{ + AutoScalingGroupName: aws.String(c.group), + LifecycleHookName: aws.String(hook), + LifecycleActionResult: aws.String("CONTINUE"), + InstanceId: aws.String(c.instanceID), + }) + if err != nil { + return fmt.Errorf("complete lifecycle action %s: %w", hook, err) + } + return nil +} diff --git a/internal/vaultcluster/lifecycle.go b/internal/vaultcluster/lifecycle.go new file mode 100644 index 0000000..db3851f --- /dev/null +++ b/internal/vaultcluster/lifecycle.go @@ -0,0 +1,121 @@ +package vaultcluster + +import ( + "context" + "errors" + "time" +) + +// Auto Scaling lifecycle hooks make a launch-template change survivable on any cluster size, including one. +// +// The launch hook holds a new instance in Pending:Wait until this node is a caught-up Raft voter, so a +// rolling refresh never retires the old node before the new one holds the data. A node that never joins is +// abandoned, which fails the refresh and rolls it back with the old node untouched. +// +// The terminate hook holds a departing instance in Terminating:Wait until this node has left the peer set, +// so the survivors keep quorum. The hook continues whether or not the leave succeeded: ABANDON terminates +// too, and the attempt is logged. +const ( + LaunchHook = "vault-join" + TerminateHook = "vault-leave" + StatePendingWait = "Pending:Wait" + StateTerminatingWait = "Terminating:Wait" +) + +var ErrNotInGroup = errors.New("instance is not in an auto scaling group") + +type LifecycleAPI interface { + State(ctx context.Context) (string, error) + Complete(ctx context.Context, hook string) error +} + +type LifecycleNode interface { + // Ready is nil once this node is an unsealed, caught-up Raft voter. + Ready() error + // Leave removes this node from the Raft peer set. ErrSoleVoter means there was nothing to hand off. + Leave() error +} + +type LifecycleOptions struct { + Poll time.Duration + LeaveTimeout time.Duration + LeaveRetry time.Duration + Logf func(string, ...any) +} + +func (o *LifecycleOptions) defaults() { + if o.Poll <= 0 { + o.Poll = 10 * time.Second + } + if o.LeaveTimeout <= 0 { + o.LeaveTimeout = 2 * time.Minute + } + if o.LeaveRetry <= 0 { + o.LeaveRetry = 3 * time.Second + } + if o.Logf == nil { + o.Logf = func(string, ...any) {} + } +} + +// RunLifecycle polls the instance's lifecycle state until ctx ends or the instance is not in a group. +func RunLifecycle(ctx context.Context, api LifecycleAPI, node LifecycleNode, opts LifecycleOptions) error { + opts.defaults() + left, continued := false, false + for { + state, err := api.State(ctx) + switch { + case errors.Is(err, ErrNotInGroup): + return err + case err != nil: + opts.Logf("lifecycle state: %v", err) + case state == StatePendingWait: + if err := node.Ready(); err != nil { + opts.Logf("waiting to join: %v", err) + } else if err := api.Complete(ctx, LaunchHook); err != nil { + opts.Logf("launch hook: %v", err) + } else { + opts.Logf("joined the raft cluster; launch hook continued") + } + case state == StateTerminatingWait && !continued: + if !left { + left = true + leaveWithRetry(ctx, node, opts) + } + if err := api.Complete(ctx, TerminateHook); err != nil { + opts.Logf("terminate hook: %v", err) + } else { + continued = true + opts.Logf("terminate hook continued") + } + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(opts.Poll): + } + } +} + +func leaveWithRetry(ctx context.Context, node LifecycleNode, opts LifecycleOptions) { + ctx, cancel := context.WithTimeout(ctx, opts.LeaveTimeout) + defer cancel() + for { + err := node.Leave() + switch { + case err == nil: + opts.Logf("left the raft cluster") + return + case errors.Is(err, ErrSoleVoter): + opts.Logf("%v", err) + return + } + opts.Logf("leaving the raft cluster: %v", err) + select { + case <-ctx.Done(): + opts.Logf("giving up on leaving the raft cluster; terminating anyway") + return + case <-time.After(opts.LeaveRetry): + } + } +} diff --git a/internal/vaultcluster/lifecycle_test.go b/internal/vaultcluster/lifecycle_test.go new file mode 100644 index 0000000..708b374 --- /dev/null +++ b/internal/vaultcluster/lifecycle_test.go @@ -0,0 +1,146 @@ +package vaultcluster + +import ( + "context" + "errors" + "sync" + "testing" + "time" +) + +type fakeLifecycleAPI struct { + mu sync.Mutex + states []string + completed []string + failHook string +} + +func (f *fakeLifecycleAPI) State(context.Context) (string, error) { + f.mu.Lock() + defer f.mu.Unlock() + if len(f.states) == 0 { + return "", ErrNotInGroup + } + s := f.states[0] + if len(f.states) > 1 { + f.states = f.states[1:] + } + return s, nil +} + +func (f *fakeLifecycleAPI) Complete(_ context.Context, hook string) error { + f.mu.Lock() + defer f.mu.Unlock() + if hook == f.failHook { + f.failHook = "" + return errors.New("throttled") + } + f.completed = append(f.completed, hook) + if hook == LaunchHook { + f.states = []string{"InService"} + } + if hook == TerminateHook { + f.states = []string{"Terminating:Proceed"} + } + return nil +} + +type fakeNode struct { + readyAfter int + readyCalls int + leaveErrs []error + leaveCalls int +} + +func (n *fakeNode) Ready() error { + n.readyCalls++ + if n.readyCalls <= n.readyAfter { + return errors.New("not a voter yet") + } + return nil +} + +func (n *fakeNode) Leave() error { + n.leaveCalls++ + if len(n.leaveErrs) == 0 { + return nil + } + err := n.leaveErrs[0] + n.leaveErrs = n.leaveErrs[1:] + return err +} + +func runFor(t *testing.T, api *fakeLifecycleAPI, node *fakeNode, d time.Duration) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), d) + defer cancel() + opts := LifecycleOptions{Poll: time.Millisecond, LeaveTimeout: 20 * time.Millisecond, LeaveRetry: time.Millisecond, Logf: t.Logf} + if err := RunLifecycle(ctx, api, node, opts); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("RunLifecycle: %v", err) + } +} + +func TestLifecycleLaunchHookWaitsForVoter(t *testing.T) { + api := &fakeLifecycleAPI{states: []string{StatePendingWait}} + node := &fakeNode{readyAfter: 3} + runFor(t, api, node, 100*time.Millisecond) + if node.readyCalls < 4 { + t.Fatalf("Ready called %d times, want at least 4", node.readyCalls) + } + if len(api.completed) != 1 || api.completed[0] != LaunchHook { + t.Fatalf("completed %v, want [%s] once", api.completed, LaunchHook) + } +} + +func TestLifecycleTerminateLeavesOnceThenContinues(t *testing.T) { + api := &fakeLifecycleAPI{states: []string{"InService", StateTerminatingWait}} + node := &fakeNode{leaveErrs: []error{errors.New("no leader"), nil}} + runFor(t, api, node, 100*time.Millisecond) + if node.leaveCalls != 2 { + t.Fatalf("Leave called %d times, want 2 (one retry)", node.leaveCalls) + } + if len(api.completed) != 1 || api.completed[0] != TerminateHook { + t.Fatalf("completed %v, want [%s] once", api.completed, TerminateHook) + } +} + +func TestLifecycleTerminateContinuesWhenLeaveNeverSucceeds(t *testing.T) { + api := &fakeLifecycleAPI{states: []string{StateTerminatingWait}} + node := &fakeNode{leaveErrs: []error{errors.New("down"), errors.New("down"), errors.New("down"), errors.New("down"), errors.New("down"), errors.New("down"), errors.New("down"), errors.New("down"), errors.New("down"), errors.New("down"), errors.New("down"), errors.New("down"), errors.New("down"), errors.New("down"), errors.New("down"), errors.New("down"), errors.New("down"), errors.New("down"), errors.New("down"), errors.New("down"), errors.New("down"), errors.New("down"), errors.New("down"), errors.New("down"), errors.New("down"), errors.New("down"), errors.New("down"), errors.New("down"), errors.New("down"), errors.New("down")}} + runFor(t, api, node, 150*time.Millisecond) + if len(api.completed) != 1 || api.completed[0] != TerminateHook { + t.Fatalf("completed %v, want [%s]: termination must continue after the leave timeout", api.completed, TerminateHook) + } +} + +func TestLifecycleSoleVoterContinuesWithoutRetry(t *testing.T) { + api := &fakeLifecycleAPI{states: []string{StateTerminatingWait}} + node := &fakeNode{leaveErrs: []error{ErrSoleVoter}} + runFor(t, api, node, 50*time.Millisecond) + if node.leaveCalls != 1 { + t.Fatalf("Leave called %d times, want 1", node.leaveCalls) + } + if len(api.completed) != 1 { + t.Fatalf("completed %v, want the terminate hook", api.completed) + } +} + +func TestLifecycleRetriesHookCompletion(t *testing.T) { + api := &fakeLifecycleAPI{states: []string{StateTerminatingWait}, failHook: TerminateHook} + node := &fakeNode{} + runFor(t, api, node, 100*time.Millisecond) + if node.leaveCalls != 1 { + t.Fatalf("Leave called %d times, want 1: a failed hook call must not re-run the leave", node.leaveCalls) + } + if len(api.completed) != 1 { + t.Fatalf("completed %v, want one successful terminate hook after a retry", api.completed) + } +} + +func TestLifecycleStopsOutsideGroup(t *testing.T) { + api := &fakeLifecycleAPI{} + err := RunLifecycle(context.Background(), api, &fakeNode{}, LifecycleOptions{Poll: time.Millisecond}) + if !errors.Is(err, ErrNotInGroup) { + t.Fatalf("got %v, want ErrNotInGroup", err) + } +} diff --git a/internal/vaultcluster/policies/templates/operator.hcl.tpl b/internal/vaultcluster/policies/templates/operator.hcl.tpl index 8a69b59..8f7827a 100644 --- a/internal/vaultcluster/policies/templates/operator.hcl.tpl +++ b/internal/vaultcluster/policies/templates/operator.hcl.tpl @@ -1,4 +1,5 @@ -# Local operator: health and config read. Can start generate-root (recovery keys still required). Not a tenant secret reader. No restore. +# Local operator: health and config read. Can start generate-root (recovery keys still required). Steps down and +# leaves the Raft peer set on scale-in. Not a tenant secret reader. No restore. path "sys/health" { capabilities = ["read", "sudo"] } @@ -103,6 +104,14 @@ path "sys/storage/raft/autopilot/state" { capabilities = ["read"] } +path "sys/step-down" { + capabilities = ["update", "sudo"] +} + +path "sys/storage/raft/remove-peer" { + capabilities = ["update", "sudo"] +} + path "auth/token/lookup-self" { capabilities = ["read"] } diff --git a/internal/vaultcluster/policy_test.go b/internal/vaultcluster/policy_test.go index b495e62..403ee36 100644 --- a/internal/vaultcluster/policy_test.go +++ b/internal/vaultcluster/policy_test.go @@ -18,8 +18,8 @@ func goldenConfig(shared bool) Config { return Config{KVMount: "kv", TenantPrefix: "customers", EnvPrefix: "envs", DatabaseMount: "database", AuthMount: "approle", SharedEnvs: shared} } -// The unshared goldens are the 0.1.x policies. They must never change: an unshared cluster is unaffected -// by shared-env support. The shared goldens document what a shared cluster applies. +// The unshared goldens must not move for shared-env support: an unshared cluster is unaffected by it. +// Any other change to a policy is deliberate and updates both sets. The shared goldens document what a shared cluster applies. func TestRenderedPoliciesMatchGoldens(t *testing.T) { for _, mode := range []struct { dir string diff --git a/internal/vaultcluster/raftpeers.go b/internal/vaultcluster/raftpeers.go new file mode 100644 index 0000000..4e526d9 --- /dev/null +++ b/internal/vaultcluster/raftpeers.go @@ -0,0 +1,119 @@ +package vaultcluster + +import ( + "encoding/json" + "errors" + "fmt" + "time" +) + +// A leader hands off before leaving, and the new leader needs a moment to win the election. +const leaderHandoffTimeout = 20 * time.Second + +// ErrSoleVoter: this node is the only voter, so there is nobody to hand the data to. +// Removing it would only make the cluster unreachable sooner, so it is left in place. +var ErrSoleVoter = errors.New("this node is the only raft voter; leaving the peer set unchanged") + +type RaftPeer struct { + ID string + Address string + Leader bool + Voter bool +} + +func parseRaftConfiguration(data map[string]any) []RaftPeer { + raw, _ := json.Marshal(data) + var parsed struct { + Config struct { + Servers []struct { + ID string `json:"node_id"` + Address string `json:"address"` + Leader bool `json:"leader"` + Voter bool `json:"voter"` + } `json:"servers"` + } `json:"config"` + } + if err := json.Unmarshal(raw, &parsed); err != nil { + return nil + } + out := make([]RaftPeer, 0, len(parsed.Config.Servers)) + for _, s := range parsed.Config.Servers { + out = append(out, RaftPeer{ID: s.ID, Address: s.Address, Leader: s.Leader, Voter: s.Voter}) + } + return out +} + +// otherVoters counts voters besides nodeID and returns nodeID's own entry, or nil when it is not a peer. +func otherVoters(peers []RaftPeer, nodeID string) (*RaftPeer, int) { + var self *RaftPeer + n := 0 + for i := range peers { + if peers[i].ID == nodeID { + self = &peers[i] + continue + } + if peers[i].Voter { + n++ + } + } + return self, n +} + +func (c *Client) RaftPeers() ([]RaftPeer, error) { + sec, err := c.API.Logical().Read("sys/storage/raft/configuration") + if err != nil { + return nil, err + } + if sec == nil || sec.Data == nil { + return nil, fmt.Errorf("raft configuration is empty") + } + return parseRaftConfiguration(sec.Data), nil +} + +// LeaveRaft removes this node from the peer set so the remaining voters keep quorum. +// A leader steps down first, so the removal is committed by a node that stays. +func (c *Client) LeaveRaft(nodeID string) error { + peers, err := c.RaftPeers() + if err != nil { + return err + } + self, others := otherVoters(peers, nodeID) + if self == nil { + return nil + } + if others == 0 { + return ErrSoleVoter + } + if self.Leader { + if err := c.API.Sys().StepDown(); err != nil { + return fmt.Errorf("step down: %w", err) + } + if err := c.waitOtherLeader(leaderHandoffTimeout); err != nil { + return err + } + } + if _, err := c.API.Logical().Write("sys/storage/raft/remove-peer", map[string]any{"server_id": nodeID}); err != nil { + return fmt.Errorf("remove peer %s: %w", nodeID, err) + } + // A removed node may refuse further requests; that is also confirmation. + peers, err = c.RaftPeers() + if err != nil { + return nil + } + if self, _ := otherVoters(peers, nodeID); self != nil { + return fmt.Errorf("node %s is still a raft peer after remove-peer", nodeID) + } + return nil +} + +func (c *Client) waitOtherLeader(timeout time.Duration) error { + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + l, err := c.API.Sys().Leader() + if err == nil && !l.IsSelf && l.LeaderAddress != "" { + return nil + } + time.Sleep(500 * time.Millisecond) + } + return fmt.Errorf("no other node took leadership within %s", timeout) +} diff --git a/internal/vaultcluster/raftpeers_test.go b/internal/vaultcluster/raftpeers_test.go new file mode 100644 index 0000000..e75abdd --- /dev/null +++ b/internal/vaultcluster/raftpeers_test.go @@ -0,0 +1,62 @@ +package vaultcluster + +import ( + "encoding/json" + "testing" +) + +const raftConfigurationJSON = `{"config":{"index":42,"servers":[ + {"address":"10.0.1.10:8201","leader":true,"node_id":"i-old","protocol_version":"3","voter":true}, + {"address":"10.0.2.20:8201","leader":false,"node_id":"i-new","protocol_version":"3","voter":true}, + {"address":"10.0.3.30:8201","leader":false,"node_id":"i-learner","protocol_version":"3","voter":false}]}}` + +func raftConfiguration(t *testing.T) map[string]any { + t.Helper() + var data map[string]any + if err := json.Unmarshal([]byte(raftConfigurationJSON), &data); err != nil { + t.Fatal(err) + } + return data +} + +func TestParseRaftConfiguration(t *testing.T) { + peers := parseRaftConfiguration(raftConfiguration(t)) + if len(peers) != 3 { + t.Fatalf("got %d peers, want 3", len(peers)) + } + if !peers[0].Leader || !peers[0].Voter || peers[0].ID != "i-old" || peers[0].Address != "10.0.1.10:8201" { + t.Fatalf("leader parsed wrong: %+v", peers[0]) + } + if peers[2].Voter { + t.Fatalf("learner must not be a voter: %+v", peers[2]) + } +} + +func TestOtherVotersIgnoresSelfAndLearners(t *testing.T) { + peers := parseRaftConfiguration(raftConfiguration(t)) + + self, others := otherVoters(peers, "i-old") + if self == nil || !self.Leader || others != 1 { + t.Fatalf("old leader: self=%+v others=%d, want leader with 1 other voter", self, others) + } + + self, others = otherVoters(peers, "i-new") + if self == nil || self.Leader || others != 1 { + t.Fatalf("new node: self=%+v others=%d, want follower with 1 other voter", self, others) + } + + if self, _ := otherVoters(peers, "i-gone"); self != nil { + t.Fatalf("a node outside the configuration must have no self entry") + } + + single := []RaftPeer{{ID: "i-only", Leader: true, Voter: true}, {ID: "i-learner"}} + if _, others := otherVoters(single, "i-only"); others != 0 { + t.Fatalf("a lone voter with a learner has %d other voters, want 0", others) + } +} + +func TestParseRaftConfigurationEmpty(t *testing.T) { + if peers := parseRaftConfiguration(map[string]any{}); len(peers) != 0 { + t.Fatalf("got %v, want none", peers) + } +} diff --git a/internal/vaultcluster/testdata/policies/shared/operator.hcl b/internal/vaultcluster/testdata/policies/shared/operator.hcl index 0625271..4ac9334 100644 --- a/internal/vaultcluster/testdata/policies/shared/operator.hcl +++ b/internal/vaultcluster/testdata/policies/shared/operator.hcl @@ -1,4 +1,5 @@ -# Local operator: health and config read. Can start generate-root (recovery keys still required). Not a tenant secret reader. No restore. +# Local operator: health and config read. Can start generate-root (recovery keys still required). Steps down and +# leaves the Raft peer set on scale-in. Not a tenant secret reader. No restore. path "sys/health" { capabilities = ["read", "sudo"] } @@ -103,6 +104,14 @@ path "sys/storage/raft/autopilot/state" { capabilities = ["read"] } +path "sys/step-down" { + capabilities = ["update", "sudo"] +} + +path "sys/storage/raft/remove-peer" { + capabilities = ["update", "sudo"] +} + path "auth/token/lookup-self" { capabilities = ["read"] } diff --git a/internal/vaultcluster/testdata/policies/unshared/operator.hcl b/internal/vaultcluster/testdata/policies/unshared/operator.hcl index 0625271..4ac9334 100644 --- a/internal/vaultcluster/testdata/policies/unshared/operator.hcl +++ b/internal/vaultcluster/testdata/policies/unshared/operator.hcl @@ -1,4 +1,5 @@ -# Local operator: health and config read. Can start generate-root (recovery keys still required). Not a tenant secret reader. No restore. +# Local operator: health and config read. Can start generate-root (recovery keys still required). Steps down and +# leaves the Raft peer set on scale-in. Not a tenant secret reader. No restore. path "sys/health" { capabilities = ["read", "sudo"] } @@ -103,6 +104,14 @@ path "sys/storage/raft/autopilot/state" { capabilities = ["read"] } +path "sys/step-down" { + capabilities = ["update", "sudo"] +} + +path "sys/storage/raft/remove-peer" { + capabilities = ["update", "sudo"] +} + path "auth/token/lookup-self" { capabilities = ["read"] } diff --git a/vault-node/files/vault-lifecycle.service b/vault-node/files/vault-lifecycle.service new file mode 100644 index 0000000..cca6c00 --- /dev/null +++ b/vault-node/files/vault-lifecycle.service @@ -0,0 +1,17 @@ +[Unit] +Description=Vault Auto Scaling lifecycle hooks +# Ordered after bootstrap but not bound to it, for the same reason as vault-health.service. +Requires=vault.service +After=vault.service vault-bootstrap.service + +[Service] +EnvironmentFile=/etc/vault.d/vault-utils.env +EnvironmentFile=/etc/vault.d/node.env +ExecStart=/usr/local/bin/vault-utils lifecycle +Restart=always +RestartSec=10 +User=vault +Group=vault + +[Install] +WantedBy=multi-user.target diff --git a/vault-node/provision.sh b/vault-node/provision.sh index 800df2c..076c528 100755 --- a/vault-node/provision.sh +++ b/vault-node/provision.sh @@ -22,6 +22,7 @@ install -m 0644 /tmp/vault-image/vault-configure.service /etc/systemd/system/vau install -m 0644 /tmp/vault-image/vault-bootstrap.service /etc/systemd/system/vault-bootstrap.service install -m 0644 /tmp/vault-image/vault-health.service /etc/systemd/system/vault-health.service install -m 0644 /tmp/vault-image/vault-snapshot.service /etc/systemd/system/vault-snapshot.service +install -m 0644 /tmp/vault-image/vault-lifecycle.service /etc/systemd/system/vault-lifecycle.service bash -n /usr/local/bin/vault-node-configure /usr/local/bin/vault version @@ -34,4 +35,5 @@ systemctl enable \ vault.service \ vault-bootstrap.service \ vault-health.service \ - vault-snapshot.service + vault-snapshot.service \ + vault-lifecycle.service