feat(aws): make instance refresh safe on one-node clusters

CHANGELOG.md

@@ -1,9 +1,21 @@
+# 0.4.0
+
+Module: `aws-ec2-vault-cluster` 0.4.0. Needs a fresh node AMI. The operator policy gains `sys/step-down` and `sys/storage/raft/remove-peer`; a cluster bootstrapped before this release must rewrite it with a break-glass root before the leave step works. A launch-template change rolls the nodes.
+
+* A rolling instance refresh is safe on a one-node cluster. A launch lifecycle hook (`vault-join`) holds a new node until `vault-utils lifecycle` reports it an unsealed, caught-up Raft voter; a node that never joins is abandoned after 30 minutes, which fails the refresh and rolls it back with the old node untouched. A terminate hook (`vault-leave`) holds a departing node until it has stepped down and removed itself from the peer set, so the survivor keeps quorum. Before this, the refresh counted a new node healthy on EC2 status alone and terminated the only copy of the data.
+
 # 0.3.1
 
 Module: `aws-ec2-vault-cluster` 0.3.1. Launch-template change; nodes roll.
 
 * Nodes advertise `api_addr` as the URL clients use: `user_vault_addr` when a subdomain is connected, else `vault_addr`. It was always `https://vault.internal:8200`, so a standby redirect sent browsers to an origin the NLB certificate does not name and the web UI failed with "The request failed and the interceptors did not return an alternative response".
 
+# 0.3.0
+
+Modules: `aws-ec2-vault-cluster` 0.3.0, `aws-vault-access` 0.3.0. No runtime change.
+
+* A `v*` tag push publishes both modules at that version (`.github/workflows/publish.yml`). The cluster module packages `templates/*`.
+
 # 0.2.0
 
 Modules: `aws-ec2-vault-cluster` 0.2.0, `aws-vault-access` 0.3.0. Needs `vault-admin` 0.3.0 and a fresh node AMI. Nothing changes for a cluster outside the shared previews env.

README.md

@@ -46,7 +46,7 @@ Implemented:
 - Auto-init (first start) and one-shot Shamir unseal via `vault-utils`
 - Isolation tests in Go (`go test`); credentials tests in Go (`TestCredentialsMatrix`)
 - `bootstrap aws` (KMS auto-unseal, Secrets Manager tokens), health on 8210, S3 snapshots and S3 restore
-- AWS AMI, user-data, internal NLB (TLS 8200, health 8210), `vault.internal`, optional user-facing subdomain SNI, ASG (`cluster_size`), rolling instance refresh on launch-template change
+- AWS AMI, user-data, internal NLB (TLS 8200, health 8210), `vault.internal`, optional user-facing subdomain SNI, ASG (`cluster_size`), rolling instance refresh on launch-template change gated by lifecycle hooks (join before the old node leaves; leave the peer set before termination)
 
 Not implemented:
 

aws/aws-ec2-vault-cluster/README.md

@@ -68,6 +68,12 @@ Per-tenant CLI on a shared cluster takes `--env`; on an unshared one `--env` is
 
 Nodes advertise `api_addr` as `user_vault_addr` when a subdomain is connected, else `vault_addr`. Redirects from a standby therefore land on the origin clients already use. Clients on `vault.internal` set `VAULT_TLS_SERVER_NAME` to `tls_server_name`.
 
+## Upgrades
+
+A launch-template change (AMI, instance type, user-data) starts a rolling instance refresh. Two lifecycle hooks make it safe at any `cluster_size`, including 1. `vault-join` holds a new instance in `Pending:Wait` until `vault-utils lifecycle` sees it unsealed, a Raft voter, and caught up; a node that never joins is abandoned after 30 minutes and the refresh rolls back. `vault-leave` holds a departing instance in `Terminating:Wait` until it has stepped down and removed itself from the peer set; it terminates either way. Watch a node with `journalctl -u vault-lifecycle`.
+
+The ASG health check stays `EC2`. An `ELB` check would replace a one-node cluster whenever Vault was briefly unhealthy, and the data lives on that instance.
+
 ## Outputs
 
 `vault_addr`, `user_vault_addr`, `tls_server_name`, `vault_fqdn`, `user_fqdn`, `vault_api_port`, `nlb_security_group_id`, `admin_function_name`, `shared`, `operator_secret_arn`, `provisioning_secret_arn`, `db_hostname`, `db_port`, `db_endpoint`, `private_urls`, `public_urls`, plus node identifiers (`role_name`, `instance_profile_name`, `security_group_id`, `autoscaling_group_name`, `ami_id`).

aws/aws-ec2-vault-cluster/asg.tf

@@ -22,6 +22,8 @@ resource "aws_autoscaling_group" "this" {
     strategy = "Rolling"
 
     preferences {
+      # Health after the launch hook continues; the hook, not warmup, is what proves the node joined.
+      instance_warmup        = 60
       min_healthy_percentage = 100
       max_healthy_percentage = local.asg_max_healthy_percentage
       auto_rollback          = true
@@ -41,3 +43,23 @@ resource "aws_autoscaling_group" "this" {
     propagate_at_launch = true
   }
 }
+
+# A new instance waits here until vault-utils lifecycle reports it an unsealed, caught-up Raft voter. One that
+# never joins is abandoned, which fails the instance refresh and rolls it back with the old node untouched.
+resource "aws_autoscaling_lifecycle_hook" "join" {
+  name                   = "vault-join"
+  autoscaling_group_name = aws_autoscaling_group.this.name
+  lifecycle_transition   = "autoscaling:EC2_INSTANCE_LAUNCHING"
+  default_result         = "ABANDON"
+  heartbeat_timeout      = 1800
+}
+
+# A departing instance waits here until it has left the Raft peer set, so the remaining nodes keep quorum.
+# It terminates either way; vault-utils logs the attempt.
+resource "aws_autoscaling_lifecycle_hook" "leave" {
+  name                   = "vault-leave"
+  autoscaling_group_name = aws_autoscaling_group.this.name
+  lifecycle_transition   = "autoscaling:EC2_INSTANCE_TERMINATING"
+  default_result         = "CONTINUE"
+  heartbeat_timeout      = 300
+}

aws/aws-ec2-vault-cluster/iam.tf

@@ -94,6 +94,26 @@ data "aws_iam_policy_document" "this" {
     resources = ["*"]
   }
 
+  statement {
+    sid       = "LifecycleState"
+    effect    = "Allow"
+    actions   = ["autoscaling:DescribeAutoScalingInstances"]
+    resources = ["*"]
+  }
+
+  # The group name is unknown until it exists, so the hook permission is scoped by the cluster tag instead.
+  statement {
+    sid       = "LifecycleHooks"
+    effect    = "Allow"
+    actions   = ["autoscaling:CompleteLifecycleAction"]
+    resources = ["*"]
+    condition {
+      test     = "StringEquals"
+      variable = "autoscaling:ResourceTag/${local.vault_cluster_tag_key}"
+      values   = [local.vault_cluster_tag_value]
+    }
+  }
+
   statement {
     sid    = "VaultAwsAuth"
     effect = "Allow"

aws/aws-ec2-vault-cluster/outputs.tftest.hcl

@@ -146,6 +146,11 @@ run "is_not_shared_in_a_pipeline_env" {
     condition     = output.shared == false && strcontains(base64decode(aws_launch_template.this.user_data), "SHARED_ENVS=false")
     error_message = "A cluster in any env but the shared previews env is unshared and tells its nodes so."
   }
+
+  assert {
+    condition     = aws_autoscaling_lifecycle_hook.join.default_result == "ABANDON" && aws_autoscaling_lifecycle_hook.leave.default_result == "CONTINUE"
+    error_message = "A node that never joins is abandoned so the refresh rolls back; a departing node terminates after trying to leave."
+  }
 }
 
 run "is_not_shared_in_a_preview_env" {

cmd/vault-utils/main.go

@@ -1,6 +1,8 @@
 package main
 
 import (
+	"context"
+	"errors"
 	"fmt"
 	"log"
 	"os"
@@ -9,6 +11,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/hashicorp/vault/api"
+	"github.com/nullstone-modules/vault-cluster/internal/aws/asg"
 	"github.com/nullstone-modules/vault-cluster/internal/aws/s3"
 	"github.com/nullstone-modules/vault-cluster/internal/aws/secretsmanager"
 	"github.com/nullstone-modules/vault-cluster/internal/vaultcluster"
@@ -45,6 +49,7 @@ Commands:
   snapshot schedule                 Cron loop (BACKUP_SCHEDULE; empty disables)
   health                            Print seal status
   health serve                      HTTP on :8210 (200 only if this node is a Raft voter and caught up)
+  lifecycle                         ASG node: continue the launch hook once a caught-up voter; leave Raft before termination
   env --org --stack --env --block   Print shell settings for a Nullstone Vault cluster workspace
 
 Without VAULT_TOKEN, tenants, envs, snapshot take|restore, and health use the token from "vault login".
@@ -87,6 +92,8 @@ func run(cmd string, args []string) error {
 			return runHealthServe(c)
 		}
 		return c.Health()
+	case "lifecycle":
+		return runLifecycle(c)
 	default:
 		usage()
 		return fmt.Errorf("unknown command %q", cmd)
@@ -502,3 +509,49 @@ func getenv(k, def string) string {
 	}
 	return def
 }
+
+func runLifecycle(c *vaultcluster.Client) error {
+	nodeID := os.Getenv("VAULT_RAFT_NODE_ID")
+	if nodeID == "" {
+		return fmt.Errorf("VAULT_RAFT_NODE_ID is required for lifecycle")
+	}
+	hooks, err := asg.New(nodeID)
+	if err != nil {
+		return err
+	}
+	log.Printf("lifecycle watching instance %s", nodeID)
+	return vaultcluster.RunLifecycle(context.Background(), hooks, &lifecycleNode{c: c, nodeID: nodeID}, vaultcluster.LifecycleOptions{Logf: log.Printf})
+}
+
+// lifecycleNode loads the operator token lazily: on a fresh cluster it does not exist until bootstrap
+// finishes, and on a rebuilt one Secrets Manager may still hold the previous cluster's token.
+type lifecycleNode struct {
+	c        *vaultcluster.Client
+	nodeID   string
+	renewing bool
+}
+
+func (n *lifecycleNode) withToken(op func() error) error {
+	if err := useOperatorToken(n.c); err != nil {
+		return err
+	}
+	if !n.renewing {
+		n.renewing = true
+		go n.c.RenewToken(tokenRenewInterval, nil)
+	}
+	err := op()
+	var re *api.ResponseError
+	if errors.As(err, &re) && re.StatusCode == 403 {
+		n.c.Cfg.Token = ""
+		n.c.API.ClearToken()
+	}
+	return err
+}
+
+func (n *lifecycleNode) Ready() error {
+	return n.withToken(func() error { return n.c.NodeHealthOK(n.nodeID) })
+}
+
+func (n *lifecycleNode) Leave() error {
+	return n.withToken(func() error { return n.c.LeaveRaft(n.nodeID) })
+}

go.mod

@@ -3,11 +3,12 @@ module github.com/nullstone-modules/vault-cluster
 go 1.26.5
 
 require (
-	github.com/aws/aws-sdk-go-v2 v1.41.2
+	github.com/aws/aws-sdk-go-v2 v1.47.1
 	github.com/aws/aws-sdk-go-v2/config v1.29.14
+	github.com/aws/aws-sdk-go-v2/service/autoscaling v1.78.2
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.79.3
 	github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.35.6
-	github.com/aws/smithy-go v1.24.1
+	github.com/aws/smithy-go v1.28.1
 	github.com/hashicorp/vault/api v1.16.0
 	github.com/robfig/cron/v3 v3.0.1
 	gopkg.in/nullstone-io/go-api-client.v0 v0.0.0-20260929173059-6bbecdaf8ea3
@@ -20,8 +21,8 @@ require (
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.5 // indirect
 	github.com/aws/aws-sdk-go-v2/credentials v1.19.10 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.18 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.18 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.18 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.18 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.5 // indirect

go.sum

@@ -5,8 +5,8 @@ github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmms
 github.com/apparentlymart/go-textseg/v17 v17.0.1 h1:bpMXRgQ5cEoRNuQke1a80/Nl6w3G5eoIbWo9f3gXkAs=
 github.com/apparentlymart/go-textseg/v17 v17.0.1/go.mod h1:fa8X4jgGeevslICIY6LcdjkSecWnXmYd9Lk34z/VxZs=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
-github.com/aws/aws-sdk-go-v2 v1.41.2 h1:LuT2rzqNQsauaGkPK/7813XxcZ3o3yePY0Iy891T2ls=
-github.com/aws/aws-sdk-go-v2 v1.41.2/go.mod h1:IvvlAZQXvTXznUPfRVfryiG1fbzE2NGK6m9u39YQ+S4=
+github.com/aws/aws-sdk-go-v2 v1.47.1 h1:uOIZnp4PK3ZhKI0dNrJrhTEsLxbpXHTAJlwoS1pvAtw=
+github.com/aws/aws-sdk-go-v2 v1.47.1/go.mod h1:bttEH6JqnUL8LepvDVfdrds/fZ5bCIxzpe3abyUrhDU=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.5 h1:zWFmPmgw4sveAYi1mRqG+E/g0461cJ5M4bJ8/nc6d3Q=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.5/go.mod h1:nVUlMLVV8ycXSb7mSkcNu9e3v/1TJq2RTlrPwhYWr5c=
 github.com/aws/aws-sdk-go-v2/config v1.29.14 h1:f+eEi/2cKCg9pqKBoAIwRGzVb70MRKqWX4dg1BDcSJM=
@@ -15,14 +15,16 @@ github.com/aws/aws-sdk-go-v2/credentials v1.19.10 h1:EEhmEUFCE1Yhl7vDhNOI5OCL/iK
 github.com/aws/aws-sdk-go-v2/credentials v1.19.10/go.mod h1:RnnlFCAlxQCkN2Q379B67USkBMu1PipEEiibzYN5UTE=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.18 h1:Ii4s+Sq3yDfaMLpjrJsqD6SmG/Wq/P5L/hw2qa78UAY=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.18/go.mod h1:6x81qnY++ovptLE6nWQeWrpXxbnlIex+4H4eYYGcqfc=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.18 h1:F43zk1vemYIqPAwhjTjYIz0irU2EY7sOb/F5eJ3HuyM=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.18/go.mod h1:w1jdlZXrGKaJcNoL+Nnrj+k5wlpGXqnNrKoP22HvAug=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.18 h1:xCeWVjj0ki0l3nruoyP2slHsGArMxeiiaoPN5QZH6YQ=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.18/go.mod h1:r/eLGuGCBw6l36ZRWiw6PaZwPXb6YOj+i/7MizNl5/k=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4 h1:CLq4+8UHCI+ZZYl/EuJxXovaIVN2xeeT8JV+dsApQ5E=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4/go.mod h1:Wv4q5sAM04xAMkoOedxLx2inVf6K5FdxYp+A61L+q/0=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4 h1:dD4MR81I7YkpEBRk6UP9rocC2QnT3qVuXwzlYTtfGEs=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4/go.mod h1:EcXV1kAFd5XwSkDHlj94gnF3q5CkJyYiIJfH8N0VmrE=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo=
 github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.18 h1:eZioDaZGJ0tMM4gzmkNIO2aAoQd+je7Ug7TkvAzlmkU=
 github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.18/go.mod h1:CCXwUKAJdoWr6/NcxZ+zsiPr6oH/Q5aTooRGYieAyj4=
+github.com/aws/aws-sdk-go-v2/service/autoscaling v1.78.2 h1:3zCxKK7jLLtQLxpauXY56hmAZGhyYSR1+XQ9QviouPg=
+github.com/aws/aws-sdk-go-v2/service/autoscaling v1.78.2/go.mod h1:4roDw8gYFhAVo1b2ckuzEa0QPtpRXgU4o+dn44IvNF0=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.5 h1:CeY9LUdur+Dxoeldqoun6y4WtJ3RQtzk0JMP2gfUay0=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.5/go.mod h1:AZLZf2fMaahW5s/wMRciu1sYbdsikT/UHwbUjOdEVTc=
 github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.10 h1:fJvQ5mIBVfKtiyx0AHY6HeWcRX5LGANLpq8SVR+Uazs=
@@ -41,8 +43,8 @@ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.15 h1:edCcNp9eGIUDUCrzoCu1jWA
 github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.15/go.mod h1:lyRQKED9xWfgkYC/wmmYfv7iVIM68Z5OQ88ZdcV1QbU=
 github.com/aws/aws-sdk-go-v2/service/sts v1.41.7 h1:NITQpgo9A5NrDZ57uOWj+abvXSb83BbyggcUBVksN7c=
 github.com/aws/aws-sdk-go-v2/service/sts v1.41.7/go.mod h1:sks5UWBhEuWYDPdwlnRFn1w7xWdH29Jcpe+/PJQefEs=
-github.com/aws/smithy-go v1.24.1 h1:VbyeNfmYkWoxMVpGUAbQumkODcYmfMRfZ8yQiH30SK0=
-github.com/aws/smithy-go v1.24.1/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
+github.com/aws/smithy-go v1.28.1 h1:R/nXH00c8qcfCzQVELtRw+eLQWtzv+VAIEFJ1/xxXlQ=
+github.com/aws/smithy-go v1.28.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
 github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
 github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=

internal/aws/asg/asg.go

@@ -0,0 +1,66 @@
+// Package asg answers lifecycle hooks on the node's own Auto Scaling group.
+package asg
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/service/autoscaling"
+
+	"github.com/nullstone-modules/vault-cluster/internal/vaultcluster"
+)
+
+type Client struct {
+	inner      *autoscaling.Client
+	instanceID string
+	group      string
+}
+
+// New talks about one instance, which is this node: the Raft node ID is the instance ID.
+func New(instanceID string) (*Client, error) {
+	if instanceID == "" {
+		return nil, fmt.Errorf("VAULT_RAFT_NODE_ID (the instance ID) is required")
+	}
+	cfg, err := config.LoadDefaultConfig(context.Background())
+	if err != nil {
+		return nil, fmt.Errorf("AWS credentials: %w", err)
+	}
+	return &Client{inner: autoscaling.NewFromConfig(cfg), instanceID: instanceID}, nil
+}
+
+// State is the instance's lifecycle state, e.g. Pending:Wait, InService, Terminating:Wait.
+func (c *Client) State(ctx context.Context) (string, error) {
+	out, err := c.inner.DescribeAutoScalingInstances(ctx, &autoscaling.DescribeAutoScalingInstancesInput{
+		InstanceIds: []string{c.instanceID},
+	})
+	if err != nil {
+		return "", fmt.Errorf("describe auto scaling instance %s: %w", c.instanceID, err)
+	}
+	if len(out.AutoScalingInstances) == 0 {
+		return "", vaultcluster.ErrNotInGroup
+	}
+	inst := out.AutoScalingInstances[0]
+	c.group = aws.ToString(inst.AutoScalingGroupName)
+	return aws.ToString(inst.LifecycleState), nil
+}
+
+// Complete continues the named hook for this instance.
+func (c *Client) Complete(ctx context.Context, hook string) error {
+	if c.group == "" {
+		if _, err := c.State(ctx); err != nil {
+			return err
+		}
+	}
+	_, err := c.inner.CompleteLifecycleAction(ctx, &autoscaling.CompleteLifecycleActionInput{
+		AutoScalingGroupName:  aws.String(c.group),
+		LifecycleHookName:     aws.String(hook),
+		LifecycleActionResult: aws.String("CONTINUE"),
+		InstanceId:            aws.String(c.instanceID),
+	})
+	if err != nil {
+		return fmt.Errorf("complete lifecycle action %s: %w", hook, err)
+	}
+	return nil
+}