fix(export): expand --redact-secrets for sanitize delivery

CHANGELOG.md

@@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 
 ## [Unreleased]
 
+### Changed
+
+- **threescale-export** — `--redact-secrets` now fully redacts the consulting sanitize key set (certificates/PEM blocks, `password`, `apikey`, `secret_token`, `username`, and infra URLs such as `private_endpoint`, `admin_url`, `*BaseURL`, issuer endpoints). Issuer URLs are no longer host-preserved (credentials+host become `***REDACTED***`). Multi-line YAML PEM bodies are stripped.
+
 ## [0.4.6] - 2026-09-25
 
 ### Added

README.md

@@ -283,15 +283,21 @@ Self-signed TLS (Admin Portal or toolbox):
 
 `--redact-secrets` is **opt-in** (default off). When set, every `.json`, `.yaml`, and `.yml` file under the export root is processed before `manifest.json` is written.
 
+Matching is case-insensitive and underscore-insensitive (`api_key` / `apikey`, `issuerEndpoint` / `issuer_endpoint`).
+
 **Fully redacted keys** (value becomes `***REDACTED***`):
 
-`access_token`, `api_key`, `app_id`, `app_key`, `client_id`, `client_secret`, `provider_key`, `provider_verification_key`, `secret`, `user_key`
+Secrets / credentials: `access_token`, `api_key` / `apikey`, `app_id`, `app_key`, `ca_certificates`, `certificate`, `certificate_key`, `client_id`, `client_secret`, `password`, `provider_key`, `provider_verification_key`, `secret`, `secret_token`, `user_key`, `username`
+
+Infra / URL fields: `admin_url`, `api_backend`, `endpoint`, `href`, `issuer_endpoint` / `issuerEndpoint`, `oidc_issuer_endpoint`, `private_endpoint`, `privateBaseURL`, `productionPublicBaseURL`, `sandbox_endpoint`, `staging_endpoint`, `stagingPublicBaseURL`, `token_url`
 
-**Issuer URL stripping** (`issuer_endpoint`, `oidc_issuer_endpoint`): embedded credentials are removed (`https://user:pass@host/path` → `https://host/path`); host, path, and query stay visible.
+YAML literal/folded blocks under those keys (e.g. PEM `certificate` / `certificate_key`) are removed entirely so multi-line key material does not remain in the file.
 
 **Preserved auth-mode flags** (not secrets): `auth_user_key`, `auth_app_id`, `auth_app_key`
 
-After redaction, a cleartext scan runs over the same artifacts. If any sensitive value or issuer userinfo remains, **export fails** with a path-qualified error.
+After redaction, a cleartext scan runs over the same artifacts (including residual PEM markers). If any sensitive value remains, **export fails** with a path-qualified error.
+
+**Impact:** topology reports lose concrete URLs/issuers when redaction is on; product/backend counts, auth modes, and policy **names** remain usable for sizing.
 
 ### Hit metrics (Analytics API)
 

internal/export/exporter.go

@@ -163,8 +163,14 @@ func (s *Service) Export(ctx context.Context, opts Options) (*output.Manifest, e
 		applyMetricsManifest(manifest, since, until, granularity, metricName)
 	}
 
+	if err := writer.WriteManifest(manifest); err != nil {
+		return manifest, err
+	}
+
 	if opts.RedactSecrets {
 		rep.Phase("Redacting secrets")
+		// Run after WriteManifest so admin_url and other sanitize keys in
+		// manifest.json are redacted with the rest of the export tree.
 		if err := RedactDirectory(writer.Root()); err != nil {
 			return manifest, err
 		}
@@ -173,9 +179,6 @@ func (s *Service) Export(ctx context.Context, opts Options) (*output.Manifest, e
 		}
 	}
 
-	if err := writer.WriteManifest(manifest); err != nil {
-		return manifest, err
-	}
 	return manifest, nil
 }
 

internal/export/exporter_test.go

@@ -370,11 +370,12 @@ func TestExportRedactSecrets(t *testing.T) {
 		t.Fatal(err)
 	}
 	oidcStr := string(oidcData)
-	if strings.Contains(oidcStr, "zync-user") || strings.Contains(oidcStr, "zync-pass") {
-		t.Fatalf("issuer userinfo still present: %s", oidcData)
+	if strings.Contains(oidcStr, "zync-user") || strings.Contains(oidcStr, "zync-pass") ||
+		strings.Contains(oidcStr, "idp.example.com") {
+		t.Fatalf("issuer endpoint should be fully redacted: %s", oidcData)
 	}
-	if !strings.Contains(oidcStr, "https://idp.example.com/realms/demo") {
-		t.Fatalf("expected stripped issuer host in %s", oidcData)
+	if !strings.Contains(oidcStr, "***REDACTED***") {
+		t.Fatalf("expected redaction marker in oidc config: %s", oidcData)
 	}
 	if strings.Contains(oidcStr, "oidc-client-id") {
 		t.Fatalf("client_id not redacted: %s", oidcData)

internal/export/redact.go

@@ -3,36 +3,83 @@ package export
 import (
 	"encoding/json"
 	"fmt"
-	"net/url"
 	"os"
 	"path/filepath"
 	"regexp"
 	"strings"
+	"unicode"
 )
 
-var sensitiveJSONKeys = map[string]struct{}{
-	"access_token":              {},
-	"client_secret":               {},
-	"secret":                      {},
-	"api_key":                     {},
-	"user_key":                    {},
-	"app_key":                     {},
-	"provider_key":                {},
-	"provider_verification_key":   {},
-	"client_id":                   {},
-	"app_id":                      {},
+// sensitiveRedactKeys lists field names that become ***REDACTED*** under --redact-secrets.
+// Matching uses normalizeRedactKey (lower-case, underscores removed) so api_key and apikey,
+// issuerEndpoint and issuer_endpoint, stagingPublicBaseURL and staging_public_base_url align.
+//
+// Covers the consulting sanitize contract (secrets + infra URLs) including the BPBA key list.
+var sensitiveRedactKeys = map[string]struct{}{
+	// Original secret keys
+	"accesstoken":             {},
+	"clientsecret":            {},
+	"secret":                  {},
+	"apikey":                  {},
+	"userkey":                 {},
+	"appkey":                  {},
+	"providerkey":             {},
+	"providerverificationkey": {},
+	"clientid":                {},
+	"appid":                   {},
+	// Expanded secrets (client findings)
+	"password":        {},
+	"secrettoken":     {},
+	"certificate":     {},
+	"certificatekey":  {},
+	"cacertificates":  {},
+	"username":        {},
+	// Infra / URL fields (client findings) — fully redacted for sanitize delivery
+	"href":                    {},
+	"privateendpoint":         {},
+	"tokenurl":                {},
+	"apibackend":              {},
+	"endpoint":                {},
+	"oidcissuerendpoint":      {},
+	"issuerendpoint":          {},
+	"sandboxendpoint":         {},
+	"stagingendpoint":         {},
+	"stagingpublicbaseurl":    {},
+	"productionpublicbaseurl": {},
+	"privatebaseurl":          {},
+	"adminurl":                {},
 }
 
-var issuerJSONKeys = map[string]struct{}{
-	"issuer_endpoint":      {},
-	"oidc_issuer_endpoint": {},
-}
+// yamlSensitiveKeyPattern matches a YAML mapping key that should be fully redacted.
+// Captures indent+key+colon so the value (same line and/or indented block) can be replaced.
+var yamlSensitiveKeyPattern = regexp.MustCompile(`(?i)^([ \t]*)(` +
+	`access_token|client_secret|secret_token|secret|api_key|apikey|user_key|app_key|` +
+	`provider_key|provider_verification_key|client_id|app_id|password|username|` +
+	`certificate_key|certificate|ca_certificates|` +
+	`href|private_endpoint|token_url|api_backend|endpoint|` +
+	`oidc_issuer_endpoint|issuer_endpoint|issuerEndpoint|sandbox_endpoint|staging_endpoint|` +
+	`stagingPublicBaseURL|productionPublicBaseURL|privateBaseURL|admin_url` +
+	`)[ \t]*:[ \t]*(.*)$`)
 
-var yamlSecretPattern = regexp.MustCompile(`(?m)^(\s*(?:access_token|client_secret|secret|api_key|user_key|app_key|provider_key|provider_verification_key|client_id|app_id)\s*:\s*).+$`)
+const redactedMarker = "***REDACTED***"
+const redactedValue = `"` + redactedMarker + `"`
 
-var yamlIssuerPattern = regexp.MustCompile(`(?m)^(\s*(?:issuer_endpoint|oidc_issuer_endpoint)\s*:\s*)(.+)$`)
+func normalizeRedactKey(k string) string {
+	var b strings.Builder
+	b.Grow(len(k))
+	for _, r := range k {
+		if r == '_' || r == '-' {
+			continue
+		}
+		b.WriteRune(unicode.ToLower(r))
+	}
+	return b.String()
+}
 
-const redactedValue = `"***REDACTED***"`
+func isSensitiveRedactKey(k string) bool {
+	_, ok := sensitiveRedactKeys[normalizeRedactKey(k)]
+	return ok
+}
 
 func RedactDirectory(root string) error {
 	return filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
@@ -81,32 +128,77 @@ func redactYAMLFile(path string) error {
 }
 
 func redactYAMLContent(content string) string {
-	content = yamlSecretPattern.ReplaceAllString(content, `${1}`+redactedValue)
-	return yamlIssuerPattern.ReplaceAllStringFunc(content, func(match string) string {
-		sub := yamlIssuerPattern.FindStringSubmatch(match)
-		if len(sub) < 3 {
-			return match
+	lines := strings.Split(content, "\n")
+	out := make([]string, 0, len(lines))
+	for i := 0; i < len(lines); {
+		line := lines[i]
+		sub := yamlSensitiveKeyPattern.FindStringSubmatch(line)
+		if sub == nil {
+			out = append(out, line)
+			i++
+			continue
 		}
-		return sub[1] + stripURLUserinfo(strings.TrimSpace(sub[2]))
-	})
+		indent, key, rest := sub[1], sub[2], strings.TrimSpace(sub[3])
+		out = append(out, indent+key+": "+redactedValue)
+		i++
+		// Drop folded/literal block body (PEM certificates, multi-line secrets).
+		if rest == "|" || rest == "|-" || rest == "|+" || rest == ">" || rest == ">-" || rest == ">+" || rest == "" {
+			keyIndent := len(indent)
+			for i < len(lines) {
+				next := lines[i]
+				if strings.TrimSpace(next) == "" {
+					// Blank line inside a block: keep skipping while still in block context.
+					if i+1 < len(lines) && leadingWSLen(lines[i+1]) > keyIndent {
+						i++
+						continue
+					}
+					break
+				}
+				if leadingWSLen(next) > keyIndent {
+					i++
+					continue
+				}
+				break
+			}
+		}
+	}
+	return strings.Join(out, "\n")
+}
+
+func leadingWSLen(s string) int {
+	n := 0
+	for _, r := range s {
+		if r == ' ' || r == '\t' {
+			n++
+			continue
+		}
+		break
+	}
+	return n
 }
 
 func redactJSONValue(v any) {
 	switch node := v.(type) {
 	case map[string]any:
 		for k, val := range node {
-			lowerK := strings.ToLower(k)
-			if _, ok := sensitiveJSONKeys[lowerK]; ok {
-				node[k] = "***REDACTED***"
+			if !isSensitiveRedactKey(k) {
+				redactJSONValue(val)
 				continue
 			}
-			if _, ok := issuerJSONKeys[lowerK]; ok {
-				if s, ok := val.(string); ok {
-					node[k] = stripURLUserinfo(s)
+			switch typed := val.(type) {
+			case string:
+				node[k] = redactedMarker
+			case []any:
+				redactSensitiveArray(typed)
+				node[k] = typed
+			case map[string]any:
+				// Schema/object under a sensitive key name (e.g. policy catalog): recurse, do not wipe structure.
+				redactJSONValue(typed)
+			default:
+				if typed != nil {
+					redactJSONValue(typed)
 				}
-				continue
 			}
-			redactJSONValue(val)
 		}
 	case []any:
 		for _, item := range node {
@@ -115,21 +207,15 @@ func redactJSONValue(v any) {
 	}
 }
 
-func stripURLUserinfo(raw string) string {
-	u, err := url.Parse(raw)
-	if err != nil || u.User == nil {
-		return raw
-	}
-	u.User = nil
-	return u.String()
-}
-
-func containsIssuerUserinfo(raw string) bool {
-	u, err := url.Parse(raw)
-	if err != nil {
-		return false
+func redactSensitiveArray(items []any) {
+	for i, item := range items {
+		switch typed := item.(type) {
+		case string:
+			items[i] = redactedMarker
+		default:
+			redactJSONValue(typed)
+		}
 	}
-	return u.User != nil
 }
 
 func ContainsCleartextSecret(data []byte) bool {
@@ -142,45 +228,53 @@ func ContainsCleartextSecret(data []byte) bool {
 
 func yamlHasCleartextSecret(data []byte) bool {
 	for _, line := range strings.Split(string(data), "\n") {
-		if sub := yamlSecretPattern.FindStringSubmatch(line); len(sub) == 2 {
-			val := yamlValueFromLine(sub[0])
-			if val != "" && val != "***REDACTED***" {
-				return true
-			}
+		sub := yamlSensitiveKeyPattern.FindStringSubmatch(line)
+		if sub == nil {
+			continue
 		}
-		if sub := yamlIssuerPattern.FindStringSubmatch(line); len(sub) == 3 {
-			val := strings.TrimSpace(sub[2])
-			val = strings.Trim(val, `"'`)
-			if containsIssuerUserinfo(val) {
-				return true
-			}
+		rest := strings.TrimSpace(sub[3])
+		rest = strings.Trim(rest, `"'`)
+		if rest == "" || rest == "|" || rest == "|-" || rest == "|+" || rest == ">" || rest == ">-" || rest == ">+" {
+			// Block scalar or empty value: residual PEM lines are detected below.
+			continue
+		}
+		if rest != redactedMarker {
+			return true
 		}
 	}
-	return false
-}
-
-func yamlValueFromLine(line string) string {
-	parts := strings.SplitN(line, ":", 2)
-	if len(parts) < 2 {
-		return ""
+	// Residual PEM / private-key material left after incomplete redaction.
+	lower := strings.ToLower(string(data))
+	if strings.Contains(lower, "-----begin ") && (strings.Contains(lower, " private key-----") || strings.Contains(lower, " certificate-----")) {
+		return true
 	}
-	return strings.Trim(strings.TrimSpace(parts[1]), `"'`)
+	return false
 }
 
 func jsonHasCleartextSecret(v any) bool {
 	switch node := v.(type) {
 	case map[string]any:
 		for k, val := range node {
-			lowerK := strings.ToLower(k)
-			if _, ok := sensitiveJSONKeys[lowerK]; ok {
-				if s, ok := val.(string); ok && s != "" && s != "***REDACTED***" {
-					return true
-				}
-			}
-			if _, ok := issuerJSONKeys[lowerK]; ok {
-				if s, ok := val.(string); ok && containsIssuerUserinfo(s) {
-					return true
+			if isSensitiveRedactKey(k) {
+				switch typed := val.(type) {
+				case string:
+					if typed != "" && typed != redactedMarker {
+						return true
+					}
+				case []any:
+					for _, item := range typed {
+						if s, ok := item.(string); ok && s != "" && s != redactedMarker {
+							return true
+						}
+						if jsonHasCleartextSecret(item) {
+							return true
+						}
+					}
+				default:
+					if jsonHasCleartextSecret(val) {
+						return true
+					}
 				}
+				continue
 			}
 			if jsonHasCleartextSecret(val) {
 				return true