From 3c732f2834cba21d18bf9a84d61448d8127f6f2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= Date: Wed, 24 Jun 2026 19:05:58 +0200 Subject: [PATCH 01/30] feat: transformer state export, summary and vcs triggers - pull workspace state from the TFC API instead of per-workspace init - write migration-summary.{md,json}; replace workspace.tmpl with summary.tmpl - map TFC vcs settings to SG VCSTriggers (push, PR speculative, file filters) - add SGDefault*/workspaceOverrides for vcs auth, source kind and triggers --- transformer/terraform-cloud/data.tf | 4 + .../terraform-cloud/example_payload.jsonc | 21 +- transformer/terraform-cloud/locals.tf | 245 +++++++++++++----- transformer/terraform-cloud/main.tf | 4 +- transformer/terraform-cloud/resources.tf | 87 ++++--- transformer/terraform-cloud/summary.tmpl | 52 ++++ .../terraform-cloud/terraform.tfvars.example | 48 +++- transformer/terraform-cloud/variables.tf | 40 +++ transformer/terraform-cloud/workspace.tmpl | 8 - 9 files changed, 408 insertions(+), 101 deletions(-) create mode 100644 transformer/terraform-cloud/summary.tmpl delete mode 100644 transformer/terraform-cloud/workspace.tmpl diff --git a/transformer/terraform-cloud/data.tf b/transformer/terraform-cloud/data.tf index 8ea9d91..be241df 100644 --- a/transformer/terraform-cloud/data.tf +++ b/transformer/terraform-cloud/data.tf @@ -16,4 +16,8 @@ data "tfe_variables" "data" { for_each = toset(local.workflowIds) workspace_id = each.key +} + +data "tfe_projects" "data" { + organization = var.tfOrg } \ No newline at end of file diff --git a/transformer/terraform-cloud/example_payload.jsonc b/transformer/terraform-cloud/example_payload.jsonc index d7034a7..cdc11b0 100644 --- a/transformer/terraform-cloud/example_payload.jsonc +++ b/transformer/terraform-cloud/example_payload.jsonc @@ -53,5 +53,24 @@ "useMarketplaceTemplate": false } }, - "WfType": "TERRAFORM" // could be "TERRAFORM" or "CUSTOM" + "WfType": "TERRAFORM", // could be "TERRAFORM" or "CUSTOM" + "VCSTriggers": { + // Pre-configured VCS triggers, remapped from the workspace's TFC settings. + // Omit (or null) for no triggers. NOT applied by the bulk create API — the + // importer sets it in a second pass via POST .../wfs//webhooks/vcs_triggers/ + // which registers the repo webhook. Server-assigned fields (gh_webhook_url, + // *_hook_id, github_app_installation_id) are intentionally omitted. + "type": "GITHUB_COM", // GITHUB_COM, GITLAB_COM, BITBUCKET_ORG, AZURE_DEVOPS + "tracked_branch": "main", + "approval_pre_apply": true, // inverse of TFC auto_apply + "plan_only": false, + "gh_check": true, + "gl_pipeline": true, + "post_comments": true, + "push": { "createWfRun": { "enabled": true } }, // run on push to tracked branch + "pull_request_opened": { "createWfRun": { "enabled": true } }, // TFC speculative plan + "pull_request_modified": { "createWfRun": { "enabled": true } }, + "file_triggers_enabled": true, // from TFC file_triggers_enabled + "file_trigger_patterns": ["path/to/workdir/*"] // TFC trigger_patterns / trigger_prefixes / working_directory + } } diff --git a/transformer/terraform-cloud/locals.tf b/transformer/terraform-cloud/locals.tf index 356730c..48a6ebc 100644 --- a/transformer/terraform-cloud/locals.tf +++ b/transformer/terraform-cloud/locals.tf @@ -1,73 +1,200 @@ locals { - workflowIds = [for i, v in data.tfe_workspace_ids.data.ids : v] - workflowNames = [for i, v in data.tfe_workspace_ids.data.ids : i] - workflows = [for i, v in data.tfe_workspace_ids.data.ids : { - CLIConfiguration = { - "WorkflowGroup" : { - "name" : data.tfe_workspace.data[i].project_id - }, - "TfStateFilePath" : "${abspath(path.root)}/../../${var.exportPath}/states/${data.tfe_workspace.data[i].name}.tfstate" - } - ResourceName = data.tfe_workspace.data[i].name - Description = "" - Tags = data.tfe_workspace.data[i].tag_names - EnvironmentVariables = [for i, v in data.tfe_variables.data[v].variables : - { "config" : { - "textValue" : v.value, - "varName" : v.name + # data.tfe_workspace_ids.data.ids is a map of workspace name => workspace id. + workflowIds = [for name, id in data.tfe_workspace_ids.data.ids : id] + workflowNames = [for name, id in data.tfe_workspace_ids.data.ids : name] + + # project id => project name, used to name the per-project payload files and + # to set a human-readable WorkflowGroup name in the payload. + projectNames = { for p in data.tfe_projects.data.projects : p.id => p.name } + + # SG workflow-name (ResourceName) sanitization. Per the SG OpenAPI spec, + # ResourceName must be 1-100 chars; SG's name convention is ^[-a-zA-Z0-9_]+$. + # TFC workspace names already satisfy both, so for normal inputs this is a + # no-op; the steps below defensively guarantee a valid, unique name and the + # summary reports any workspace that was actually renamed. + # 1. replace any disallowed character with "-" + nameCleaned = { for name in local.workflowNames : name => replace(name, "/[^-a-zA-Z0-9_]/", "-") } + # 2. enforce the 100-char maximum + nameTruncated = { for name, cleaned in local.nameCleaned : name => length(cleaned) > 100 ? substr(cleaned, 0, 100) : cleaned } + # 3. group originals by their sanitized name to detect collisions + sanitizedGroups = { for name, san in local.nameTruncated : san => name... } + # 4. disambiguate collisions with a short deterministic suffix (<=100 chars) + resourceNames = { + for name, san in local.nameTruncated : + name => length(local.sanitizedGroups[san]) > 1 ? "${length(san) > 93 ? substr(san, 0, 93) : san}-${substr(md5(name), 0, 6)}" : san + } + + # TFC never returns values for sensitive variables, so they cannot be + # migrated. Record them per workspace so the summary can flag them. + sensitiveVars = { + for name, id in data.tfe_workspace_ids.data.ids : + name => [for v in data.tfe_variables.data[id].variables : "${v.category}:${v.name}" if v.sensitive] + } + + # Workspaces whose terraform_version is not a pinned semver (e.g. "latest" or + # a constraint) and have no per-workspace override fall back to the default. + versionFallbacks = { + for name in local.workflowNames : + name => data.tfe_workspace.data[name].terraform_version + if !can(regex("^[0-9]+\\.[0-9]+\\.[0-9]+$", data.tfe_workspace.data[name].terraform_version)) && try(var.workspaceOverrides[name].terraformVersion, null) == null + } + + # Workspaces not using "remote" execution may not store their state in TFC, so + # the API state export can come back empty; flag them in the summary. + nonRemoteModes = { + for name in local.workflowNames : + name => data.tfe_workspace.data[name].execution_mode + if data.tfe_workspace.data[name].execution_mode != "remote" + } + + # Resolved SG VCS provider per workspace (override wins over the default). + # Drives both the source config kind and which workspaces get VCS triggers. + sourceKind = { + for name in local.workflowNames : + name => try(var.workspaceOverrides[name].sourceConfigDestKind, null) != null ? var.workspaceOverrides[name].sourceConfigDestKind : var.SGDefaultSourceConfigDestKind + } + + # One SG workflow payload per workspace. Per-workspace overrides win over the + # SGDefault* values; everything else is derived from the TFC workspace. + workflowPayload = { + for wsName, wsId in data.tfe_workspace_ids.data.ids : wsName => { + CLIConfiguration = { + "WorkflowGroup" : { + # SG workflow group per TFC project: tfc- (matches the group + # the importer creates/targets and the per-project payload filename). + "name" : "tfc-${local.projectFileSegment[data.tfe_workspace.data[wsName].project_id]}" }, - "kind" : "PLAIN_TEXT" } if v.category == "env" && v.sensitive == false] - - DeploymentPlatformConfig = var.SGDefaultDeploymentPlatformConfig - RunnerConstraints = { "type" : "shared" } - VCSConfig = { - "iacVCSConfig" : { - "useMarketplaceTemplate" : false, - "customSource" : { - "sourceConfigDestKind" : var.SGDefaultSourceConfigDestKind - "config" : { - "includeSubModule" : false, - "ref" : length(data.tfe_workspace.data[i].vcs_repo) > 0 ? data.tfe_workspace.data[i].vcs_repo[0].branch != "" ? data.tfe_workspace.data[i].vcs_repo[0].branch : "" : "", - "isPrivate" : length(data.tfe_workspace.data[i].vcs_repo) > 0 ? length(data.tfe_workspace.data[i].vcs_repo[0].oauth_token_id) > 0 || length(data.tfe_workspace.data[i].vcs_repo[0].github_app_installation_id) > 0 ? true : false : false, - "auth" : length(data.tfe_workspace.data[i].vcs_repo) > 0 ? length(data.tfe_workspace.data[i].vcs_repo[0].oauth_token_id) > 0 || length(data.tfe_workspace.data[i].vcs_repo[0].github_app_installation_id) > 0 ? var.SGDefaultVCSAuthIntegrationID : "" : "", - "workingDir" : data.tfe_workspace.data[i].working_directory, - "repo" : length(data.tfe_workspace.data[i].vcs_repo) > 0 ? format("%s/%s", var.SGDefaultIACVCSRepoPrefix, data.tfe_workspace.data[i].vcs_repo[0].identifier) : "" + "TfStateFilePath" : "${abspath(path.root)}/../../${var.exportPath}/states/${data.tfe_workspace.data[wsName].name}.tfstate" + } + ResourceName = local.resourceNames[wsName] + Description = "" + Tags = data.tfe_workspace.data[wsName].tag_names + EnvironmentVariables = concat( + [for v in data.tfe_variables.data[wsId].variables : + { "config" : { "textValue" : v.value, "varName" : v.name }, "kind" : "PLAIN_TEXT" } + if v.category == "env" && v.sensitive == false], + try(var.workspaceOverrides[wsName].extraEnvironmentVariables, []) + ) + + DeploymentPlatformConfig = try(var.workspaceOverrides[wsName].DeploymentPlatformConfig, null) != null ? var.workspaceOverrides[wsName].DeploymentPlatformConfig : var.SGDefaultDeploymentPlatformConfig + RunnerConstraints = try(var.workspaceOverrides[wsName].RunnerConstraints, null) != null ? var.workspaceOverrides[wsName].RunnerConstraints : { "type" : "shared" } + + VCSConfig = { + "iacVCSConfig" : { + "useMarketplaceTemplate" : false, + "customSource" : { + "sourceConfigDestKind" : local.sourceKind[wsName] + "config" : { + "includeSubModule" : false, + "ref" : length(data.tfe_workspace.data[wsName].vcs_repo) > 0 ? data.tfe_workspace.data[wsName].vcs_repo[0].branch : "", + "isPrivate" : length(data.tfe_workspace.data[wsName].vcs_repo) > 0 ? length(data.tfe_workspace.data[wsName].vcs_repo[0].oauth_token_id) > 0 || length(data.tfe_workspace.data[wsName].vcs_repo[0].github_app_installation_id) > 0 : false, + "auth" : length(data.tfe_workspace.data[wsName].vcs_repo) > 0 ? (length(data.tfe_workspace.data[wsName].vcs_repo[0].oauth_token_id) > 0 || length(data.tfe_workspace.data[wsName].vcs_repo[0].github_app_installation_id) > 0 ? (try(var.workspaceOverrides[wsName].vcsAuthIntegrationID, null) != null ? var.workspaceOverrides[wsName].vcsAuthIntegrationID : var.SGDefaultVCSAuthIntegrationID) : "") : "", + "workingDir" : data.tfe_workspace.data[wsName].working_directory, + "repo" : length(data.tfe_workspace.data[wsName].vcs_repo) > 0 ? format("%s/%s", try(var.workspaceOverrides[wsName].vcsRepoPrefix, null) != null ? var.workspaceOverrides[wsName].vcsRepoPrefix : var.SGDefaultIACVCSRepoPrefix, data.tfe_workspace.data[wsName].vcs_repo[0].identifier) : "" + } } + }, + "iacInputData" : { + "schemaType" : "RAW_JSON", + "data" : { for v in data.tfe_variables.data[wsId].variables : v.name => try(jsondecode(v.value), v.value) if v.category == "terraform" && v.sensitive == false } } - }, - "iacInputData" : { - "schemaType" : "RAW_JSON", - "data" : { for i, v in data.tfe_variables.data[v].variables : v.name => try(jsondecode(v.value), v.value) if v.category == "terraform" } } - } - MiniSteps = { - "wfChaining" : { - "ERRORED" : [], - "COMPLETED" : [] - }, - "notifications" : { - "email" : { + # VCS triggers, remapped 1:1 from the workspace's own TFC VCS settings. + # Only emitted for VCS-backed workspaces on a provider SG can webhook + # (GIT_OTHER has no webhook support); null otherwise. This block is NOT + # accepted by the bulk workflow-create API — the importer applies it in a + # second pass via POST .../wfs//webhooks/vcs_triggers/ (see migrate.sh + # cmd_triggers), which is what actually registers the repo webhook. Field + # shape matches a real SG workflow + the landfast set_vcs_triggers call; + # only the truly server-assigned fields (gh_webhook_url, *_hook_id, + # github_app_installation_id) are omitted. + VCSTriggers = try(var.workspaceOverrides[wsName].VCSTriggers, null) != null ? var.workspaceOverrides[wsName].VCSTriggers : ( + var.SGDefaultEnableVCSTriggers && + length(data.tfe_workspace.data[wsName].vcs_repo) > 0 && + contains(["GITHUB_COM", "GITLAB_COM", "BITBUCKET_ORG", "AZURE_DEVOPS"], local.sourceKind[wsName]) + ? { + "type" : local.sourceKind[wsName], + "tracked_branch" : data.tfe_workspace.data[wsName].vcs_repo[0].branch, + "approval_pre_apply" : !data.tfe_workspace.data[wsName].auto_apply, + "plan_only" : false, + "gh_check" : true, + "gl_pipeline" : true, + "post_comments" : true, + # TFC runs on every push to the tracked branch when VCS-connected. + "push" : { "createWfRun" : { "enabled" : true } }, + # TFC speculative plans on PRs map to the PR-open/update triggers. + "pull_request_opened" : { "createWfRun" : { "enabled" : data.tfe_workspace.data[wsName].speculative_enabled } }, + "pull_request_modified" : { "createWfRun" : { "enabled" : data.tfe_workspace.data[wsName].speculative_enabled } }, + "file_triggers_enabled" : data.tfe_workspace.data[wsName].file_triggers_enabled, + # Prefer TFC's glob trigger_patterns; fall back to legacy + # trigger_prefixes (prefix -> "/*"); else, when file triggers + # are on with a working dir, scope to that dir like TFC does. + "file_trigger_patterns" : ( + try(length(data.tfe_workspace.data[wsName].trigger_patterns), 0) > 0 ? data.tfe_workspace.data[wsName].trigger_patterns : + try(length(data.tfe_workspace.data[wsName].trigger_prefixes), 0) > 0 ? [for p in data.tfe_workspace.data[wsName].trigger_prefixes : "${p}/*"] : + data.tfe_workspace.data[wsName].file_triggers_enabled && data.tfe_workspace.data[wsName].working_directory != "" ? ["${data.tfe_workspace.data[wsName].working_directory}/*"] : [] + ) + } + : null + ) + + MiniSteps = { + "wfChaining" : { "ERRORED" : [], - "COMPLETED" : [], - "APPROVAL_REQUIRED" : [], - "CANCELLED" : [] + "COMPLETED" : [] + }, + "notifications" : { + "email" : { + "ERRORED" : [], + "COMPLETED" : [], + "APPROVAL_REQUIRED" : [], + "CANCELLED" : [] + } } } - } - Approvers = data.tfe_workspace.data[i].auto_apply == true ? [] : var.SGDefaultWfApprovers + Approvers = try(var.workspaceOverrides[wsName].Approvers, null) != null ? var.workspaceOverrides[wsName].Approvers : (data.tfe_workspace.data[wsName].auto_apply ? [] : var.SGDefaultWfApprovers) - TerraformConfig = { - "managedTerraformState" : true, - "terraformVersion" : "TERRAFORM-${data.tfe_workspace.data[i].terraform_version}", - "approvalPreApply" : !data.tfe_workspace.data[i].auto_apply + TerraformConfig = { + "managedTerraformState" : true, + "terraformVersion" : ( + try(var.workspaceOverrides[wsName].terraformVersion, null) != null ? var.workspaceOverrides[wsName].terraformVersion : + can(regex("^[0-9]+\\.[0-9]+\\.[0-9]+$", data.tfe_workspace.data[wsName].terraform_version)) ? "TERRAFORM-${data.tfe_workspace.data[wsName].terraform_version}" : var.SGDefaultTerraformVersion + ), + "approvalPreApply" : !data.tfe_workspace.data[wsName].auto_apply + } + + WfType = "TERRAFORM" + UserSchedules = [] } + } + + # Group payloads by TFC project so each project imports into its own SG + # workflow group (the bulk import takes a single --workflow-group per file). + workflowProject = { for wsName, wsId in data.tfe_workspace_ids.data.ids : wsName => data.tfe_workspace.data[wsName].project_id } + projectsUsed = toset(values(local.workflowProject)) + + payloadByProject = { + for pid in local.projectsUsed : + pid => [for name, payload in local.workflowPayload : payload if local.workflowProject[name] == pid] + } + + # project id => filesystem-safe segment for the per-project payload filename. + projectFileSegment = { + for pid in local.projectsUsed : + pid => replace(lower(try(local.projectNames[pid], pid)), "/[^a-z0-9-]+/", "-") + } - WfType = "TERRAFORM" - UserSchedules = [] - }] - data = jsonencode( - local.workflows - ) + # Machine-readable migration summary (also rendered to markdown). + summary = { + organization = var.tfOrg + workspaceCount = length(local.workflowNames) + projectWorkspaceCounts = { for pid in local.projectsUsed : try(local.projectNames[pid], pid) => length(local.payloadByProject[pid]) } + skippedSensitiveVars = { for name, vars in local.sensitiveVars : name => vars if length(vars) > 0 } + terraformVersionFallbacks = local.versionFallbacks + nonRemoteExecutionModes = local.nonRemoteModes + renamedWorkspaces = { for name in local.workflowNames : name => local.resourceNames[name] if local.resourceNames[name] != name } + variableSetsReminder = "TFC Variable Set variables are merged by the 'enrich' step (non-sensitive only). Sensitive set vars can't be read from the API — recreate them as SG secrets; see the enrich step output." + } } diff --git a/transformer/terraform-cloud/main.tf b/transformer/terraform-cloud/main.tf index 5ae44be..9a19179 100644 --- a/transformer/terraform-cloud/main.tf +++ b/transformer/terraform-cloud/main.tf @@ -1,5 +1,5 @@ terraform { - required_version = "~> 1.2" + required_version = ">= 1.3" required_providers { local = { @@ -8,7 +8,7 @@ terraform { } tfe = { source = "hashicorp/tfe" - version = "~> 0.48.0" + version = "~> 0.78" } null = { source = "hashicorp/null" diff --git a/transformer/terraform-cloud/resources.tf b/transformer/terraform-cloud/resources.tf index d53c3d5..fd58be0 100644 --- a/transformer/terraform-cloud/resources.tf +++ b/transformer/terraform-cloud/resources.tf @@ -1,40 +1,67 @@ +# One payload file per TFC project. Written directly (no -generated + mv dance) +# so re-applies always refresh the output. resource "local_file" "data" { - content = local.data - filename = "${path.module}/../../${var.exportPath}/sg-payload-generated.json" - provisioner "local-exec" { - command = "mv ${path.module}/../../${var.exportPath}/sg-payload-generated.json ${path.module}/../../${var.exportPath}/sg-payload.json" - } -} + for_each = local.payloadByProject -resource "local_file" "generateTempTfFiles" { - for_each = var.exportStateFiles ? toset(local.workflowNames) : [] - - content = templatefile("${path.module}/workspace.tmpl", { tfOrg = var.tfOrg, workspace = each.key }) - filename = "${path.module}/../../${var.exportPath}/tfDir/${each.key}/main.tf" + content = jsonencode(each.value) + filename = "${path.module}/../../${var.exportPath}/sg-payload.${local.projectFileSegment[each.key]}.json" } -resource "null_resource" "exportStateFiles" { - depends_on = [local_file.generateTempTfFiles] - triggers = { - always-update = timestamp() - } - for_each = var.exportStateFiles ? toset(local.workflowNames) : [] +resource "local_file" "summary" { + content = jsonencode(local.summary) + filename = "${path.module}/../../${var.exportPath}/migration-summary.json" +} - provisioner "local-exec" { - command = "mkdir -p ../../states && rm -rf .terraform .terraform.lock.hcl terraform.tfstate terraform.tfstate.backup && terraform init -input=false && terraform state pull > ../../states/'${each.key}.tfstate'" - working_dir = "${path.module}/../../${var.exportPath}/tfDir/${each.key}" - } +resource "local_file" "summaryMd" { + content = templatefile("${path.module}/summary.tmpl", { summary = local.summary }) + filename = "${path.module}/../../${var.exportPath}/migration-summary.md" } -resource "null_resource" "deleteTempTfFiles" { - count = var.exportStateFiles ? 1 : 0 - triggers = { - always-update = timestamp() - } - depends_on = [null_resource.exportStateFiles] +# Pull each workspace's current state directly from the TFC/TFE API — no +# `terraform init` or providers (avoids the plugin-cache concurrency issue and +# per-workspace provider downloads). The token is read at runtime from the +# `terraform login` credentials file or TFE_TOKEN, so it never enters TF state. +# Requires `curl` and `jq` on PATH (provided by the Docker image / orchestrator). +resource "null_resource" "exportState" { + for_each = var.exportStateFiles ? data.tfe_workspace_ids.data.ids : {} + + # Idempotent by default (keyed by stable workspace name/id); forceStateRefresh + # re-pulls every workspace. + triggers = merge( + { workspace = each.key, id = each.value }, + var.forceStateRefresh ? { refresh = timestamp() } : {} + ) provisioner "local-exec" { - command = "rm -rf tfDir" - working_dir = "${path.module}/../../${var.exportPath}/" + interpreter = ["/bin/bash", "-c"] + # SG_EXPORT is absolute and created in-command, so it works regardless of the + # provisioner's working directory or resource ordering. + environment = { + SG_TFC_HOST = var.tfHostname + SG_WS_ID = each.value + SG_WS_NAME = each.key + SG_EXPORT = "${abspath(path.module)}/../../${var.exportPath}" + } + # Failure isolation: missing token / no-state / download error is recorded + # in state-export-failures.log instead of aborting the whole apply. + command = <<-EOT + set -uo pipefail + mkdir -p "$SG_EXPORT/states" + creds="$HOME/.terraform.d/credentials.tfrc.json" + token="" + if [ -f "$creds" ] && command -v jq >/dev/null 2>&1; then + token="$(jq -r --arg h "$SG_TFC_HOST" '.credentials[$h].token // empty' "$creds" 2>/dev/null || true)" + fi + [ -z "$token" ] && token="$${TFE_TOKEN:-}" + if [ -z "$token" ]; then + echo "$SG_WS_NAME: no TFC token (set TFE_TOKEN or run 'terraform login')" >> "$SG_EXPORT/state-export-failures.log"; exit 0 + fi + url="$(curl -fsS -H "Authorization: Bearer $token" "https://$SG_TFC_HOST/api/v2/workspaces/$SG_WS_ID/current-state-version" | jq -r '.data.attributes."hosted-state-download-url" // empty' 2>/dev/null || true)" + if [ -z "$url" ]; then + echo "$SG_WS_NAME: no current state version" >> "$SG_EXPORT/state-export-failures.log"; exit 0 + fi + curl -fsSL -H "Authorization: Bearer $token" "$url" -o "$SG_EXPORT/states/$SG_WS_NAME.tfstate" \ + || echo "$SG_WS_NAME: state download failed" >> "$SG_EXPORT/state-export-failures.log" + EOT } -} \ No newline at end of file +} diff --git a/transformer/terraform-cloud/summary.tmpl b/transformer/terraform-cloud/summary.tmpl new file mode 100644 index 0000000..8dc1ba4 --- /dev/null +++ b/transformer/terraform-cloud/summary.tmpl @@ -0,0 +1,52 @@ +# StackGuardian migration summary + +- Organization: ${summary.organization} +- Workspaces processed: ${summary.workspaceCount} + +## Workflows per project +%{ for project, count in summary.projectWorkspaceCounts ~} +- ${project}: ${count} +%{ endfor ~} + +## Skipped sensitive variables +These are not migrated (TFC never returns sensitive values). Recreate them as SG secrets after import. +%{ if length(summary.skippedSensitiveVars) == 0 ~} +- None. +%{ else ~} +%{ for ws, vars in summary.skippedSensitiveVars ~} +- ${ws}: ${join(", ", vars)} +%{ endfor ~} +%{ endif ~} + +## Terraform version fallbacks +Workspaces whose version was not a pinned semver; the configured SGDefaultTerraformVersion was used. +%{ if length(summary.terraformVersionFallbacks) == 0 ~} +- None. +%{ else ~} +%{ for ws, ver in summary.terraformVersionFallbacks ~} +- ${ws}: reported "${ver}" +%{ endfor ~} +%{ endif ~} + +## Non-remote execution modes +TFC may not hold state for these (local/agent execution); state export can be empty. +%{ if length(summary.nonRemoteExecutionModes) == 0 ~} +- None. +%{ else ~} +%{ for ws, mode in summary.nonRemoteExecutionModes ~} +- ${ws}: ${mode} +%{ endfor ~} +%{ endif ~} + +## Renamed workspaces +%{ if length(summary.renamedWorkspaces) == 0 ~} +- None. +%{ else ~} +%{ for orig, new in summary.renamedWorkspaces ~} +- ${orig} -> ${new} +%{ endfor ~} +%{ endif ~} + +## Reminders +- ${summary.variableSetsReminder} +- If state export ran, check state-export-failures.log in the export directory for workspaces whose state could not be pulled. diff --git a/transformer/terraform-cloud/terraform.tfvars.example b/transformer/terraform-cloud/terraform.tfvars.example index 66bf280..9454029 100644 --- a/transformer/terraform-cloud/terraform.tfvars.example +++ b/transformer/terraform-cloud/terraform.tfvars.example @@ -20,7 +20,7 @@ exportPath = "export" SGDefaultWfApprovers = [] # Prefix for your repo URL -SGDefaultIACVCSRepoPrefix = "https://www.github.com" +SGDefaultIACVCSRepoPrefix = "https://github.com" # Provide an integration id like /integrations/aws-dev-account or /secrets/my-git-token SGDefaultVCSAuthIntegrationID = "/integrations/github_com" @@ -38,3 +38,49 @@ SGDefaultDeploymentPlatformConfig = [ # Choose from: GITHUB_COM, BITBUCKET_ORG, GITLAB_COM, AZURE_DEVOPS, GIT_OTHER SGDefaultSourceConfigDestKind = "GITHUB_COM" + +# SG Terraform version used when a workspace's terraform_version is not a pinned +# semver (e.g. "latest" or a constraint), or runs an engine SG cannot map. +SGDefaultTerraformVersion = "TERRAFORM-1.5.7" + +# Pre-configure VCS triggers on each VCS-backed workflow, remapped from the +# workspace's own TFC settings (tracked branch, push, PR speculative plans, file +# triggers). Set false to import workflows without any triggers. +SGDefaultEnableVCSTriggers = true + +# Re-pull state for every workspace on each apply. Leave false for idempotent runs. +forceStateRefresh = false + +# Per-workspace overrides, keyed by workspace name. Any field set here wins over +# the SGDefault* value above, for that workspace only. All fields are optional. +# workspaceOverrides = { +# "prod-networking" = { +# DeploymentPlatformConfig = [ +# { +# "kind" : "AWS_RBAC", +# "config" : { "integrationId" : "/integrations/aws-prod", "profileName" : "default" } +# } +# ] +# RunnerConstraints = { "type" : "private", "names" : ["sg-runner"] } +# Approvers = ["lead@example.com"] +# vcsAuthIntegrationID = "/integrations/github_prod" +# vcsRepoPrefix = "https://www.github.com" +# sourceConfigDestKind = "GITHUB_COM" +# terraformVersion = "TERRAFORM-1.7.5" +# extraEnvironmentVariables = [ +# { "config" : { "textValue" : "us-east-1", "varName" : "AWS_REGION" }, "kind" : "PLAIN_TEXT" } +# ] +# # Full override of the derived VCS triggers (replaces the remap entirely). +# VCSTriggers = { +# type = "GITHUB_COM" +# tracked_branch = "main" +# approval_pre_apply = true +# plan_only = false +# push = { createWfRun = { enabled = true } } +# pull_request_opened = { createWfRun = { enabled = true } } +# pull_request_modified = { createWfRun = { enabled = true } } +# file_triggers_enabled = true +# file_trigger_patterns = ["modules/networking/*"] +# } +# } +# } diff --git a/transformer/terraform-cloud/variables.tf b/transformer/terraform-cloud/variables.tf index 3e15899..52c6df6 100644 --- a/transformer/terraform-cloud/variables.tf +++ b/transformer/terraform-cloud/variables.tf @@ -15,6 +15,12 @@ variable "exportStateFiles" { type = bool } +variable "tfHostname" { + default = "app.terraform.io" + description = "TFC/TFE hostname used for the state-export API and the `terraform login` credential lookup." + type = string +} + variable "tfWorkspaceTags" { default = null description = "List of TFC/TFE workspace tags to include when exporting. Excluded tags take precedence over included ones. Wildcards are not supported." @@ -69,4 +75,38 @@ variable "SGDefaultSourceConfigDestKind" { default = "GIT_OTHER" description = "Choose from: GITHUB_COM, BITBUCKET_ORG, GITLAB_COM, AZURE_DEVOPS, GIT_OTHER" type = string +} + +variable "SGDefaultTerraformVersion" { + default = "TERRAFORM-1.5.7" + description = "SG Terraform version used when a workspace's terraform_version is not a pinned semver (e.g. 'latest' or a version constraint), or the workspace runs an engine SG cannot map. Use the SG-formatted value, e.g. TERRAFORM-1.5.7." + type = string +} + +variable "SGDefaultEnableVCSTriggers" { + default = true + description = "Pre-configure VCS triggers on each VCS-backed workflow, remapped from the workspace's own TFC settings (tracked branch, push, PR speculative plans, file triggers). Set false to import workflows without any triggers. Per-workspace overrides via workspaceOverrides[name].VCSTriggers." + type = bool +} + +variable "forceStateRefresh" { + default = false + description = "Re-pull Terraform state for every workspace on each apply. When false (default), state export is idempotent and only runs for workspaces it has not exported before." + type = bool +} + +variable "workspaceOverrides" { + default = {} + description = "Per-workspace overrides keyed by TFC/TFE workspace name. Any field set here takes precedence over the matching SGDefault* value for that workspace only." + type = map(object({ + DeploymentPlatformConfig = optional(list(any)) + RunnerConstraints = optional(any) + Approvers = optional(list(string)) + vcsAuthIntegrationID = optional(string) + vcsRepoPrefix = optional(string) + sourceConfigDestKind = optional(string) + terraformVersion = optional(string) + extraEnvironmentVariables = optional(list(any), []) + VCSTriggers = optional(any) + })) } \ No newline at end of file diff --git a/transformer/terraform-cloud/workspace.tmpl b/transformer/terraform-cloud/workspace.tmpl deleted file mode 100644 index 9aa95f4..0000000 --- a/transformer/terraform-cloud/workspace.tmpl +++ /dev/null @@ -1,8 +0,0 @@ -terraform { - cloud { - organization = "${tfOrg}" - workspaces { - name = "${workspace}" - } - } -} \ No newline at end of file From a0c0b177ec685945f43f88ee7a357e5f858fb9db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= Date: Wed, 24 Jun 2026 19:06:05 +0200 Subject: [PATCH 02/30] feat: payload json schema and validator Validate generated payloads against schema/sg-payload.schema.json via yajsv. --- schema/sg-payload.schema.json | 118 ++++++++++++++++++++++++++++++++++ scripts/validate_payload.sh | 29 +++++++++ 2 files changed, 147 insertions(+) create mode 100644 schema/sg-payload.schema.json create mode 100755 scripts/validate_payload.sh diff --git a/schema/sg-payload.schema.json b/schema/sg-payload.schema.json new file mode 100644 index 0000000..c507be2 --- /dev/null +++ b/schema/sg-payload.schema.json @@ -0,0 +1,118 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "StackGuardian bulk workflow payload", + "description": "Validates the array of workflow objects produced by the transformer (sg-payload..json). Constraints (ResourceName length, kind/sourceConfigDestKind enums) are derived from schema/sg-openapi.json (#/components/schemas/Workflow and related). Lenient by design: only the fields the transformer emits are checked; unknown fields are allowed so the schema does not need to track every optional API field.", + "type": "array", + "items": { "$ref": "#/definitions/workflow" }, + "definitions": { + "workflow": { + "type": "object", + "required": ["ResourceName", "WfType", "VCSConfig"], + "properties": { + "ResourceName": { "type": "string", "minLength": 1, "maxLength": 100 }, + "WfType": { "type": "string", "minLength": 1 }, + "Description": { "type": "string" }, + "Tags": { "type": "array", "items": { "type": "string" } }, + "Approvers": { "type": "array", "items": { "type": "string" } }, + "UserSchedules": { "type": "array" }, + "EnvironmentVariables": { + "type": "array", + "items": { + "type": "object", + "required": ["kind", "config"], + "properties": { + "kind": { "type": "string" }, + "config": { + "type": "object", + "required": ["varName"], + "properties": { + "varName": { "type": "string", "minLength": 1 }, + "textValue": { "type": "string" } + } + } + } + } + }, + "DeploymentPlatformConfig": { + "type": "array", + "items": { + "type": "object", + "required": ["kind", "config"], + "properties": { + "kind": { + "enum": ["AWS_STATIC", "AWS_RBAC", "AWS_OIDC", "AZURE_STATIC", "AZURE_OIDC", "AZURE_MANAGED_ID_OIDC", "GCP_STATIC", "GCP_OIDC"] + }, + "config": { "type": "object" } + } + } + }, + "RunnerConstraints": { + "type": "object", + "required": ["type"], + "properties": { + "type": { "type": "string" }, + "names": { "type": "array", "items": { "type": "string" } } + } + }, + "VCSConfig": { + "type": "object", + "required": ["iacVCSConfig", "iacInputData"], + "properties": { + "iacVCSConfig": { + "type": "object", + "properties": { + "customSource": { + "type": "object", + "properties": { + "sourceConfigDestKind": { + "enum": ["GITHUB_COM", "GITHUB_APP_CUSTOM", "GIT_OTHER", "INLINE", "BITBUCKET_ORG", "GITLAB_COM", "AZURE_DEVOPS"] + } + } + } + } + }, + "iacInputData": { + "type": "object", + "required": ["schemaType", "data"], + "properties": { + "schemaType": { "type": "string" }, + "data": { "type": "object" } + } + } + } + }, + "TerraformConfig": { + "type": "object", + "properties": { + "terraformVersion": { "type": "string", "minLength": 1 }, + "managedTerraformState": { "type": "boolean" }, + "approvalPreApply": { "type": "boolean" } + } + }, + "VCSTriggers": { + "type": ["object", "null"], + "required": ["type"], + "properties": { + "type": { + "enum": ["GITHUB_COM", "GITHUB_APP_CUSTOM", "GITLAB_OAUTH_SSH", "BITBUCKET_ORG", "GITLAB_COM", "AZURE_DEVOPS", "AZURE_DEVOPS_SP"] + }, + "tracked_branch": { "type": ["string", "null"] }, + "approval_pre_apply": { "type": "boolean" }, + "plan_only": { "type": "boolean" }, + "gh_check": { "type": "boolean" }, + "gl_pipeline": { "type": "boolean" }, + "post_comments": { "type": "boolean" }, + "file_triggers_enabled": { "type": "boolean" }, + "file_trigger_patterns": { "type": "array", "items": { "type": "string" } }, + "tags_regex": { "type": ["string", "null"] }, + "push": { "type": "object" }, + "pull_request_opened": { "type": "object" }, + "pull_request_modified": { "type": "object" }, + "all_pull_requests": { "type": "object" }, + "create_tag": { "type": "object" } + } + } + } + } + } +} diff --git a/scripts/validate_payload.sh b/scripts/validate_payload.sh new file mode 100755 index 0000000..234638e --- /dev/null +++ b/scripts/validate_payload.sh @@ -0,0 +1,29 @@ +#!/bin/bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=tools.sh +source "$SCRIPT_DIR/tools.sh" + +SCHEMA="$SG_REPO_ROOT/schema/sg-payload.schema.json" + +if [ "$#" -eq 0 ]; then + echo "Usage: $0 [more.json ...]" >&2 + echo " e.g. $0 export/sg-payload.*.json" >&2 + exit 1 +fi +if [ ! -f "$SCHEMA" ]; then + echo "Schema not found: $SCHEMA" >&2 + exit 1 +fi + +# Resolve yajsv from PATH (Docker image) or download+cache (native). +YAJSV_BIN=$(sg_resolve yajsv sg_ensure_yajsv) + +sg_log "validating $# file(s) against schema/sg-payload.schema.json" +# yajsv prints ": valid" per file and exits non-zero if any file fails. +# Strip the repo-root prefix from its output for readable, relative paths +# (pipefail off so the pipeline's status is sed's; yajsv's status via PIPESTATUS). +set +o pipefail +"$YAJSV_BIN" -s "$SCHEMA" "$@" 2>&1 | sed "s#${SG_REPO_ROOT}/##g" +exit "${PIPESTATUS[0]}" From e2ec6840e384a8704f382ecac6a76cf617f1a0c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= Date: Wed, 24 Jun 2026 19:06:13 +0200 Subject: [PATCH 03/30] feat: dockerized migration orchestrator - sg-migrate.sh + Dockerfile run the full pipeline inside a container - scripts/migrate.sh: init/apply/enrich/convert/validate/import/triggers/all - merge TFC variable sets, parallel convert/import, register VCS triggers - move convert_hcl_to_json.sh into scripts/ --- .dockerignore | 9 + .gitignore | 3 + Dockerfile | 53 +++ .../convert_hcl_to_json.sh | 64 +-- scripts/enrich_variable_sets.sh | 166 +++++++ scripts/migrate.sh | 443 ++++++++++++++++++ scripts/tools.sh | 162 +++++++ sg-migrate.sh | 69 +++ 8 files changed, 920 insertions(+), 49 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile rename convert_hcl_to_json.sh => scripts/convert_hcl_to_json.sh (66%) create mode 100755 scripts/enrich_variable_sets.sh create mode 100755 scripts/migrate.sh create mode 100755 scripts/tools.sh create mode 100755 sg-migrate.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..51be1ac --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +.git +.sg +export +out +**/.terraform +**/.terraform.lock.hcl +*.tfstate +*.tfstate.* +*.tfvars diff --git a/.gitignore b/.gitignore index 1d638cc..953654c 100644 --- a/.gitignore +++ b/.gitignore @@ -163,4 +163,7 @@ out/* zip zip/* +# Migrator local cache + config (downloaded tool binaries, workflow-group map) +.sg/ + .DS_Store \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..a8668cf --- /dev/null +++ b/Dockerfile @@ -0,0 +1,53 @@ +# Migrator runtime: bundles all pinned tooling so the flow behaves identically +# on Linux/macOS/Windows hosts (anywhere Docker runs). The repo is bind-mounted +# at /app at runtime; this image only provides the tools on PATH. + +# yajsv has no linux/arm64 release asset, so build it from source for the +# image's target architecture. +FROM golang:1.22-bookworm AS yajsv +ARG YAJSV_VERSION=v1.4.1 +RUN go install "github.com/neilpa/yajsv@${YAJSV_VERSION}" + +FROM debian:bookworm-slim + +ARG TERRAFORM_VERSION=1.9.8 +ARG JQ_VERSION=1.8.1 +ARG HCL2JSON_VERSION=0.6.7 + +RUN apt-get update && apt-get install -y --no-install-recommends \ + bash curl ca-certificates git unzip tar coreutils \ + && rm -rf /var/lib/apt/lists/* + +# terraform (arch from dpkg: amd64/arm64 — works with or without buildx) +RUN arch="$(dpkg --print-architecture)" \ + && curl -fsSL "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_${arch}.zip" -o /tmp/tf.zip \ + && unzip /tmp/tf.zip -d /usr/local/bin \ + && rm /tmp/tf.zip + +# jq +RUN arch="$(dpkg --print-architecture)" \ + && curl -fsSL "https://github.com/jqlang/jq/releases/download/jq-${JQ_VERSION}/jq-linux-${arch}" -o /usr/local/bin/jq \ + && chmod +x /usr/local/bin/jq + +# hcl2json +RUN arch="$(dpkg --print-architecture)" \ + && curl -fsSL "https://github.com/tmccombs/hcl2json/releases/download/v${HCL2JSON_VERSION}/hcl2json_linux_${arch}" -o /usr/local/bin/hcl2json \ + && chmod +x /usr/local/bin/hcl2json + +# yajsv (from the build stage above) +COPY --from=yajsv /go/bin/yajsv /usr/local/bin/yajsv + +# sg-cli (latest release, Go binary). Assets: sg-cli__.tar.gz. +RUN set -eu; \ + arch="$(dpkg --print-architecture)"; \ + case "$arch" in amd64) arch=x86_64 ;; arm64) arch=arm64 ;; esac; \ + curl -fsSL "https://github.com/StackGuardian/sg-cli/releases/latest/download/sg-cli_Linux_${arch}.tar.gz" -o /tmp/sg-cli.tar.gz; \ + mkdir -p /tmp/sgcli; \ + tar -xzf /tmp/sg-cli.tar.gz -C /tmp/sgcli; \ + realcli="$(find /tmp/sgcli -maxdepth 2 -type f -name sg-cli | head -1)"; \ + test -n "$realcli"; \ + install -m 0755 "$realcli" /usr/local/bin/sg-cli; \ + rm -rf /tmp/sg-cli.tar.gz /tmp/sgcli + +WORKDIR /app +ENTRYPOINT ["/bin/bash"] diff --git a/convert_hcl_to_json.sh b/scripts/convert_hcl_to_json.sh similarity index 66% rename from convert_hcl_to_json.sh rename to scripts/convert_hcl_to_json.sh index beea2f2..0446805 100755 --- a/convert_hcl_to_json.sh +++ b/scripts/convert_hcl_to_json.sh @@ -1,50 +1,12 @@ #!/bin/bash set -euo pipefail -log() { echo "[convert_hcl_to_json] $*" >&2; } +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=tools.sh +source "$SCRIPT_DIR/tools.sh" -WORKDIR=$(mktemp -d) -cleanup() { rm -rf "$WORKDIR"; } -trap cleanup EXIT - -# Normalize OS/arch to the names used by the jq and hcl2json release assets. -OS=$(uname -s) -case "$OS" in - Darwin) OS="macos" ;; - Linux) OS="linux" ;; - *) echo "Unsupported OS: $OS" >&2; exit 1 ;; -esac - -ARCH=$(uname -m) -case "$ARCH" in - x86_64 | amd64) ARCH="amd64" ;; - aarch64 | arm64) ARCH="arm64" ;; - *) echo "Unsupported architecture: $ARCH" >&2; exit 1 ;; -esac - -JQ_BIN="$WORKDIR/jq" -HCL2JSON_BIN="$WORKDIR/hcl2json" - -install_jq() { - local url="https://github.com/jqlang/jq/releases/download/jq-1.8.1/jq-${OS}-${ARCH}" - if ! curl -fsSL -o "$JQ_BIN" "$url"; then - echo "Failed to download jq from $url" >&2 - exit 1 - fi - chmod +x "$JQ_BIN" -} - -install_hcl2json() { - # hcl2json uses "darwin" rather than "macos" for the OS segment. - local hcl_os="$OS" - [[ "$hcl_os" == "macos" ]] && hcl_os="darwin" - local url="https://github.com/tmccombs/hcl2json/releases/download/v0.6.7/hcl2json_${hcl_os}_${ARCH}" - if ! curl -fsSL -o "$HCL2JSON_BIN" "$url"; then - echo "Failed to download hcl2json from $url" >&2 - exit 1 - fi - chmod +x "$HCL2JSON_BIN" -} +# Detail lines are shown only in verbose mode; warnings always show. +log() { [ "${SG_VERBOSE:-0}" = "1" ] || return 0; printf '%s[convert]%s %s\n' "$C_CYAN" "$C_RESET" "$*" >&2; } INPUT_FILE_JSON="${1:-}" if [ -z "$INPUT_FILE_JSON" ]; then @@ -56,16 +18,20 @@ if [ ! -f "$INPUT_FILE_JSON" ]; then exit 1 fi -log "Downloading jq and hcl2json..." -install_jq -install_hcl2json +WORKDIR=$(mktemp -d) +cleanup() { rm -rf "$WORKDIR"; } +trap cleanup EXIT + +# Resolve tooling from PATH (Docker image) or download+cache (native). +JQ_BIN=$(sg_resolve jq sg_ensure_jq) +HCL2JSON_BIN=$(sg_resolve hcl2json sg_ensure_hcl2json) # Read entire JSON array into a variable json_data=$(cat "$INPUT_FILE_JSON") # Use jq to get the length of array length=$($JQ_BIN length <<<"$json_data") -log "Processing $length workflow(s) from $INPUT_FILE_JSON" +log "Processing $length workflow(s) from $(sg_rel "$INPUT_FILE_JSON")" # Accumulate updated objects as newline-delimited JSON tmpfile="$WORKDIR/updated.ndjson" @@ -118,7 +84,7 @@ for ((i = 0; i < length; i++)); do log " workflow $((i + 1)): converted '$key' from HCL to JSON" new_val=$($JQ_BIN --arg k "$key" --argjson v "$parsed" '. + {($k): $v}' <<<"$new_val") else - log " workflow $((i + 1)): parsing failed, keeping original value for '$key'" + sg_warn "$(sg_rel "$INPUT_FILE_JSON") workflow $((i + 1)): could not parse '$key' as HCL; keeping original value" fi done < <($JQ_BIN -r 'keys[]' <<<"$val") @@ -133,4 +99,4 @@ done outfile="$WORKDIR/output.json" $JQ_BIN -s '.' "$tmpfile" >"$outfile" mv "$outfile" "$INPUT_FILE_JSON" -log "Done. Updated $INPUT_FILE_JSON in place." +log "Done. Updated $(sg_rel "$INPUT_FILE_JSON") in place." diff --git a/scripts/enrich_variable_sets.sh b/scripts/enrich_variable_sets.sh new file mode 100755 index 0000000..340548f --- /dev/null +++ b/scripts/enrich_variable_sets.sh @@ -0,0 +1,166 @@ +#!/bin/bash +# Enrich generated payloads with TFC/TFE Variable Set variables. +# +# The tfe provider can't enumerate variable sets, so we use the TFC API (same +# token as state export). For each workspace we compute the variable sets that +# apply (global / project-scoped / workspace-scoped), resolve set-vs-set +# precedence (priority + scope), and merge the result into the matching workflow +# payload. A workspace's own variable is only overridden by a *priority* set; +# otherwise set vars only fill keys the workspace doesn't define. Sensitive set +# vars can't be read from the API and are skipped + reported. +# +# Usage: enrich_variable_sets.sh [more.json ...] +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=tools.sh +source "$SCRIPT_DIR/tools.sh" + +ORG="${1:-}" +shift || true +if [ -z "$ORG" ] || [ "$#" -eq 0 ]; then + echo "Usage: $0 [more.json ...]" >&2 + exit 1 +fi + +HOST="${SG_TFC_HOSTNAME:-app.terraform.io}" +API="https://$HOST/api/v2" +command -v curl >/dev/null 2>&1 || { sg_err "curl is required for variable-set enrichment"; exit 1; } +JQ_BIN="$(sg_resolve jq sg_ensure_jq)" + +# TFC token (same sources as state export): credentials file or TFE_TOKEN. +creds="$HOME/.terraform.d/credentials.tfrc.json" +token="" +[ -f "$creds" ] && token="$("$JQ_BIN" -r --arg h "$HOST" '.credentials[$h].token // empty' "$creds" 2>/dev/null || true)" +[ -z "$token" ] && token="${TFE_TOKEN:-}" +if [ -z "$token" ]; then + sg_warn "no TFC token (terraform login / TFE_TOKEN); skipping variable-set enrichment" + exit 0 +fi + +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT +AUTH=(-H "Authorization: Bearer $token") + +# fetch_all — GET a paginated JSON:API collection, print the merged .data array. +fetch_all() { + local path="$1" page=1 next + : >"$WORK/acc.ndjson" + while :; do + if ! curl -fsS "${AUTH[@]}" "$API/$path?page%5Bsize%5D=100&page%5Bnumber%5D=$page" >"$WORK/page.json"; then + sg_err "TFC API request failed: $path"; return 1 + fi + "$JQ_BIN" -c '.data[]?' "$WORK/page.json" >>"$WORK/acc.ndjson" + next="$("$JQ_BIN" -r '.meta.pagination."next-page" // empty' "$WORK/page.json" 2>/dev/null || true)" + [ -z "$next" ] && break + page="$next" + done + "$JQ_BIN" -s '.' "$WORK/acc.ndjson" +} + +sg_log "fetching workspaces and variable sets from $HOST (org: $ORG)..." + +# name -> {id, project} +fetch_all "organizations/$ORG/workspaces" \ + | "$JQ_BIN" '[.[] | {name: .attributes.name, id: .id, project: (.relationships.project.data.id // "")}]' \ + >"$WORK/workspaces.json" || exit 1 + +# Each set with its scope + variables. +: >"$WORK/sets.ndjson" +sets_raw="$(fetch_all "organizations/$ORG/varsets")" || exit 1 +echo "$sets_raw" | "$JQ_BIN" -c '.[]' | while IFS= read -r s; do + sid="$(echo "$s" | "$JQ_BIN" -r '.id')" + vars="$(curl -fsS "${AUTH[@]}" "$API/varsets/$sid/relationships/vars" 2>/dev/null \ + | "$JQ_BIN" -c '[.data[]? | {key: .attributes.key, value: (.attributes.value // ""), category: .attributes.category, sensitive: (.attributes.sensitive // false), hcl: (.attributes.hcl // false)}]' 2>/dev/null || echo '[]')" + echo "$s" | "$JQ_BIN" -c --argjson vars "$vars" '{ + name: .attributes.name, + global: (.attributes.global // false), + priority: (.attributes.priority // false), + wsids: [.relationships.workspaces.data[]?.id], + projids: [.relationships.projects.data[]?.id], + vars: $vars + }' >>"$WORK/sets.ndjson" +done +"$JQ_BIN" -s '.' "$WORK/sets.ndjson" >"$WORK/sets.json" + +set_count="$("$JQ_BIN" 'length' "$WORK/sets.json")" +if [ "$set_count" -eq 0 ]; then + sg_log "no variable sets found; nothing to enrich" + exit 0 +fi +sg_log "resolving $set_count variable set(s) across workspaces..." + +# Per workspace -> list of winning vars (set-vs-set precedence resolved; tagged +# with priority + sensitive + conflict). rank: non-priority global/proj/ws = 1/2/3, +# priority = 5/6/7; workspace's own vars sit at 4 and are applied during merge. +"$JQ_BIN" -n --slurpfile ws "$WORK/workspaces.json" --slurpfile sets "$WORK/sets.json" ' + ($ws[0]) as $workspaces | ($sets[0]) as $sets + | reduce $workspaces[] as $w ({}; + . + { ($w.name): ( + [ $sets[] + | . as $s + | (($s.global == true) + or (($s.wsids // []) | index($w.id) != null) + or (($w.project != "") and (($s.projids // []) | index($w.project) != null))) as $applies + | select($applies) + | (if (($s.wsids // []) | index($w.id) != null) then 3 + elif (($w.project != "") and (($s.projids // []) | index($w.project) != null)) then 2 + else 1 end) as $scope + | ($scope + (if $s.priority then 4 else 0 end)) as $rank + | ($s.vars[]? | {key, value, category, hcl, sensitive, rank: $rank, set: $s.name, priority: ($rank >= 5)}) + ] + | group_by(.key) + | map( (max_by(.rank)) as $win + | $win + { conflict: (([ .[] | select(.rank == $win.rank) ] | length) > 1) } ) + )} + ) +' >"$WORK/effective.json" + +# Merge the effective set vars into each payload, then report counts. +for f in "$@"; do + before_tf="$("$JQ_BIN" '[.[].VCSConfig.iacInputData.data | length] | add // 0' "$f")" + out="$WORK/merged.json" + "$JQ_BIN" --slurpfile eff "$WORK/effective.json" ' + ($eff[0]) as $E + | map( + ((.CLIConfiguration.TfStateFilePath // "") | sub(".*/"; "") | sub("\\.tfstate$"; "")) as $wsName + | ($E[$wsName] // []) as $all + | ($all | map(select(.sensitive != true and .category == "terraform"))) as $tf + | ($all | map(select(.sensitive != true and .category == "env"))) as $env + | .VCSConfig.iacInputData.data = ( + reduce $tf[] as $v ((.VCSConfig.iacInputData.data // {}); + ($v.value | (fromjson? // $v.value)) as $val + | if (has($v.key) | not) then . + {($v.key): $val} + elif $v.priority then . + {($v.key): $val} + else . end)) + | .EnvironmentVariables = ( + reduce $env[] as $v ((.EnvironmentVariables // []); + (map(.config.varName) | index($v.key)) as $idx + | if ($idx == null) then . + [{config: {textValue: $v.value, varName: $v.key}, kind: "PLAIN_TEXT"}] + elif $v.priority then (.[$idx].config.textValue = $v.value) + else . end)) + ) + ' "$f" >"$out" && mv "$out" "$f" + after_tf="$("$JQ_BIN" '[.[].VCSConfig.iacInputData.data | length] | add // 0' "$f")" + sg_log "$(basename "$f"): +$((after_tf - before_tf)) terraform var(s) from variable sets" +done + +# Report sensitive set vars (cannot be migrated) and key conflicts. +"$JQ_BIN" -r ' + to_entries[] | .key as $ws | .value[] + | select(.sensitive == true) | " - \($ws): \(.category):\(.key) (set \(.set))" +' "$WORK/effective.json" | sort -u >"$WORK/sensitive.txt" +if [ -s "$WORK/sensitive.txt" ]; then + sg_warn "sensitive variable-set vars skipped (recreate as SG secrets):" + cat "$WORK/sensitive.txt" >&2 +fi +"$JQ_BIN" -r ' + to_entries[] | .key as $ws | .value[] + | select(.conflict == true) | " - \($ws): \(.category):\(.key)" +' "$WORK/effective.json" | sort -u >"$WORK/conflicts.txt" +if [ -s "$WORK/conflicts.txt" ]; then + sg_warn "variable-set key conflicts (same key in multiple equal-precedence sets; picked one):" + cat "$WORK/conflicts.txt" >&2 +fi + +sg_success "variable-set enrichment complete" diff --git a/scripts/migrate.sh b/scripts/migrate.sh new file mode 100755 index 0000000..4adc693 --- /dev/null +++ b/scripts/migrate.sh @@ -0,0 +1,443 @@ +#!/bin/bash +# End-to-end StackGuardian migration orchestrator. +# +# Runs apply -> convert -> validate -> import, resolving tooling from PATH first +# (e.g. inside the Docker image) and falling back to cached downloads when run +# natively. Designed to run both on the host and inside the migrator container. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=tools.sh +source "$SCRIPT_DIR/tools.sh" + +# SCRIPT_DIR holds the sibling scripts; SG_REPO_ROOT (from tools.sh) is the repo +# root used for all repo-relative paths. +TRANSFORMER_DIR="$SG_REPO_ROOT/transformer/terraform-cloud" +TFVARS="$TRANSFORMER_DIR/terraform.tfvars" +EXPORT_DIR="${SG_EXPORT_DIR:-$SG_REPO_ROOT/export}" +MAPPING="${SG_WFGROUP_MAP:-$SG_REPO_ROOT/.sg/workflow-groups.json}" +ORG="${SG_ORG:-}" +SG_BASE_URL="${SG_BASE_URL:-https://api.app.stackguardian.io}" +ASSUME_YES=0 +PURGE=0 +CREATE_GROUPS=1 +ENRICH_VARSETS=1 +VCS_TRIGGERS=1 +VERBOSE="${SG_VERBOSE:-0}" +CONC="${SG_CONCURRENCY:-4}" +TF_PARALLELISM="${SG_TF_PARALLELISM:-20}" +RETRIES="${SG_RETRIES:-4}" +RETRY_BASE="${SG_RETRY_BASE:-2}" +PF=() + +usage() { + cat >&2 < enrich -> convert -> validate -> import (default) + clean Remove local working artifacts for a fresh start (export/, TF state, + tool cache). Add --all to also remove config (terraform.tfvars, mapping). + +Each TFC project maps to an SG workflow group named tfc-, created via the +API if missing. Override a project's target group in .sg/workflow-groups.json +(\`{"": ""}\`); override groups are not auto-created. + +Options: + --org NAME StackGuardian org for import (or set SG_ORG) + --export-dir DIR Payload/state output dir (default: ./export) + --mapping FILE Optional project-segment -> group override map (default: .sg/workflow-groups.json) + --concurrency N Max parallel jobs for convert/import (default: 4) + --no-create-groups Do not create missing workflow groups; require them to exist + --no-variable-sets Skip merging TFC Variable Set variables in the 'all' flow + --no-vcs-triggers Skip registering VCS triggers after import + --all With 'clean': also remove config (terraform.tfvars, mapping, .sg) + -v, --verbose Show full terraform/tool output (default: concise) + -y, --yes Skip the import confirmation prompt + -h, --help Show this help + +Environment: + SG_API_TOKEN StackGuardian API token (required for import) + SG_ORG StackGuardian org (alternative to --org) + SG_RETRIES Import retry attempts on failure (default: 4) + SG_TF_PARALLELISM terraform apply -parallelism (default: 20) +EOF +} + +die() { sg_err "$*"; exit 1; } + +throttle() { while [ "$(jobs -rp | wc -l | tr -d ' ')" -ge "$1" ]; do sleep 0.2; done; } + +# run_parallel — runs fn over items, up to max at a time. +# Each job's stdout+stderr is buffered to its own file (so concurrent tools see +# a non-TTY and don't scatter spinner output across the terminal), then flushed +# as a clean labeled block in submission order. Returns non-zero if any failed. +run_parallel() { + local fn="$1" max="$2"; shift 2 + local statusdir i=0 rc=0 item + statusdir="$(mktemp -d)" + for item in "$@"; do + throttle "$max" + ( "$fn" "$item" >"$statusdir/$i.out" 2>&1; echo "$?" >"$statusdir/$i.rc" ) & + i=$((i + 1)) + done + wait + i=0 + for item in "$@"; do + printf '%s── %s ──%s\n' "$C_CYAN" "$(basename "$item")" "$C_RESET" >&2 + [ -s "$statusdir/$i.out" ] && cat "$statusdir/$i.out" >&2 + [ "$(cat "$statusdir/$i.rc" 2>/dev/null)" = "0" ] || rc=1 + i=$((i + 1)) + done + rm -rf "$statusdir" + return "$rc" +} + +# Populate PF with the generated payload files. +payload_files() { + shopt -s nullglob + PF=("$EXPORT_DIR"/sg-payload.*.json) + shopt -u nullglob +} + +seg_of() { local b; b="$(basename "$1")"; b="${b#sg-payload.}"; echo "${b%.json}"; } + +cmd_init() { + sg_step "Phase: init" + mkdir -p "$SG_REPO_ROOT/.sg" "$SG_CACHE_BIN" + if [ ! -f "$TFVARS" ]; then + cp "$TRANSFORMER_DIR/terraform.tfvars.example" "$TFVARS" + sg_log "created $(sg_rel "$TFVARS") — edit it before 'apply'" + else + sg_log "$(sg_rel "$TFVARS") already exists" + fi + sg_log "workflow groups are created automatically as tfc-; no mapping needed" + sg_success "init complete" +} + +# --- StackGuardian API helpers (workflow groups) --------------------------- + +# group_for -> the SG workflow group for a project segment: an entry +# from the optional override map, else the default tfc-. +group_for() { + local seg="$1" override="" + if [ -f "$MAPPING" ]; then + override="$("$JQ_BIN" -r --arg k "$seg" '.[$k] // empty' "$MAPPING" 2>/dev/null || true)" + fi + [ -n "$override" ] && echo "$override" || echo "tfc-$seg" +} + +# wfgroup_http_code -> HTTP status of GET (200 exists, 404 missing). +wfgroup_http_code() { + curl -sS -o /dev/null -w '%{http_code}' \ + -H "Authorization: apikey $SG_API_TOKEN" \ + "$SG_BASE_URL/api/v1/orgs/$ORG/wfgrps/$1/" +} + +# wfgroup_create — create a workflow group (idempotent at call sites). +wfgroup_create() { + local body + body="$("$JQ_BIN" -nc --arg n "$1" '{ResourceName:$n, Description:"Created by stackguardian-migrator (Terraform Cloud import)"}')" + sg_retry "$RETRIES" "$RETRY_BASE" -- \ + curl -fsS -X POST \ + -H "Authorization: apikey $SG_API_TOKEN" -H "Content-Type: application/json" \ + -d "$body" "$SG_BASE_URL/api/v1/orgs/$ORG/wfgrps/" >/dev/null +} + +# wf_triggers_endpoint -> the VCS-triggers webhook URL for a workflow. +wf_triggers_endpoint() { + echo "$SG_BASE_URL/api/v1/orgs/$ORG/wfgrps/$1/wfs/$2/webhooks/vcs_triggers/" +} + +# do_set_triggers — for each workflow in the file with a non-null +# VCSTriggers block, register its VCS triggers via the dedicated webhooks +# endpoint (the bulk create API silently drops VCSTriggers; this second pass is +# what actually wires up the repo webhook). Sends {VCSConfig, VCSTriggers} taken +# straight from the (converted) payload. Per-workflow failures are surfaced but +# do not abort the rest of the file. +do_set_triggers() { + local f="$1" seg grp n i wf body rc=0 set=0 skip=0 + seg="$(seg_of "$f")" + grp="$(group_for "$seg")" + n="$("$JQ_BIN" 'length' "$f")" + for ((i = 0; i < n; i++)); do + if [ "$("$JQ_BIN" -r --argjson i "$i" '(.[$i].VCSTriggers // null) != null' "$f")" != "true" ]; then + skip=$((skip + 1)); continue + fi + wf="$("$JQ_BIN" -r --argjson i "$i" '.[$i].ResourceName' "$f")" + body="$("$JQ_BIN" -c --argjson i "$i" '{VCSConfig: .[$i].VCSConfig, VCSTriggers: .[$i].VCSTriggers}' "$f")" + if sg_retry "$RETRIES" "$RETRY_BASE" -- \ + curl -fsS -X POST \ + -H "Authorization: apikey $SG_API_TOKEN" -H "Content-Type: application/json" \ + -d "$body" "$(wf_triggers_endpoint "$grp" "$wf")" >/dev/null; then + set=$((set + 1)) + else + sg_warn " vcs triggers failed: $grp/$wf"; rc=1 + fi + done + sg_log "$(basename "$f"): set triggers on $set workflow(s) (skipped $skip without triggers)" + return "$rc" +} + +# set_triggers_pass — run do_set_triggers over all payload files (assumes +# JQ_BIN/SG_API_TOKEN/ORG are already set up by the caller). +set_triggers_pass() { + local total + total="$("$JQ_BIN" -s 'map(map(select((.VCSTriggers // null) != null)) | length) | add // 0' "${PF[@]}")" + if [ "$total" -eq 0 ]; then + sg_log "no workflows carry VCS triggers — nothing to register" + return 0 + fi + sg_log "registering VCS triggers for $total workflow(s), up to $CONC in parallel (retries: $RETRIES)" + if run_parallel do_set_triggers "$CONC" "${PF[@]}"; then + sg_success "vcs triggers registered" + else + sg_err "one or more VCS trigger registrations failed (re-run: $0 triggers)"; return 1 + fi +} + +cmd_triggers() { + sg_step "Phase: vcs triggers" + [ -n "${SG_API_TOKEN:-}" ] || die "SG_API_TOKEN is not set." + [ -n "$ORG" ] || die "StackGuardian org not set (use --org or SG_ORG)." + command -v curl >/dev/null 2>&1 || die "curl is required for VCS trigger registration." + export SG_API_TOKEN SG_BASE_URL + JQ_BIN="$(sg_resolve jq sg_ensure_jq)" + payload_files + [ "${#PF[@]}" -gt 0 ] || die "No payload files in $(sg_rel "$EXPORT_DIR")." + set_triggers_pass +} + +cmd_clean() { + sg_step "Phase: clean" + sg_log "removing local working artifacts..." + rm -rf "$EXPORT_DIR" + rm -rf "$TRANSFORMER_DIR/.terraform" "$TRANSFORMER_DIR/.terraform.lock.hcl" \ + "$TRANSFORMER_DIR/terraform.tfstate" "$TRANSFORMER_DIR/terraform.tfstate.backup" + rm -rf "$SG_CACHE_DIR" + if [ "$PURGE" -eq 1 ]; then + rm -f "$TFVARS" "$MAPPING" + rm -rf "$SG_REPO_ROOT/.sg" + sg_log "also removed config (terraform.tfvars, workflow-groups.json, .sg)" + fi + sg_success "clean complete" +} + +cmd_apply() { + sg_step "Phase: apply (terraform)" + command -v terraform >/dev/null 2>&1 || die "terraform not found on PATH" + [ -f "$TFVARS" ] || die "Missing $(sg_rel "$TFVARS"). Run: $0 init (then edit it)." + # State export (TFC API) calls curl + jq from terraform's local-exec; make sure + # both are on PATH for the apply (jq from cache if not already installed). + command -v curl >/dev/null 2>&1 || die "curl is required for state export" + local jqdir tflog rc=0 + jqdir="$(dirname "$(sg_resolve jq sg_ensure_jq)")" + + if [ "$VERBOSE" -eq 1 ]; then + ( cd "$TRANSFORMER_DIR" && export PATH="$jqdir:$PATH" TF_IN_AUTOMATION=1 \ + && terraform init -input=false \ + && terraform apply -auto-approve -compact-warnings -parallelism="$TF_PARALLELISM" -var-file=terraform.tfvars ) || rc=$? + else + # Quiet: capture terraform's verbose plan/output; surface only progress, the + # final summary, and (on failure) the captured log. + tflog="$(mktemp)" + sg_log "initializing terraform (providers)..." + ( cd "$TRANSFORMER_DIR" && export PATH="$jqdir:$PATH" TF_IN_AUTOMATION=1 && terraform init -input=false -no-color ) >"$tflog" 2>&1 || rc=$? + if [ "$rc" -eq 0 ]; then + sg_log "reading workspaces, generating payloads, exporting state..." + ( cd "$TRANSFORMER_DIR" && export PATH="$jqdir:$PATH" TF_IN_AUTOMATION=1 \ + && terraform apply -auto-approve -compact-warnings -no-color -parallelism="$TF_PARALLELISM" -var-file=terraform.tfvars ) >"$tflog" 2>&1 || rc=$? + fi + if [ "$rc" -ne 0 ]; then + sg_err "terraform failed (rc=$rc):"; cat "$tflog" >&2 + else + grep -E '^(Apply complete|No changes)' "$tflog" | sed 's/^/ /' >&2 || true + fi + rm -f "$tflog" + fi + + [ "$rc" -eq 0 ] || return "$rc" + sg_success "apply complete — payloads in $(sg_rel "$EXPORT_DIR")" +} + +cmd_enrich() { + sg_step "Phase: variable sets" + [ -f "$TFVARS" ] || die "Missing $(sg_rel "$TFVARS") (run: $0 init)." + payload_files + [ "${#PF[@]}" -gt 0 ] || die "No payload files in $(sg_rel "$EXPORT_DIR") (run 'apply' first)." + # Variable sets belong to the TFC org (tfOrg in terraform.tfvars), not SG_ORG. + local jqb h2j tforg tfhost + jqb="$(sg_resolve jq sg_ensure_jq)" + h2j="$(sg_resolve hcl2json sg_ensure_hcl2json)" + tforg="$("$h2j" "$TFVARS" | "$jqb" -r '.tfOrg // empty')" + [ -n "$tforg" ] || die "tfOrg not found in $(sg_rel "$TFVARS")" + tfhost="$("$h2j" "$TFVARS" | "$jqb" -r '.tfHostname // empty')" + SG_TFC_HOSTNAME="${tfhost:-app.terraform.io}" "$SCRIPT_DIR/enrich_variable_sets.sh" "$tforg" "${PF[@]}" +} + +do_convert() { "$SCRIPT_DIR/convert_hcl_to_json.sh" "$1"; } + +cmd_convert() { + sg_step "Phase: convert (HCL → JSON)" + payload_files + [ "${#PF[@]}" -gt 0 ] || die "No payload files in $(sg_rel "$EXPORT_DIR") (run 'apply' first)." + sg_log "converting ${#PF[@]} payload(s), up to $CONC in parallel" + if run_parallel do_convert "$CONC" "${PF[@]}"; then + sg_success "converted ${#PF[@]} payload(s)" + else + sg_err "conversion failed for one or more payloads"; return 1 + fi +} + +cmd_validate() { + sg_step "Phase: validate" + payload_files + [ "${#PF[@]}" -gt 0 ] || die "No payload files in $(sg_rel "$EXPORT_DIR")." + if "$SCRIPT_DIR/validate_payload.sh" "${PF[@]}"; then + sg_success "all ${#PF[@]} payload(s) valid" + else + sg_err "validation failed"; return 1 + fi +} + +do_import() { + local f="$1" seg grp + seg="$(seg_of "$f")" + grp="$(group_for "$seg")" + sg_log "importing $(basename "$f") -> $grp" + sg_retry "$RETRIES" "$RETRY_BASE" -- \ + "$SGCLI_BIN" workflow create --bulk --workflow-group "$grp" --org "$ORG" "$f" +} + +cmd_import() { + sg_step "Phase: import" + [ -n "${SG_API_TOKEN:-}" ] || die "SG_API_TOKEN is not set." + [ -n "$ORG" ] || die "StackGuardian org not set (use --org or SG_ORG)." + command -v curl >/dev/null 2>&1 || die "curl is required for workflow-group checks/creation." + # Both our API calls and sg-cli honor SG_BASE_URL (e.g. non-prod); export it + # so the sg-cli child process inherits the same target. + export SG_API_TOKEN SG_BASE_URL + JQ_BIN="$(sg_resolve jq sg_ensure_jq)" + + payload_files + [ "${#PF[@]}" -gt 0 ] || die "No payload files in $(sg_rel "$EXPORT_DIR")." + + # Build the plan: resolve each project's group, check existence, and decide + # which groups need creating. Override groups (from the map) must already exist. + local fail=0 to_create=" " f seg grp count override is_override code status + printf '%sImport plan%s (org: %s%s%s, %s)\n' "$C_BOLD" "$C_RESET" "$C_CYAN" "$ORG" "$C_RESET" "$SG_BASE_URL" >&2 + printf ' %s%-34s %-26s %-9s %s%s\n' "$C_BOLD" "FILE" "WORKFLOW GROUP" "WORKFLOWS" "STATUS" "$C_RESET" >&2 + for f in "${PF[@]}"; do + seg="$(seg_of "$f")" + count="$("$JQ_BIN" 'length' "$f")" + override="" + [ -f "$MAPPING" ] && override="$("$JQ_BIN" -r --arg k "$seg" '.[$k] // empty' "$MAPPING" 2>/dev/null || true)" + if [ -n "$override" ]; then grp="$override"; is_override=1; else grp="tfc-$seg"; is_override=0; fi + + code="$(wfgroup_http_code "$grp")" + case "$code" in + 200) status="${C_GREEN}exists${C_RESET}" ;; + 404) + if [ "$is_override" -eq 1 ]; then + status="${C_RED}missing!${C_RESET}"; fail=1 + elif [ "$CREATE_GROUPS" -eq 1 ]; then + status="${C_YELLOW}create${C_RESET}" + case "$to_create" in *" $grp "*) ;; *) to_create="$to_create$grp " ;; esac + else + status="${C_RED}missing!${C_RESET}"; fail=1 + fi ;; + 401 | 403) die "auth failed (HTTP $code) for org '$ORG' — check SG_API_TOKEN" ;; + 000) die "could not reach $SG_BASE_URL" ;; + *) die "unexpected HTTP $code checking group '$grp'" ;; + esac + printf ' %-34s %-26s %-9s %s\n' "$(basename "$f")" "$grp" "$count" "$status" >&2 + done + if [ "$fail" -ne 0 ]; then + die "some groups are missing (override groups are not auto-created; create them or remove the override)." + fi + + if [ "$ASSUME_YES" -ne 1 ]; then + printf '%sProceed?%s This imports to %s and creates any "create" groups. [y/N] ' "$C_BOLD$C_YELLOW" "$C_RESET" "$ORG" >&2 + read -r ans || ans="" + case "$ans" in y | Y | yes | YES) ;; *) die "Aborted." ;; esac + fi + + # Create the missing tfc-* groups before importing into them. + for grp in $to_create; do + sg_log "creating workflow group $grp" + wfgroup_create "$grp" || die "failed to create workflow group $grp" + done + + SGCLI_BIN="$(sg_resolve sg-cli sg_ensure_sgcli)" + sg_log "importing ${#PF[@]} payload(s), up to $CONC in parallel (retries: $RETRIES)" + if run_parallel do_import "$CONC" "${PF[@]}"; then + sg_success "import complete (${#PF[@]} payload(s))" + else + sg_err "one or more imports failed"; return 1 + fi + + # VCS triggers are not accepted by the bulk create API; register them in a + # second pass against the dedicated webhooks endpoint (skip with --no-vcs-triggers). + if [ "$VCS_TRIGGERS" -eq 1 ]; then + set_triggers_pass + fi +} + +CMD="" +while [ $# -gt 0 ]; do + case "$1" in + -y | --yes) ASSUME_YES=1 ;; + --org) ORG="$2"; shift ;; + --org=*) ORG="${1#*=}" ;; + --export-dir) EXPORT_DIR="$2"; shift ;; + --export-dir=*) EXPORT_DIR="${1#*=}" ;; + --mapping) MAPPING="$2"; shift ;; + --mapping=*) MAPPING="${1#*=}" ;; + --concurrency) CONC="$2"; shift ;; + --concurrency=*) CONC="${1#*=}" ;; + --no-create-groups) CREATE_GROUPS=0 ;; + --no-variable-sets) ENRICH_VARSETS=0 ;; + --no-vcs-triggers) VCS_TRIGGERS=0 ;; + -v | --verbose) VERBOSE=1 ;; + --all) PURGE=1 ;; + -h | --help) usage; exit 0 ;; + init | apply | enrich | convert | validate | import | triggers | all | clean) CMD="$1" ;; + *) echo "Unknown argument: $1" >&2; usage; exit 1 ;; + esac + shift +done +CMD="${CMD:-all}" +export SG_VERBOSE="$VERBOSE" + +case "$CMD" in + init) cmd_init ;; + clean) cmd_clean ;; + apply) cmd_apply ;; + enrich) cmd_enrich ;; + convert) cmd_convert ;; + validate) cmd_validate ;; + import) cmd_import ;; + triggers) cmd_triggers ;; + all) + if [ ! -f "$TFVARS" ]; then + cmd_init + die "Edit $(sg_rel "$TFVARS"), then re-run '$0 all'." + fi + # Fail fast on import prerequisites before the (long) apply. + [ -n "${SG_API_TOKEN:-}" ] || die "SG_API_TOKEN is not set (needed for import)." + [ -n "$ORG" ] || die "StackGuardian org not set (use --org or SG_ORG)." + cmd_apply + if [ "$ENRICH_VARSETS" -eq 1 ]; then cmd_enrich; fi + cmd_convert + cmd_validate + cmd_import + ;; +esac diff --git a/scripts/tools.sh b/scripts/tools.sh new file mode 100755 index 0000000..f3b568f --- /dev/null +++ b/scripts/tools.sh @@ -0,0 +1,162 @@ +#!/bin/bash +# Shared tool bootstrap + cache for the StackGuardian migrator. +# +# Source this file; it exposes the repo root and sg_* helpers that download and +# cache the required CLIs under .sg/cached/ (override with SG_CACHE_DIR). Each +# sg_ensure_* function prints the absolute path to a ready-to-run binary on +# stdout; all human-facing logs go to stderr so the path can be captured with +# command substitution: JQ_BIN=$(sg_ensure_jq). + +# This file lives in scripts/; the repo root is its parent directory. +SG_REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +SG_CACHE_DIR="${SG_CACHE_DIR:-$SG_REPO_ROOT/.sg/cached}" +SG_CACHE_BIN="$SG_CACHE_DIR/bin" + +# Pinned versions. +SG_JQ_VERSION="jq-1.8.1" +SG_HCL2JSON_VERSION="v0.6.7" +SG_YAJSV_VERSION="v1.4.1" + +# Colored logging — disabled when stderr is not a TTY, NO_COLOR is set, or +# TERM=dumb, so piped/CI output stays clean. +if [ -t 2 ] && [ -z "${NO_COLOR:-}" ] && [ "${TERM:-}" != "dumb" ]; then + C_RESET=$'\033[0m'; C_BOLD=$'\033[1m'; C_DIM=$'\033[2m' + C_RED=$'\033[31m'; C_GREEN=$'\033[32m'; C_YELLOW=$'\033[33m'; C_CYAN=$'\033[36m' +else + C_RESET=""; C_BOLD=""; C_DIM=""; C_RED=""; C_GREEN=""; C_YELLOW=""; C_CYAN="" +fi + +# sg_rel — render a path relative to the repo root for readable logs +# (operations still use absolute paths; this is display-only). +sg_rel() { + case "$1" in + "$SG_REPO_ROOT"/*) printf '%s' "${1#"$SG_REPO_ROOT"/}" ;; + "$SG_REPO_ROOT") printf '.' ;; + *) printf '%s' "$1" ;; + esac +} + +sg_log() { printf '%s[sg-migrate]%s %s\n' "$C_CYAN" "$C_RESET" "$*" >&2; } +sg_warn() { printf '%s[sg-migrate] WARN%s %s\n' "$C_YELLOW" "$C_RESET" "$*" >&2; } +sg_err() { printf '%s[sg-migrate] ERROR%s %s\n' "$C_RED$C_BOLD" "$C_RESET" "$*" >&2; } +sg_success() { printf '%s[sg-migrate] ✓%s %s\n' "$C_GREEN$C_BOLD" "$C_RESET" "$*" >&2; } +sg_step() { printf '\n%s==> %s%s\n' "$C_CYAN$C_BOLD" "$*" "$C_RESET" >&2; } +sg_dim() { printf '%s %s%s\n' "$C_DIM" "$*" "$C_RESET" >&2; } + +sg_arch() { + case "$(uname -m)" in + x86_64 | amd64) echo "amd64" ;; + aarch64 | arm64) echo "arm64" ;; + *) echo "Unsupported architecture: $(uname -m)" >&2; return 1 ;; + esac +} + +# sg_retry -- +# Runs the command, retrying with exponential backoff (capped at 60s, with a +# little jitter) until it succeeds or attempts are exhausted. Returns the +# command's last exit code. Used to ride out transient API failures/rate limits. +sg_retry() { + local max="$1" base="$2"; shift 2 + [ "${1:-}" = "--" ] && shift + local attempt=1 delay="$base" rc=0 + while :; do + if "$@"; then return 0; fi + rc=$? + if [ "$attempt" -ge "$max" ]; then + sg_err "failed after ${attempt} attempt(s) (exit ${rc}): $*" + return "$rc" + fi + local jitter=$((RANDOM % (base + 1))) + sg_warn "attempt ${attempt}/${max} failed (exit ${rc}); retrying in $((delay + jitter))s..." + sleep "$((delay + jitter))" + attempt=$((attempt + 1)) + delay=$((delay * 2)) + [ "$delay" -gt 60 ] && delay=60 + done +} + +# sg_resolve : prefer a binary already on PATH (e.g. +# installed in the Docker image); otherwise download+cache via the ensure fn. +# Prints the path to use on stdout. +sg_resolve() { + local name="$1" ensure="$2" + if command -v "$name" >/dev/null 2>&1; then command -v "$name"; return 0; fi + if [ -n "$ensure" ]; then "$ensure"; return $?; fi + echo "Required tool '$name' not found on PATH" >&2 + return 1 +} + +# sg_download +sg_download() { + local url="$1" dest="$2" + command -v curl >/dev/null 2>&1 || { echo "curl is required but not found" >&2; return 1; } + mkdir -p "$(dirname "$dest")" + if ! curl -fsSL -o "$dest" "$url"; then + echo "Failed to download: $url" >&2 + return 1 + fi +} + +sg_ensure_jq() { + local bin="$SG_CACHE_BIN/jq" arch os + if [ ! -x "$bin" ]; then + arch=$(sg_arch) || return 1 + case "$(uname -s)" in Darwin) os="macos" ;; Linux) os="linux" ;; *) echo "Unsupported OS: $(uname -s)" >&2; return 1 ;; esac + sg_log "caching jq ${SG_JQ_VERSION}..." + sg_download "https://github.com/jqlang/jq/releases/download/${SG_JQ_VERSION}/jq-${os}-${arch}" "$bin" || return 1 + chmod +x "$bin" + fi + echo "$bin" +} + +sg_ensure_hcl2json() { + local bin="$SG_CACHE_BIN/hcl2json" arch os + if [ ! -x "$bin" ]; then + arch=$(sg_arch) || return 1 + case "$(uname -s)" in Darwin) os="darwin" ;; Linux) os="linux" ;; *) echo "Unsupported OS: $(uname -s)" >&2; return 1 ;; esac + sg_log "caching hcl2json ${SG_HCL2JSON_VERSION}..." + sg_download "https://github.com/tmccombs/hcl2json/releases/download/${SG_HCL2JSON_VERSION}/hcl2json_${os}_${arch}" "$bin" || return 1 + chmod +x "$bin" + fi + echo "$bin" +} + +sg_ensure_yajsv() { + local bin="$SG_CACHE_BIN/yajsv" arch os + if [ ! -x "$bin" ]; then + arch=$(sg_arch) || return 1 + case "$(uname -s)" in Darwin) os="darwin" ;; Linux) os="linux" ;; *) echo "Unsupported OS: $(uname -s)" >&2; return 1 ;; esac + if [ "$os" = "linux" ] && [ "$arch" = "arm64" ]; then + echo "No yajsv prebuilt binary for linux/arm64; install yajsv manually." >&2 + return 1 + fi + sg_log "caching yajsv ${SG_YAJSV_VERSION}..." + sg_download "https://github.com/neilpa/yajsv/releases/download/${SG_YAJSV_VERSION}/yajsv.${os}.${arch}" "$bin" || return 1 + chmod +x "$bin" + fi + echo "$bin" +} + +# Caches the sg-cli Go binary (latest release) for the host OS/arch and prints +# its path. Release assets are named sg-cli__.tar.gz (Darwin/Linux, +# arm64/x86_64). +sg_ensure_sgcli() { + local bin="$SG_CACHE_BIN/sg-cli" os arch tmp realcli + if [ ! -x "$bin" ]; then + case "$(uname -s)" in Darwin) os="Darwin" ;; Linux) os="Linux" ;; *) echo "Unsupported OS: $(uname -s)" >&2; return 1 ;; esac + case "$(uname -m)" in x86_64 | amd64) arch="x86_64" ;; aarch64 | arm64) arch="arm64" ;; *) echo "Unsupported architecture: $(uname -m)" >&2; return 1 ;; esac + sg_log "caching sg-cli (latest release, ${os}/${arch})..." + tmp=$(mktemp -d) + if ! curl -fsSL "https://github.com/StackGuardian/sg-cli/releases/latest/download/sg-cli_${os}_${arch}.tar.gz" -o "$tmp/sg-cli.tar.gz"; then + echo "Failed to download sg-cli" >&2; rm -rf "$tmp"; return 1 + fi + tar -xzf "$tmp/sg-cli.tar.gz" -C "$tmp" + realcli=$(find "$tmp" -maxdepth 2 -type f -name sg-cli | head -1) + [ -n "$realcli" ] || { echo "sg-cli binary not found in release archive" >&2; rm -rf "$tmp"; return 1; } + mkdir -p "$SG_CACHE_BIN" + cp "$realcli" "$bin" + chmod +x "$bin" + rm -rf "$tmp" + fi + echo "$bin" +} diff --git a/sg-migrate.sh b/sg-migrate.sh new file mode 100755 index 0000000..d39287d --- /dev/null +++ b/sg-migrate.sh @@ -0,0 +1,69 @@ +#!/bin/bash +# Host entrypoint for the StackGuardian migrator. +# +# Runs scripts/migrate.sh inside the Docker image (so all tooling is isolated and +# identical across OSes). Mounts the repo at /app, mounts the Terraform Cloud +# credentials file read-only, and forwards SG_API_TOKEN/SG_ORG. +# +# Runs natively (no Docker) when: --native/--local is passed, SG_NATIVE=1 is set, +# the command is 'clean' (a local filesystem op), or docker is unavailable. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=scripts/tools.sh +source "$SCRIPT_DIR/scripts/tools.sh" +IMAGE="${SG_IMAGE:-stackguardian/migrator:local}" +CREDS="${TF_CREDENTIALS_FILE:-$HOME/.terraform.d/credentials.tfrc.json}" + +# Parse host-only flags (--native/--local, --build); everything else passes through. +NATIVE="${SG_NATIVE:-0}" +BUILD=0 +ARGS=() +for a in "$@"; do + case "$a" in + --native | --local) NATIVE=1 ;; + --build) BUILD=1 ;; + *) ARGS+=("$a") ;; + esac +done + +# 'clean' only touches the local filesystem — no container needed. +for a in ${ARGS[@]+"${ARGS[@]}"}; do + [ "$a" = "clean" ] && NATIVE=1 +done + +if [ "$NATIVE" = "1" ] || ! command -v docker >/dev/null 2>&1; then + [ "$NATIVE" = "1" ] || sg_warn "docker not found; running natively" + exec "$SCRIPT_DIR/scripts/migrate.sh" ${ARGS[@]+"${ARGS[@]}"} +fi + +if [ "$BUILD" = "1" ] || ! docker image inspect "$IMAGE" >/dev/null 2>&1; then + sg_log "building image $IMAGE ..." + docker build -t "$IMAGE" "$SCRIPT_DIR" +fi + +DOCKER_ARGS=(--rm -i + -v "$SCRIPT_DIR:/app" -w /app + -e SG_API_TOKEN -e SG_ORG -e SG_BASE_URL -e SG_CONCURRENCY -e SG_RETRIES -e SG_TF_PARALLELISM + -e TFE_TOKEN) + +# Interactive TTY only when attached to one (so the confirmation prompt works, +# but CI/non-tty invocations still run — use -y there). +if [ -t 0 ] && [ -t 1 ]; then DOCKER_ARGS+=(-t); fi + +# TFC auth: prefer a long-lived TFE_TOKEN (forwarded via -e above); otherwise +# mount the `terraform login` credentials file read-only. +if [ -n "${TFE_TOKEN:-}" ]; then + : +elif [ -f "$CREDS" ]; then + DOCKER_ARGS+=(-v "$CREDS:/root/.terraform.d/credentials.tfrc.json:ro") +else + sg_warn "no TFC auth found — set TFE_TOKEN (long-lived API token) or run 'terraform login'" +fi + +# Forward any TF_TOKEN_* env vars (alternative TFC/TFE auth) if present. +while IFS='=' read -r name _; do + case "$name" in TF_TOKEN_*) DOCKER_ARGS+=(-e "$name") ;; esac +done < <(env) + +exec docker run "${DOCKER_ARGS[@]}" "$IMAGE" /app/scripts/migrate.sh ${ARGS[@]+"${ARGS[@]}"} From 2e0242d0a4f1a37962cb7923574e565595687763 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= Date: Wed, 24 Jun 2026 19:06:18 +0200 Subject: [PATCH 04/30] docs: update README for the dockerized pipeline --- README.md | 84 ++++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 67 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index a10bfd3..0b884fe 100644 --- a/README.md +++ b/README.md @@ -12,16 +12,44 @@ Migrate workloads from other platforms to [StackGuardian Platform](https://app.s - Review the bulk workflow creation payload. - Run sg-cli with the bulk workflow creation payload. +## Quick start (orchestrated) + +`./sg-migrate.sh` runs the whole flow — `terraform apply` → HCL→JSON conversion → schema validation → bulk import — with all tooling (Terraform, `jq`, `hcl2json`, `yajsv`, `sg-cli`) isolated in a Docker image, so it behaves identically on Linux, macOS, and Windows. Docker is required for this path; without it the script automatically falls back to running natively (downloading pinned tools into `.sg/cached/`). + +```shell +export TFE_TOKEN= # long-lived API token (User/Team/Org token from the TFC UI) +export SG_API_TOKEN= +export SG_ORG= + +./sg-migrate.sh init # scaffolds terraform.tfvars +# edit transformer/terraform-cloud/terraform.tfvars (org, integrations, workspaceOverrides) + +./sg-migrate.sh all # apply -> enrich -> convert -> validate -> import (prompts before importing) +``` + +That's it — no workflow-group mapping to fill in. Each TFC project is imported into an SG workflow group named `tfc-`, **created automatically via the API** if it doesn't exist. The import prompt shows each group as `exists` or `create` before anything is written. + +- Single phase: `./sg-migrate.sh apply|enrich|convert|validate|import`. +- TFC **Variable Set** variables are merged into the payloads automatically (the `enrich` phase, via the TFC API); skip it with `--no-variable-sets`. +- `./sg-migrate.sh clean` removes local working artifacts (`export/`, Terraform state, tool cache) for a fresh start; add `--all` to also remove config. `clean` always runs locally. +- **Override** a project's target group (to reuse an existing group) in `.sg/workflow-groups.json`: `{"": ""}`. Override groups must already exist (they're not auto-created). +- Output is concise by default (terraform's plan/init noise is hidden; shown on error). Add `-v`/`--verbose` for full output. +- Flags: `-y` skip the import prompt (CI), `--concurrency N` parallel jobs, `--org NAME`, `--no-create-groups` require groups to pre-exist, `--build` rebuild the image, `--native`/`--local` force a local run even when Docker is available. +- Tuning via env: `SG_RETRIES`, `SG_TF_PARALLELISM`, `SG_NATIVE=1`. +- TFC auth: set `TFE_TOKEN` (recommended — a long-lived token avoids re-running `terraform login`); otherwise the `terraform login` credentials file is mounted read-only into the container. SG/TFC tokens are passed as env vars. + +The manual, step-by-step flow below remains supported for fine-grained control and is what each phase runs under the hood (the helper scripts live in `scripts/`). + ## Prerequisites - An organization on [StackGuardian Platform](https://app.stackguardian.io) - Optionally, pre-configure VCS, cloud integrations or private runners to use when importing into StackGuardian Platform. - Terraform -- [sg-cli](https://github.com/StackGuardian/sg-cli/tree/main/shell) +- [sg-cli](https://github.com/StackGuardian/sg-cli) -### Perform terraform login +### Authenticate to Terraform Cloud/Enterprise -Perform `terraform login` to ensure that your local Terraform can interact with your Terraform Cloud/Enterprise account. +Set `TFE_TOKEN` to a long-lived API token (create one under **User Settings → Tokens**, or use a Team/Organization token) — this is the recommended path and avoids session expiry. Alternatively run `terraform login`, which writes `~/.terraform.d/credentials.tfrc.json`. The `tfe` provider, the API state export, and variable-set enrichment all use whichever is present. ### Export the resource definitions and Terraform state @@ -35,9 +63,14 @@ terraform init terraform apply -auto-approve -var-file=terraform.tfvars ``` -A new `export` folder should have been created. The `sg-payload.json` file contains the definition for each workflow that will be created for each Terraform Workspace, and the `states` folder contains the files for the Terraform state for each of your workspaces, if the state export was enabled. +A new `export` folder should have been created, containing: -After completing the export, edit the `sg-payload.json` file to tune each workflow configuration with the following: +- One payload file **per TFC project**, named `sg-payload..json`. Each contains the workflow definitions for the workspaces in that project, and is imported into its own StackGuardian workflow group. +- `migration-summary.md` (and `migration-summary.json`) — a report of what was migrated and what needs manual attention: skipped sensitive variables, Terraform-version fallbacks, renamed workspaces, and workspaces whose state could not be exported. **Read this before importing.** +- The `states` folder with the Terraform state for each workspace, if state export was enabled. +- `state-export-failures.log`, if any workspace's state could not be pulled. + +After completing the export, tune each `sg-payload..json` file with the fields below. For values that differ per workspace (cloud integration, VCS auth, approvers, Terraform version), prefer setting `workspaceOverrides` in `terraform.tfvars` and re-running `terraform apply` instead of editing the JSON by hand — see `terraform.tfvars.example`. ### Use the example_payload.jsonc file as a reference and edit the schema of the `sg-payload.json` @@ -82,19 +115,29 @@ After completing the export, edit the `sg-payload.json` file to tune each workfl ### Convert HCL variables to JSON -HCL variables from Terraform Cloud appear as strings in `sg-payload.json` and need to be converted to JSON before importing. +HCL variables from Terraform Cloud appear as strings in the payload files and need to be converted to JSON before importing. + +Run the script from the repo root, once per payload file. It rewrites the file in place — converting the HCL-string variable values under `VCSConfig.iacInputData.data` into JSON — so none of the following steps need any change. The script downloads `jq` and `hcl2json` at runtime. + +```shell +for f in export/sg-payload.*.json; do ./scripts/convert_hcl_to_json.sh "$f"; done +``` + +### Validate the payloads (recommended) -Run the script from the repo root, passing the exported payload. It rewrites the file in place — converting the HCL-string variable values under `VCSConfig.iacInputData.data` into JSON — so none of the following steps need any change. The script downloads `jq` and `hcl2json` at runtime. +Validate each payload against the StackGuardian workflow schema before importing, to catch malformed payloads before a bulk API run. The schema (`schema/sg-payload.schema.json`) is derived from the SG OpenAPI spec; the script downloads `yajsv` at runtime. ```shell -./convert_hcl_to_json.sh export/sg-payload.json +./scripts/validate_payload.sh export/sg-payload.*.json ``` ### Bulk import workflows to StackGuardian Platform -- Fetch [sg-cli](https://github.com/StackGuardian/sg-cli.git) and set it up locally (documentation present in repo) -- Run the following commands and pass the `sg-payload.json` as payload (represented below) -- `--workflow-group` is required even though `wfgrpName` is set in the payload. Pass the workflow group ID (e.g. `prj-ThpsFFz59kqFaVr4`). +> The orchestrated flow above creates the `tfc-` groups for you. If you run sg-cli by hand, the target workflow group must already exist — create it in the SG UI/API first, or just use `./sg-migrate.sh import`. + +- Fetch the [sg-cli](https://github.com/StackGuardian/sg-cli) Go binary for your platform (assets: `sg-cli__.tar.gz`). +- Import **one payload file at a time**, each into its own workflow group. There is one file per TFC project (`sg-payload..json`), and the payload's group name is `tfc-`. +- `--workflow-group` takes the group name (e.g. `tfc-networking`); the group must exist. - Get your SG API Key here: - Login to Stackguardian. - Go to profile at the bottom left. Click on the email or the username. @@ -104,13 +147,20 @@ Run the script from the repo root, passing the exported payload. It rewrites the cd ../../export export SG_API_TOKEN= -wget -q "$(wget -qO- "https://api.github.com/repos/stackguardian/sg-cli/releases/latest" | jq -r '.tarball_url')" -O sg-cli.tar.gz && tar -xf sg-cli.tar.gz && rm -f sg-cli.tar.gz && /bin/cp -rf StackGuardian-sg-cli*/shell/sg-cli . && rm -rfd StackGuardian-sg-cli* +OS=$(uname -s); ARCH=$(uname -m); case "$ARCH" in x86_64|amd64) ARCH=x86_64;; arm64|aarch64) ARCH=arm64;; esac +curl -fsSL "https://github.com/StackGuardian/sg-cli/releases/latest/download/sg-cli_${OS}_${ARCH}.tar.gz" | tar -xz sg-cli -./sg-cli workflow create --bulk --workflow-group "" --org "" -- sg-payload.json +# Run once per project file (group tfc- must already exist): +./sg-cli workflow create --bulk --workflow-group "tfc-" --org "" sg-payload..json ``` -if you want to update a workflow with different details, please re-run the sg-cli command with the modified sg-payload.json and your workflow will be updated with the new details, as long as the ResourceName (Workflow name) remains the same. +To update workflows with different details, re-run the sg-cli command with the modified payload file; workflows are updated as long as the `ResourceName` (workflow name) stays the same. Add `--dry-run` to preview a payload without applying. -```shell -./sg-cli workflow create --bulk --workflow-group "" --org "" -- sg-payload.json -``` +## Notes and limitations + +- **Workflow groups.** Each TFC project imports into an SG workflow group `tfc-`, created via the API if missing (disable with `--no-create-groups`). Override the target group per project in `.sg/workflow-groups.json`; override groups must already exist. +- **Variable Sets are migrated** (the `enrich` phase) — global, project-, and workspace-scoped sets are resolved per workspace with TFC precedence (priority sets override workspace vars; otherwise workspace vars win). **Sensitive** set variables can't be read from the API, so they're skipped and reported — recreate them as StackGuardian secrets. +- **Sensitive variables are skipped.** TFC never returns sensitive values via the API, so they are omitted from the payload and listed in `migration-summary.md`. Recreate them as StackGuardian secrets. +- **Terraform version fallback.** Workspaces set to `latest` or a version constraint (or running an engine SG can't map) use `SGDefaultTerraformVersion`. Override per workspace via `workspaceOverrides`. +- **State export is idempotent.** Re-running `terraform apply` only pulls state for workspaces not yet exported. Set `forceStateRefresh = true` to re-pull everything. Workspaces not using `remote` execution may export incomplete state — see the summary. +- **Workflow naming.** `ResourceName` currently mirrors the TFC workspace name. Confirm it satisfies StackGuardian's naming rules before import. From 5ea39ab356a362dd75cdf58a1415792663f48aa6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= Date: Mon, 7 Sep 2026 07:57:06 +0200 Subject: [PATCH 05/30] Reformat scripts and example payload Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01YKBeQrpJgR9DVFtR83myqX --- scripts/convert_hcl_to_json.sh | 5 ++- scripts/enrich_variable_sets.sh | 18 ++++++---- sg-migrate.sh | 6 ++-- transformer/terraform-cloud/data.tf | 2 +- .../terraform-cloud/example_payload.jsonc | 36 +++++++++++++++---- 5 files changed, 48 insertions(+), 19 deletions(-) diff --git a/scripts/convert_hcl_to_json.sh b/scripts/convert_hcl_to_json.sh index 0446805..cb778f5 100755 --- a/scripts/convert_hcl_to_json.sh +++ b/scripts/convert_hcl_to_json.sh @@ -6,7 +6,10 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/tools.sh" # Detail lines are shown only in verbose mode; warnings always show. -log() { [ "${SG_VERBOSE:-0}" = "1" ] || return 0; printf '%s[convert]%s %s\n' "$C_CYAN" "$C_RESET" "$*" >&2; } +log() { + [ "${SG_VERBOSE:-0}" = "1" ] || return 0 + printf '%s[convert]%s %s\n' "$C_CYAN" "$C_RESET" "$*" >&2 +} INPUT_FILE_JSON="${1:-}" if [ -z "$INPUT_FILE_JSON" ]; then diff --git a/scripts/enrich_variable_sets.sh b/scripts/enrich_variable_sets.sh index 340548f..d7bbc6b 100755 --- a/scripts/enrich_variable_sets.sh +++ b/scripts/enrich_variable_sets.sh @@ -25,7 +25,10 @@ fi HOST="${SG_TFC_HOSTNAME:-app.terraform.io}" API="https://$HOST/api/v2" -command -v curl >/dev/null 2>&1 || { sg_err "curl is required for variable-set enrichment"; exit 1; } +command -v curl >/dev/null 2>&1 || { + sg_err "curl is required for variable-set enrichment" + exit 1 +} JQ_BIN="$(sg_resolve jq sg_ensure_jq)" # TFC token (same sources as state export): credentials file or TFE_TOKEN. @@ -48,7 +51,8 @@ fetch_all() { : >"$WORK/acc.ndjson" while :; do if ! curl -fsS "${AUTH[@]}" "$API/$path?page%5Bsize%5D=100&page%5Bnumber%5D=$page" >"$WORK/page.json"; then - sg_err "TFC API request failed: $path"; return 1 + sg_err "TFC API request failed: $path" + return 1 fi "$JQ_BIN" -c '.data[]?' "$WORK/page.json" >>"$WORK/acc.ndjson" next="$("$JQ_BIN" -r '.meta.pagination."next-page" // empty' "$WORK/page.json" 2>/dev/null || true)" @@ -61,17 +65,17 @@ fetch_all() { sg_log "fetching workspaces and variable sets from $HOST (org: $ORG)..." # name -> {id, project} -fetch_all "organizations/$ORG/workspaces" \ - | "$JQ_BIN" '[.[] | {name: .attributes.name, id: .id, project: (.relationships.project.data.id // "")}]' \ - >"$WORK/workspaces.json" || exit 1 +fetch_all "organizations/$ORG/workspaces" | + "$JQ_BIN" '[.[] | {name: .attributes.name, id: .id, project: (.relationships.project.data.id // "")}]' \ + >"$WORK/workspaces.json" || exit 1 # Each set with its scope + variables. : >"$WORK/sets.ndjson" sets_raw="$(fetch_all "organizations/$ORG/varsets")" || exit 1 echo "$sets_raw" | "$JQ_BIN" -c '.[]' | while IFS= read -r s; do sid="$(echo "$s" | "$JQ_BIN" -r '.id')" - vars="$(curl -fsS "${AUTH[@]}" "$API/varsets/$sid/relationships/vars" 2>/dev/null \ - | "$JQ_BIN" -c '[.data[]? | {key: .attributes.key, value: (.attributes.value // ""), category: .attributes.category, sensitive: (.attributes.sensitive // false), hcl: (.attributes.hcl // false)}]' 2>/dev/null || echo '[]')" + vars="$(curl -fsS "${AUTH[@]}" "$API/varsets/$sid/relationships/vars" 2>/dev/null | + "$JQ_BIN" -c '[.data[]? | {key: .attributes.key, value: (.attributes.value // ""), category: .attributes.category, sensitive: (.attributes.sensitive // false), hcl: (.attributes.hcl // false)}]' 2>/dev/null || echo '[]')" echo "$s" | "$JQ_BIN" -c --argjson vars "$vars" '{ name: .attributes.name, global: (.attributes.global // false), diff --git a/sg-migrate.sh b/sg-migrate.sh index d39287d..c295768 100755 --- a/sg-migrate.sh +++ b/sg-migrate.sh @@ -21,9 +21,9 @@ BUILD=0 ARGS=() for a in "$@"; do case "$a" in - --native | --local) NATIVE=1 ;; - --build) BUILD=1 ;; - *) ARGS+=("$a") ;; + --native | --local) NATIVE=1 ;; + --build) BUILD=1 ;; + *) ARGS+=("$a") ;; esac done diff --git a/transformer/terraform-cloud/data.tf b/transformer/terraform-cloud/data.tf index be241df..5925934 100644 --- a/transformer/terraform-cloud/data.tf +++ b/transformer/terraform-cloud/data.tf @@ -20,4 +20,4 @@ data "tfe_variables" "data" { data "tfe_projects" "data" { organization = var.tfOrg -} \ No newline at end of file +} diff --git a/transformer/terraform-cloud/example_payload.jsonc b/transformer/terraform-cloud/example_payload.jsonc index cdc11b0..60a67d9 100644 --- a/transformer/terraform-cloud/example_payload.jsonc +++ b/transformer/terraform-cloud/example_payload.jsonc @@ -26,10 +26,15 @@ "ERRORED": [] // list of emails to notify } }, - "wfChaining": { "COMPLETED": [], "ERRORED": [] } + "wfChaining": { + "COMPLETED": [], + "ERRORED": [] + } }, "ResourceName": "", // workspace name - "RunnerConstraints": { "type": "" }, // type should be "shared" or "private" i.e. "RunnerConstraints": { "type": "private", "names":["runner-group-name"]} + "RunnerConstraints": { + "type": "" + }, // type should be "shared" or "private" i.e. "RunnerConstraints": { "type": "private", "names":["runner-group-name"]} "Tags": [], // workflow tags "TerraformConfig": { "managedTerraformState": true, // managed state from StackGuardian @@ -37,7 +42,10 @@ }, "UserSchedules": [], "VCSConfig": { - "iacInputData": { "data": {}, "schemaType": "RAW_JSON" }, // data key consists of key value pairs { "env1" : "secret} as terraform environment variables + "iacInputData": { + "data": {}, + "schemaType": "RAW_JSON" + }, // data key consists of key value pairs { "env1" : "secret} as terraform environment variables "iacVCSConfig": { "customSource": { "config": { @@ -67,10 +75,24 @@ "gh_check": true, "gl_pipeline": true, "post_comments": true, - "push": { "createWfRun": { "enabled": true } }, // run on push to tracked branch - "pull_request_opened": { "createWfRun": { "enabled": true } }, // TFC speculative plan - "pull_request_modified": { "createWfRun": { "enabled": true } }, + "push": { + "createWfRun": { + "enabled": true + } + }, // run on push to tracked branch + "pull_request_opened": { + "createWfRun": { + "enabled": true + } + }, // TFC speculative plan + "pull_request_modified": { + "createWfRun": { + "enabled": true + } + }, "file_triggers_enabled": true, // from TFC file_triggers_enabled - "file_trigger_patterns": ["path/to/workdir/*"] // TFC trigger_patterns / trigger_prefixes / working_directory + "file_trigger_patterns": [ + "path/to/workdir/*" + ] // TFC trigger_patterns / trigger_prefixes / working_directory } } From 1af25ebaab3d6ef2c89230e72977d46dd4594284 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= Date: Mon, 7 Sep 2026 07:58:34 +0200 Subject: [PATCH 06/30] fix: fall back to default terraform version above SG ceiling on import sg-cli exits 0 even when individual workflows fail, so the importer now parses its output. Workflows rejected because their pinned Terraform version is above SG's managed ceiling (1.5.7, last MPL/FOSS release) are re-imported with SGDefaultTerraformVersion, the payload is patched in place, and each case is logged to export/terraform-version-fallbacks.log with a printed notice. Any other per-workflow failure now fails the run. Also: sg_retry captured the wrong exit status (always 0), definitive HTTP 4xx responses are no longer retried, the trigger pass skips workflows that were never created, and failure output no longer echoes the API token. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01YKBeQrpJgR9DVFtR83myqX --- README.md | 3 +- scripts/migrate.sh | 337 ++++++++++++++---- scripts/tools.sh | 114 ++++-- .../terraform-cloud/terraform.tfvars.example | 20 +- transformer/terraform-cloud/variables.tf | 4 +- 5 files changed, 360 insertions(+), 118 deletions(-) diff --git a/README.md b/README.md index 0b884fe..63819d2 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,7 @@ A new `export` folder should have been created, containing: - One payload file **per TFC project**, named `sg-payload..json`. Each contains the workflow definitions for the workspaces in that project, and is imported into its own StackGuardian workflow group. - `migration-summary.md` (and `migration-summary.json`) — a report of what was migrated and what needs manual attention: skipped sensitive variables, Terraform-version fallbacks, renamed workspaces, and workspaces whose state could not be exported. **Read this before importing.** +- `terraform-version-fallbacks.log` (written by the `import` phase) — workflows that were created with `SGDefaultTerraformVersion` because their pinned version is above StackGuardian's managed ceiling; see _Notes and limitations_. - The `states` folder with the Terraform state for each workspace, if state export was enabled. - `state-export-failures.log`, if any workspace's state could not be pulled. @@ -161,6 +162,6 @@ To update workflows with different details, re-run the sg-cli command with the m - **Workflow groups.** Each TFC project imports into an SG workflow group `tfc-`, created via the API if missing (disable with `--no-create-groups`). Override the target group per project in `.sg/workflow-groups.json`; override groups must already exist. - **Variable Sets are migrated** (the `enrich` phase) — global, project-, and workspace-scoped sets are resolved per workspace with TFC precedence (priority sets override workspace vars; otherwise workspace vars win). **Sensitive** set variables can't be read from the API, so they're skipped and reported — recreate them as StackGuardian secrets. - **Sensitive variables are skipped.** TFC never returns sensitive values via the API, so they are omitted from the payload and listed in `migration-summary.md`. Recreate them as StackGuardian secrets. -- **Terraform version fallback.** Workspaces set to `latest` or a version constraint (or running an engine SG can't map) use `SGDefaultTerraformVersion`. Override per workspace via `workspaceOverrides`. +- **Terraform version fallback (FOSS ceiling).** Workspaces set to `latest` or a version constraint use `SGDefaultTerraformVersion` at export time. Pinned versions are carried over as-is and tried first at import, so a custom runtime image or private runner that ships that binary keeps working. StackGuardian's _managed_ runtimes only go up to **1.5.7**, the last MPL-licensed (FOSS) Terraform release; newer versions are BSL-licensed and are not bundled. When the API rejects a workflow for that reason, the importer **automatically re-imports it with `SGDefaultTerraformVersion`**, patches the payload file to match, prints a notice, and records each case in `export/terraform-version-fallbacks.log`. Those workflows run a different Terraform than they did in TFC, so check compatibility before the first run. To keep a newer version, set `workspaceOverrides[].terraformVersion` to a binary path mounted from a private runner (or use a custom runtime container template) and re-import. - **State export is idempotent.** Re-running `terraform apply` only pulls state for workspaces not yet exported. Set `forceStateRefresh = true` to re-pull everything. Workspaces not using `remote` execution may export incomplete state — see the summary. - **Workflow naming.** `ResourceName` currently mirrors the TFC workspace name. Confirm it satisfies StackGuardian's naming rules before import. diff --git a/scripts/migrate.sh b/scripts/migrate.sh index 4adc693..57bd964 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -72,7 +72,10 @@ Environment: EOF } -die() { sg_err "$*"; exit 1; } +die() { + sg_err "$*" + exit 1 +} throttle() { while [ "$(jobs -rp | wc -l | tr -d ' ')" -ge "$1" ]; do sleep 0.2; done; } @@ -81,12 +84,16 @@ throttle() { while [ "$(jobs -rp | wc -l | tr -d ' ')" -ge "$1" ]; do sleep 0.2; # a non-TTY and don't scatter spinner output across the terminal), then flushed # as a clean labeled block in submission order. Returns non-zero if any failed. run_parallel() { - local fn="$1" max="$2"; shift 2 + local fn="$1" max="$2" + shift 2 local statusdir i=0 rc=0 item statusdir="$(mktemp -d)" for item in "$@"; do throttle "$max" - ( "$fn" "$item" >"$statusdir/$i.out" 2>&1; echo "$?" >"$statusdir/$i.rc" ) & + ( + "$fn" "$item" >"$statusdir/$i.out" 2>&1 + echo "$?" >"$statusdir/$i.rc" + ) & i=$((i + 1)) done wait @@ -108,7 +115,12 @@ payload_files() { shopt -u nullglob } -seg_of() { local b; b="$(basename "$1")"; b="${b#sg-payload.}"; echo "${b%.json}"; } +seg_of() { + local b + b="$(basename "$1")" + b="${b#sg-payload.}" + echo "${b%.json}" +} cmd_init() { sg_step "Phase: init" @@ -123,6 +135,7 @@ cmd_init() { sg_success "init complete" } + # --- StackGuardian API helpers (workflow groups) --------------------------- # group_for -> the SG workflow group for a project segment: an entry @@ -170,23 +183,60 @@ do_set_triggers() { n="$("$JQ_BIN" 'length' "$f")" for ((i = 0; i < n; i++)); do if [ "$("$JQ_BIN" -r --argjson i "$i" '(.[$i].VCSTriggers // null) != null' "$f")" != "true" ]; then - skip=$((skip + 1)); continue + skip=$((skip + 1)) + continue fi wf="$("$JQ_BIN" -r --argjson i "$i" '.[$i].ResourceName' "$f")" + # A workflow that failed to import has nothing to attach triggers to. + if [ "$(sg_http_code GET "$SG_BASE_URL/api/v1/orgs/$ORG/wfgrps/$grp/wfs/$wf/")" != "200" ]; then + sg_warn " $grp/$wf does not exist in SG (import failed?) — skipping triggers" + rc=1 + continue + fi body="$("$JQ_BIN" -c --argjson i "$i" '{VCSConfig: .[$i].VCSConfig, VCSTriggers: .[$i].VCSTriggers}' "$f")" - if sg_retry "$RETRIES" "$RETRY_BASE" -- \ - curl -fsS -X POST \ - -H "Authorization: apikey $SG_API_TOKEN" -H "Content-Type: application/json" \ - -d "$body" "$(wf_triggers_endpoint "$grp" "$wf")" >/dev/null; then + if SG_NO_RETRY_RC=22 sg_retry "$RETRIES" "$RETRY_BASE" -- \ + sg_api_post "$(wf_triggers_endpoint "$grp" "$wf")" "$body"; then set=$((set + 1)) else - sg_warn " vcs triggers failed: $grp/$wf"; rc=1 + sg_warn " vcs triggers failed: $grp/$wf" + rc=1 fi done sg_log "$(basename "$f"): set triggers on $set workflow(s) (skipped $skip without triggers)" return "$rc" } +# sg_http_code — prints the HTTP status (000 on network error). +sg_http_code() { + curl -sS -o /dev/null -w '%{http_code}' -X "$1" -H "Authorization: apikey $SG_API_TOKEN" "$2" 2>/dev/null || echo 000 +} + +# sg_api_post — POST to the SG API. Exit 0 on 2xx, 22 on a +# definitive 4xx (not retryable; response echoed), 1 on 5xx/network (retryable). +sg_api_post() { + local url="$1" body="$2" tmp code + tmp="$(mktemp)" + code="$(curl -sS -o "$tmp" -w '%{http_code}' -X POST \ + -H "Authorization: apikey $SG_API_TOKEN" -H "Content-Type: application/json" \ + -d "$body" "$url" 2>/dev/null || echo 000)" + case "$code" in + 2*) + rm -f "$tmp" + return 0 + ;; + 4*) + sg_err " HTTP $code from ${url#"$SG_BASE_URL"}: $(head -c 400 "$tmp")" + rm -f "$tmp" + return 22 + ;; + *) + sg_warn " HTTP $code from ${url#"$SG_BASE_URL"}" + rm -f "$tmp" + return 1 + ;; + esac +} + # set_triggers_pass — run do_set_triggers over all payload files (assumes # JQ_BIN/SG_API_TOKEN/ORG are already set up by the caller). set_triggers_pass() { @@ -200,7 +250,8 @@ set_triggers_pass() { if run_parallel do_set_triggers "$CONC" "${PF[@]}"; then sg_success "vcs triggers registered" else - sg_err "one or more VCS trigger registrations failed (re-run: $0 triggers)"; return 1 + sg_err "one or more VCS trigger registrations failed (re-run: $0 triggers)" + return 1 fi } @@ -242,22 +293,23 @@ cmd_apply() { jqdir="$(dirname "$(sg_resolve jq sg_ensure_jq)")" if [ "$VERBOSE" -eq 1 ]; then - ( cd "$TRANSFORMER_DIR" && export PATH="$jqdir:$PATH" TF_IN_AUTOMATION=1 \ - && terraform init -input=false \ - && terraform apply -auto-approve -compact-warnings -parallelism="$TF_PARALLELISM" -var-file=terraform.tfvars ) || rc=$? + (cd "$TRANSFORMER_DIR" && export PATH="$jqdir:$PATH" TF_IN_AUTOMATION=1 && + terraform init -input=false && + terraform apply -auto-approve -compact-warnings -parallelism="$TF_PARALLELISM" -var-file=terraform.tfvars) || rc=$? else # Quiet: capture terraform's verbose plan/output; surface only progress, the # final summary, and (on failure) the captured log. tflog="$(mktemp)" sg_log "initializing terraform (providers)..." - ( cd "$TRANSFORMER_DIR" && export PATH="$jqdir:$PATH" TF_IN_AUTOMATION=1 && terraform init -input=false -no-color ) >"$tflog" 2>&1 || rc=$? + (cd "$TRANSFORMER_DIR" && export PATH="$jqdir:$PATH" TF_IN_AUTOMATION=1 && terraform init -input=false -no-color) >"$tflog" 2>&1 || rc=$? if [ "$rc" -eq 0 ]; then sg_log "reading workspaces, generating payloads, exporting state..." - ( cd "$TRANSFORMER_DIR" && export PATH="$jqdir:$PATH" TF_IN_AUTOMATION=1 \ - && terraform apply -auto-approve -compact-warnings -no-color -parallelism="$TF_PARALLELISM" -var-file=terraform.tfvars ) >"$tflog" 2>&1 || rc=$? + (cd "$TRANSFORMER_DIR" && export PATH="$jqdir:$PATH" TF_IN_AUTOMATION=1 && + terraform apply -auto-approve -compact-warnings -no-color -parallelism="$TF_PARALLELISM" -var-file=terraform.tfvars) >"$tflog" 2>&1 || rc=$? fi if [ "$rc" -ne 0 ]; then - sg_err "terraform failed (rc=$rc):"; cat "$tflog" >&2 + sg_err "terraform failed (rc=$rc):" + cat "$tflog" >&2 else grep -E '^(Apply complete|No changes)' "$tflog" | sed 's/^/ /' >&2 || true fi @@ -293,7 +345,8 @@ cmd_convert() { if run_parallel do_convert "$CONC" "${PF[@]}"; then sg_success "converted ${#PF[@]} payload(s)" else - sg_err "conversion failed for one or more payloads"; return 1 + sg_err "conversion failed for one or more payloads" + return 1 fi } @@ -304,17 +357,102 @@ cmd_validate() { if "$SCRIPT_DIR/validate_payload.sh" "${PF[@]}"; then sg_success "all ${#PF[@]} payload(s) valid" else - sg_err "validation failed"; return 1 + sg_err "validation failed" + return 1 fi } +# sgcli_bulk — run the bulk create, teeing output to . +# sg-cli exits 0 even when individual workflows fail, so callers must inspect +# the output ("Failed to create : ..." lines). +sgcli_bulk() { + "$SGCLI_BIN" workflow create --bulk --workflow-group "$1" --org "$ORG" "$2" 2>&1 | tee "$3" + return "${PIPESTATUS[0]}" +} + +# Regex for the API's rejection of a Terraform version above SG's managed +# ceiling. SG bundles managed runtimes only up to the last MPL-licensed (FOSS) +# Terraform release; newer versions are BSL and are not shipped. +TF_CEILING_RE='Failed to create ([^:]+): 400: .*above the highest managed version \(([0-9.]+)\)' + +# names_json — JSON array of the given names (for jq --argjson). +names_json() { printf '%s\n' "$@" | "$JQ_BIN" -R . | "$JQ_BIN" -s .; } + +# do_import — bulk-import one file. Workflows rejected because their +# Terraform version is above the SG ceiling are re-imported with +# SG_DEFAULT_TF_VERSION (the payload file is patched in place so re-runs and +# the trigger pass see what was actually imported); each fallback is appended to +# terraform-version-fallbacks.log. Any other per-workflow failure fails the file. do_import() { - local f="$1" seg grp + local f="$1" seg grp out rc=0 ceiling="" failed=() fb=() name line tmp names seg="$(seg_of "$f")" grp="$(group_for "$seg")" sg_log "importing $(basename "$f") -> $grp" - sg_retry "$RETRIES" "$RETRY_BASE" -- \ - "$SGCLI_BIN" workflow create --bulk --workflow-group "$grp" --org "$ORG" "$f" + out="$(mktemp)" + sg_retry "$RETRIES" "$RETRY_BASE" -- sgcli_bulk "$grp" "$f" "$out" || rc=1 + + while IFS= read -r line; do + if [[ "$line" =~ $TF_CEILING_RE ]]; then + fb+=("${BASH_REMATCH[1]}") + ceiling="${BASH_REMATCH[2]}" + elif [[ "$line" =~ Failed\ to\ create\ ([^:]+): ]]; then + failed+=("${BASH_REMATCH[1]}") + fi + done <"$out" + rm -f "$out" + + if [ "${#fb[@]}" -gt 0 ]; then + sg_warn "${#fb[@]} workflow(s) pinned above SG's managed Terraform ceiling ($ceiling); re-importing with $SG_DEFAULT_TF_VERSION" + names="$(names_json "${fb[@]}")" + tmp="$(mktemp "$EXPORT_DIR/.fallback.$seg.XXXXXX")" + # Patch the affected workflows in the payload and re-import only those. + "$JQ_BIN" --arg v "$SG_DEFAULT_TF_VERSION" --argjson names "$names" \ + 'map(if (.ResourceName as $n | $names | index($n)) != null then .TerraformConfig.terraformVersion = $v else . end)' "$f" >"$tmp.full" && + "$JQ_BIN" --argjson names "$names" \ + 'map(select(.ResourceName as $n | $names | index($n) != null))' "$tmp.full" >"$tmp" || + { + rm -f "$tmp" "$tmp.full" + die "could not patch $(basename "$f") for the Terraform version fallback" + } + out="$(mktemp)" + sg_retry "$RETRIES" "$RETRY_BASE" -- sgcli_bulk "$grp" "$tmp" "$out" || rc=1 + for name in "${fb[@]}"; do + if grep -q "Failed to create $name:" "$out"; then + failed+=("$name") + else + line="$("$JQ_BIN" -r --arg n "$name" '.[] | select(.ResourceName == $n) | .TerraformConfig.terraformVersion' "$f")" + printf '%s/%s: %s -> %s (above SG managed ceiling %s)\n' "$grp" "$name" "$line" "$SG_DEFAULT_TF_VERSION" "$ceiling" >>"$EXPORT_DIR/terraform-version-fallbacks.log" + fi + done + rm -f "$out" + mv -f "$tmp.full" "$f" + rm -f "$tmp" + fi + + if [ "${#failed[@]}" -gt 0 ]; then + sg_err "$(basename "$f"): ${#failed[@]} workflow(s) failed to import: ${failed[*]}" + return 1 + fi + return "$rc" +} + +# Print the customer-facing notice when any workflow fell back to the default +# Terraform version during this import. +tf_fallback_notice() { + local log="$EXPORT_DIR/terraform-version-fallbacks.log" + [ -s "$log" ] || return 0 + sg_warn "Terraform version fallback applied to $(wc -l <"$log" | tr -d ' ') workflow(s):" + sed 's/^/ /' "$log" >&2 + cat >&2 <].terraformVersion to a binary path mounted from a + private runner, or use a custom runtime container template + (wfStepTemplateRevisionId), and re-import. +NOTICE } cmd_import() { @@ -340,23 +478,32 @@ cmd_import() { count="$("$JQ_BIN" 'length' "$f")" override="" [ -f "$MAPPING" ] && override="$("$JQ_BIN" -r --arg k "$seg" '.[$k] // empty' "$MAPPING" 2>/dev/null || true)" - if [ -n "$override" ]; then grp="$override"; is_override=1; else grp="tfc-$seg"; is_override=0; fi + if [ -n "$override" ]; then + grp="$override" + is_override=1 + else + grp="tfc-$seg" + is_override=0 + fi code="$(wfgroup_http_code "$grp")" case "$code" in - 200) status="${C_GREEN}exists${C_RESET}" ;; - 404) - if [ "$is_override" -eq 1 ]; then - status="${C_RED}missing!${C_RESET}"; fail=1 - elif [ "$CREATE_GROUPS" -eq 1 ]; then - status="${C_YELLOW}create${C_RESET}" - case "$to_create" in *" $grp "*) ;; *) to_create="$to_create$grp " ;; esac - else - status="${C_RED}missing!${C_RESET}"; fail=1 - fi ;; - 401 | 403) die "auth failed (HTTP $code) for org '$ORG' — check SG_API_TOKEN" ;; - 000) die "could not reach $SG_BASE_URL" ;; - *) die "unexpected HTTP $code checking group '$grp'" ;; + 200) status="${C_GREEN}exists${C_RESET}" ;; + 404) + if [ "$is_override" -eq 1 ]; then + status="${C_RED}missing!${C_RESET}" + fail=1 + elif [ "$CREATE_GROUPS" -eq 1 ]; then + status="${C_YELLOW}create${C_RESET}" + case "$to_create" in *" $grp "*) ;; *) to_create="$to_create$grp " ;; esac + else + status="${C_RED}missing!${C_RESET}" + fail=1 + fi + ;; + 401 | 403) die "auth failed (HTTP $code) for org '$ORG' — check SG_API_TOKEN" ;; + 000) die "could not reach $SG_BASE_URL" ;; + *) die "unexpected HTTP $code checking group '$grp'" ;; esac printf ' %-34s %-26s %-9s %s\n' "$(basename "$f")" "$grp" "$count" "$status" >&2 done @@ -377,40 +524,72 @@ cmd_import() { done SGCLI_BIN="$(sg_resolve sg-cli sg_ensure_sgcli)" + # Fallback Terraform version for workflows the API rejects as above the + # managed ceiling: SGDefaultTerraformVersion from terraform.tfvars, else 1.5.7. + SG_DEFAULT_TF_VERSION="${SG_DEFAULT_TF_VERSION:-}" + if [ -z "$SG_DEFAULT_TF_VERSION" ] && [ -f "$TFVARS" ]; then + SG_DEFAULT_TF_VERSION="$("$(sg_resolve hcl2json sg_ensure_hcl2json)" "$TFVARS" | "$JQ_BIN" -r '.SGDefaultTerraformVersion // empty')" + fi + SG_DEFAULT_TF_VERSION="${SG_DEFAULT_TF_VERSION:-TERRAFORM-1.5.7}" + rm -f "$EXPORT_DIR/terraform-version-fallbacks.log" + sg_log "importing ${#PF[@]} payload(s), up to $CONC in parallel (retries: $RETRIES)" - if run_parallel do_import "$CONC" "${PF[@]}"; then + local import_rc=0 + run_parallel do_import "$CONC" "${PF[@]}" || import_rc=1 + tf_fallback_notice + if [ "$import_rc" -eq 0 ]; then sg_success "import complete (${#PF[@]} payload(s))" else - sg_err "one or more imports failed"; return 1 + sg_err "one or more workflows failed to import (see above); VCS triggers are still registered for the ones that succeeded" fi # VCS triggers are not accepted by the bulk create API; register them in a # second pass against the dedicated webhooks endpoint (skip with --no-vcs-triggers). if [ "$VCS_TRIGGERS" -eq 1 ]; then - set_triggers_pass + set_triggers_pass || import_rc=1 fi + return "$import_rc" } CMD="" while [ $# -gt 0 ]; do case "$1" in - -y | --yes) ASSUME_YES=1 ;; - --org) ORG="$2"; shift ;; - --org=*) ORG="${1#*=}" ;; - --export-dir) EXPORT_DIR="$2"; shift ;; - --export-dir=*) EXPORT_DIR="${1#*=}" ;; - --mapping) MAPPING="$2"; shift ;; - --mapping=*) MAPPING="${1#*=}" ;; - --concurrency) CONC="$2"; shift ;; - --concurrency=*) CONC="${1#*=}" ;; - --no-create-groups) CREATE_GROUPS=0 ;; - --no-variable-sets) ENRICH_VARSETS=0 ;; - --no-vcs-triggers) VCS_TRIGGERS=0 ;; - -v | --verbose) VERBOSE=1 ;; - --all) PURGE=1 ;; - -h | --help) usage; exit 0 ;; - init | apply | enrich | convert | validate | import | triggers | all | clean) CMD="$1" ;; - *) echo "Unknown argument: $1" >&2; usage; exit 1 ;; + -y | --yes) ASSUME_YES=1 ;; + --org) + ORG="$2" + shift + ;; + --org=*) ORG="${1#*=}" ;; + --export-dir) + EXPORT_DIR="$2" + shift + ;; + --export-dir=*) EXPORT_DIR="${1#*=}" ;; + --mapping) + MAPPING="$2" + shift + ;; + --mapping=*) MAPPING="${1#*=}" ;; + --concurrency) + CONC="$2" + shift + ;; + --concurrency=*) CONC="${1#*=}" ;; + --no-create-groups) CREATE_GROUPS=0 ;; + --no-variable-sets) ENRICH_VARSETS=0 ;; + --no-vcs-triggers) VCS_TRIGGERS=0 ;; + -v | --verbose) VERBOSE=1 ;; + --all) PURGE=1 ;; + -h | --help) + usage + exit 0 + ;; + init | apply | enrich | convert | validate | import | triggers | all | clean) CMD="$1" ;; + *) + echo "Unknown argument: $1" >&2 + usage + exit 1 + ;; esac shift done @@ -418,26 +597,26 @@ CMD="${CMD:-all}" export SG_VERBOSE="$VERBOSE" case "$CMD" in - init) cmd_init ;; - clean) cmd_clean ;; - apply) cmd_apply ;; - enrich) cmd_enrich ;; - convert) cmd_convert ;; - validate) cmd_validate ;; - import) cmd_import ;; - triggers) cmd_triggers ;; - all) - if [ ! -f "$TFVARS" ]; then - cmd_init - die "Edit $(sg_rel "$TFVARS"), then re-run '$0 all'." - fi - # Fail fast on import prerequisites before the (long) apply. - [ -n "${SG_API_TOKEN:-}" ] || die "SG_API_TOKEN is not set (needed for import)." - [ -n "$ORG" ] || die "StackGuardian org not set (use --org or SG_ORG)." - cmd_apply - if [ "$ENRICH_VARSETS" -eq 1 ]; then cmd_enrich; fi - cmd_convert - cmd_validate - cmd_import - ;; +init) cmd_init ;; +clean) cmd_clean ;; +apply) cmd_apply ;; +enrich) cmd_enrich ;; +convert) cmd_convert ;; +validate) cmd_validate ;; +import) cmd_import ;; +triggers) cmd_triggers ;; +all) + if [ ! -f "$TFVARS" ]; then + cmd_init + die "Edit $(sg_rel "$TFVARS"), then re-run '$0 all'." + fi + # Fail fast on import prerequisites before the (long) apply. + [ -n "${SG_API_TOKEN:-}" ] || die "SG_API_TOKEN is not set (needed for import)." + [ -n "$ORG" ] || die "StackGuardian org not set (use --org or SG_ORG)." + cmd_apply + if [ "$ENRICH_VARSETS" -eq 1 ]; then cmd_enrich; fi + cmd_convert + cmd_validate + cmd_import + ;; esac diff --git a/scripts/tools.sh b/scripts/tools.sh index f3b568f..901a9ff 100755 --- a/scripts/tools.sh +++ b/scripts/tools.sh @@ -20,34 +20,48 @@ SG_YAJSV_VERSION="v1.4.1" # Colored logging — disabled when stderr is not a TTY, NO_COLOR is set, or # TERM=dumb, so piped/CI output stays clean. if [ -t 2 ] && [ -z "${NO_COLOR:-}" ] && [ "${TERM:-}" != "dumb" ]; then - C_RESET=$'\033[0m'; C_BOLD=$'\033[1m'; C_DIM=$'\033[2m' - C_RED=$'\033[31m'; C_GREEN=$'\033[32m'; C_YELLOW=$'\033[33m'; C_CYAN=$'\033[36m' + C_RESET=$'\033[0m' + C_BOLD=$'\033[1m' + C_DIM=$'\033[2m' + C_RED=$'\033[31m' + C_GREEN=$'\033[32m' + C_YELLOW=$'\033[33m' + C_CYAN=$'\033[36m' else - C_RESET=""; C_BOLD=""; C_DIM=""; C_RED=""; C_GREEN=""; C_YELLOW=""; C_CYAN="" + C_RESET="" + C_BOLD="" + C_DIM="" + C_RED="" + C_GREEN="" + C_YELLOW="" + C_CYAN="" fi # sg_rel — render a path relative to the repo root for readable logs # (operations still use absolute paths; this is display-only). sg_rel() { case "$1" in - "$SG_REPO_ROOT"/*) printf '%s' "${1#"$SG_REPO_ROOT"/}" ;; - "$SG_REPO_ROOT") printf '.' ;; - *) printf '%s' "$1" ;; + "$SG_REPO_ROOT"/*) printf '%s' "${1#"$SG_REPO_ROOT"/}" ;; + "$SG_REPO_ROOT") printf '.' ;; + *) printf '%s' "$1" ;; esac } -sg_log() { printf '%s[sg-migrate]%s %s\n' "$C_CYAN" "$C_RESET" "$*" >&2; } -sg_warn() { printf '%s[sg-migrate] WARN%s %s\n' "$C_YELLOW" "$C_RESET" "$*" >&2; } -sg_err() { printf '%s[sg-migrate] ERROR%s %s\n' "$C_RED$C_BOLD" "$C_RESET" "$*" >&2; } -sg_success() { printf '%s[sg-migrate] ✓%s %s\n' "$C_GREEN$C_BOLD" "$C_RESET" "$*" >&2; } -sg_step() { printf '\n%s==> %s%s\n' "$C_CYAN$C_BOLD" "$*" "$C_RESET" >&2; } -sg_dim() { printf '%s %s%s\n' "$C_DIM" "$*" "$C_RESET" >&2; } +sg_log() { printf '%s[sg-migrate]%s %s\n' "$C_CYAN" "$C_RESET" "$*" >&2; } +sg_warn() { printf '%s[sg-migrate] WARN%s %s\n' "$C_YELLOW" "$C_RESET" "$*" >&2; } +sg_err() { printf '%s[sg-migrate] ERROR%s %s\n' "$C_RED$C_BOLD" "$C_RESET" "$*" >&2; } +sg_success() { printf '%s[sg-migrate] ✓%s %s\n' "$C_GREEN$C_BOLD" "$C_RESET" "$*" >&2; } +sg_step() { printf '\n%s==> %s%s\n' "$C_CYAN$C_BOLD" "$*" "$C_RESET" >&2; } +sg_dim() { printf '%s %s%s\n' "$C_DIM" "$*" "$C_RESET" >&2; } sg_arch() { case "$(uname -m)" in - x86_64 | amd64) echo "amd64" ;; - aarch64 | arm64) echo "arm64" ;; - *) echo "Unsupported architecture: $(uname -m)" >&2; return 1 ;; + x86_64 | amd64) echo "amd64" ;; + aarch64 | arm64) echo "arm64" ;; + *) + echo "Unsupported architecture: $(uname -m)" >&2 + return 1 + ;; esac } @@ -55,15 +69,24 @@ sg_arch() { # Runs the command, retrying with exponential backoff (capped at 60s, with a # little jitter) until it succeeds or attempts are exhausted. Returns the # command's last exit code. Used to ride out transient API failures/rate limits. +# Exit codes listed in SG_NO_RETRY_RC (space-separated) are returned immediately +# (e.g. a definitive HTTP 4xx). Only the command name is echoed on failure so +# that tokens passed as arguments never land in the log. sg_retry() { - local max="$1" base="$2"; shift 2 + local max="$1" base="$2" + shift 2 [ "${1:-}" = "--" ] && shift local attempt=1 delay="$base" rc=0 while :; do - if "$@"; then return 0; fi + "$@" && return 0 rc=$? + case " ${SG_NO_RETRY_RC:-} " in *" $rc "*) + sg_err "$(basename "$1") failed (exit ${rc}, not retryable)" + return "$rc" + ;; + esac if [ "$attempt" -ge "$max" ]; then - sg_err "failed after ${attempt} attempt(s) (exit ${rc}): $*" + sg_err "$(basename "$1") failed after ${attempt} attempt(s) (exit ${rc})" return "$rc" fi local jitter=$((RANDOM % (base + 1))) @@ -80,8 +103,14 @@ sg_retry() { # Prints the path to use on stdout. sg_resolve() { local name="$1" ensure="$2" - if command -v "$name" >/dev/null 2>&1; then command -v "$name"; return 0; fi - if [ -n "$ensure" ]; then "$ensure"; return $?; fi + if command -v "$name" >/dev/null 2>&1; then + command -v "$name" + return 0 + fi + if [ -n "$ensure" ]; then + "$ensure" + return $? + fi echo "Required tool '$name' not found on PATH" >&2 return 1 } @@ -89,7 +118,10 @@ sg_resolve() { # sg_download sg_download() { local url="$1" dest="$2" - command -v curl >/dev/null 2>&1 || { echo "curl is required but not found" >&2; return 1; } + command -v curl >/dev/null 2>&1 || { + echo "curl is required but not found" >&2 + return 1 + } mkdir -p "$(dirname "$dest")" if ! curl -fsSL -o "$dest" "$url"; then echo "Failed to download: $url" >&2 @@ -101,7 +133,11 @@ sg_ensure_jq() { local bin="$SG_CACHE_BIN/jq" arch os if [ ! -x "$bin" ]; then arch=$(sg_arch) || return 1 - case "$(uname -s)" in Darwin) os="macos" ;; Linux) os="linux" ;; *) echo "Unsupported OS: $(uname -s)" >&2; return 1 ;; esac + case "$(uname -s)" in Darwin) os="macos" ;; Linux) os="linux" ;; *) + echo "Unsupported OS: $(uname -s)" >&2 + return 1 + ;; + esac sg_log "caching jq ${SG_JQ_VERSION}..." sg_download "https://github.com/jqlang/jq/releases/download/${SG_JQ_VERSION}/jq-${os}-${arch}" "$bin" || return 1 chmod +x "$bin" @@ -113,7 +149,11 @@ sg_ensure_hcl2json() { local bin="$SG_CACHE_BIN/hcl2json" arch os if [ ! -x "$bin" ]; then arch=$(sg_arch) || return 1 - case "$(uname -s)" in Darwin) os="darwin" ;; Linux) os="linux" ;; *) echo "Unsupported OS: $(uname -s)" >&2; return 1 ;; esac + case "$(uname -s)" in Darwin) os="darwin" ;; Linux) os="linux" ;; *) + echo "Unsupported OS: $(uname -s)" >&2 + return 1 + ;; + esac sg_log "caching hcl2json ${SG_HCL2JSON_VERSION}..." sg_download "https://github.com/tmccombs/hcl2json/releases/download/${SG_HCL2JSON_VERSION}/hcl2json_${os}_${arch}" "$bin" || return 1 chmod +x "$bin" @@ -125,7 +165,11 @@ sg_ensure_yajsv() { local bin="$SG_CACHE_BIN/yajsv" arch os if [ ! -x "$bin" ]; then arch=$(sg_arch) || return 1 - case "$(uname -s)" in Darwin) os="darwin" ;; Linux) os="linux" ;; *) echo "Unsupported OS: $(uname -s)" >&2; return 1 ;; esac + case "$(uname -s)" in Darwin) os="darwin" ;; Linux) os="linux" ;; *) + echo "Unsupported OS: $(uname -s)" >&2 + return 1 + ;; + esac if [ "$os" = "linux" ] && [ "$arch" = "arm64" ]; then echo "No yajsv prebuilt binary for linux/arm64; install yajsv manually." >&2 return 1 @@ -143,16 +187,30 @@ sg_ensure_yajsv() { sg_ensure_sgcli() { local bin="$SG_CACHE_BIN/sg-cli" os arch tmp realcli if [ ! -x "$bin" ]; then - case "$(uname -s)" in Darwin) os="Darwin" ;; Linux) os="Linux" ;; *) echo "Unsupported OS: $(uname -s)" >&2; return 1 ;; esac - case "$(uname -m)" in x86_64 | amd64) arch="x86_64" ;; aarch64 | arm64) arch="arm64" ;; *) echo "Unsupported architecture: $(uname -m)" >&2; return 1 ;; esac + case "$(uname -s)" in Darwin) os="Darwin" ;; Linux) os="Linux" ;; *) + echo "Unsupported OS: $(uname -s)" >&2 + return 1 + ;; + esac + case "$(uname -m)" in x86_64 | amd64) arch="x86_64" ;; aarch64 | arm64) arch="arm64" ;; *) + echo "Unsupported architecture: $(uname -m)" >&2 + return 1 + ;; + esac sg_log "caching sg-cli (latest release, ${os}/${arch})..." tmp=$(mktemp -d) if ! curl -fsSL "https://github.com/StackGuardian/sg-cli/releases/latest/download/sg-cli_${os}_${arch}.tar.gz" -o "$tmp/sg-cli.tar.gz"; then - echo "Failed to download sg-cli" >&2; rm -rf "$tmp"; return 1 + echo "Failed to download sg-cli" >&2 + rm -rf "$tmp" + return 1 fi tar -xzf "$tmp/sg-cli.tar.gz" -C "$tmp" realcli=$(find "$tmp" -maxdepth 2 -type f -name sg-cli | head -1) - [ -n "$realcli" ] || { echo "sg-cli binary not found in release archive" >&2; rm -rf "$tmp"; return 1; } + [ -n "$realcli" ] || { + echo "sg-cli binary not found in release archive" >&2 + rm -rf "$tmp" + return 1 + } mkdir -p "$SG_CACHE_BIN" cp "$realcli" "$bin" chmod +x "$bin" diff --git a/transformer/terraform-cloud/terraform.tfvars.example b/transformer/terraform-cloud/terraform.tfvars.example index 9454029..6b0024a 100644 --- a/transformer/terraform-cloud/terraform.tfvars.example +++ b/transformer/terraform-cloud/terraform.tfvars.example @@ -27,20 +27,24 @@ SGDefaultVCSAuthIntegrationID = "/integrations/github_com" # Integration to use to authenticate against your cloud provider SGDefaultDeploymentPlatformConfig = [ - { - "kind" : "AWS_RBAC", - "config" : { - "integrationId" : "/integrations/aws-dev-account", - "profileName" : "default" - } + { + "kind" : "AWS_RBAC", + "config" : { + "integrationId" : "/integrations/aws-dev-account", + "profileName" : "default" } - ] + } +] # Choose from: GITHUB_COM, BITBUCKET_ORG, GITLAB_COM, AZURE_DEVOPS, GIT_OTHER SGDefaultSourceConfigDestKind = "GITHUB_COM" # SG Terraform version used when a workspace's terraform_version is not a pinned -# semver (e.g. "latest" or a constraint), or runs an engine SG cannot map. +# semver (e.g. "latest" or a constraint). Pinned versions are carried over as-is; +# if the SG API rejects one as above the managed ceiling (1.5.7 - the last MPL/ +# FOSS Terraform release; newer versions are BSL and not bundled), the importer +# re-creates that workflow with this version and logs it in +# export/terraform-version-fallbacks.log. SGDefaultTerraformVersion = "TERRAFORM-1.5.7" # Pre-configure VCS triggers on each VCS-backed workflow, remapped from the diff --git a/transformer/terraform-cloud/variables.tf b/transformer/terraform-cloud/variables.tf index 52c6df6..3c3b960 100644 --- a/transformer/terraform-cloud/variables.tf +++ b/transformer/terraform-cloud/variables.tf @@ -79,7 +79,7 @@ variable "SGDefaultSourceConfigDestKind" { variable "SGDefaultTerraformVersion" { default = "TERRAFORM-1.5.7" - description = "SG Terraform version used when a workspace's terraform_version is not a pinned semver (e.g. 'latest' or a version constraint), or the workspace runs an engine SG cannot map. Use the SG-formatted value, e.g. TERRAFORM-1.5.7." + description = "SG Terraform version used when a workspace's terraform_version is not a pinned semver (e.g. 'latest' or a version constraint). Also used by the importer when the SG API rejects a pinned version as above the managed ceiling (1.5.7, the last MPL/FOSS release; newer versions are BSL and not bundled). Use the SG-formatted value, e.g. TERRAFORM-1.5.7." type = string } @@ -109,4 +109,4 @@ variable "workspaceOverrides" { extraEnvironmentVariables = optional(list(any), []) VCSTriggers = optional(any) })) -} \ No newline at end of file +} From 339aecb83e5c912fb33b8df77a528c598da61159 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= Date: Mon, 7 Sep 2026 07:59:13 +0200 Subject: [PATCH 07/30] feat: shell completion subcommand 'sg-migrate.sh completion bash|zsh' prints a completion script for the current shell session; 'init' prints the source line for the user's shell. Runs natively like 'clean'. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01YKBeQrpJgR9DVFtR83myqX --- README.md | 1 + scripts/migrate.sh | 98 ++++++++++++++++++++++++++++++++++++++++++++++ sg-migrate.sh | 4 +- 3 files changed, 101 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 63819d2..4c0d291 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,7 @@ That's it — no workflow-group mapping to fill in. Each TFC project is imported - Output is concise by default (terraform's plan/init noise is hidden; shown on error). Add `-v`/`--verbose` for full output. - Flags: `-y` skip the import prompt (CI), `--concurrency N` parallel jobs, `--org NAME`, `--no-create-groups` require groups to pre-exist, `--build` rebuild the image, `--native`/`--local` force a local run even when Docker is available. - Tuning via env: `SG_RETRIES`, `SG_TF_PARALLELISM`, `SG_NATIVE=1`. +- Tab completion for the current shell session: `source <(./sg-migrate.sh completion zsh)` (or `bash`). `init` prints this line for your shell. - TFC auth: set `TFE_TOKEN` (recommended — a long-lived token avoids re-running `terraform login`); otherwise the `terraform login` credentials file is mounted read-only into the container. SG/TFC tokens are passed as env vars. The manual, step-by-step flow below remains supported for fine-grained control and is what each phase runs under the hood (the helper scripts live in `scripts/`). diff --git a/scripts/migrate.sh b/scripts/migrate.sh index 57bd964..c765e63 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -46,6 +46,8 @@ Commands: all apply -> enrich -> convert -> validate -> import (default) clean Remove local working artifacts for a fresh start (export/, TF state, tool cache). Add --all to also remove config (terraform.tfvars, mapping). + completion Print a shell completion script for the current session: + \`source <($0 completion zsh)\` (or bash) Each TFC project maps to an SG workflow group named tfc-, created via the API if missing. Override a project's target group in .sg/workflow-groups.json @@ -132,9 +134,18 @@ cmd_init() { sg_log "$(sg_rel "$TFVARS") already exists" fi sg_log "workflow groups are created automatically as tfc-; no mapping needed" + completion_hint sg_success "init complete" } +# completion_hint — tell the user how to enable tab completion for their shell. +# A child process cannot register completions in the parent shell, so this only +# prints the one-liner (nothing is written to the user's rc files). +completion_hint() { + local sh + case "$(basename "${SHELL:-}")" in bash) sh=bash ;; *) sh=zsh ;; esac + sg_log "tab completion for this shell session: source <(./sg-migrate.sh completion $sh)" +} # --- StackGuardian API helpers (workflow groups) --------------------------- @@ -551,6 +562,89 @@ cmd_import() { return "$import_rc" } +# Single source of truth for shell completion (keep in sync with the parser below +# and the host-only flags in sg-migrate.sh). +SG_COMMANDS="init apply enrich convert validate import triggers all clean completion" +SG_OPTIONS="--org --export-dir --mapping --concurrency --no-create-groups --no-variable-sets --no-vcs-triggers --all -v --verbose -y --yes -h --help --native --local --build" + +# cmd_completion — print a completion script for sg-migrate.sh / +# migrate.sh to stdout. Both shells fall back to the basename when the command +# is invoked by path, so ./sg-migrate.sh completes too. +cmd_completion() { + local shell="${1:-}" + case "$shell" in + bash) + cat < enrich -> convert -> validate -> import (default)' + 'clean:Remove local working artifacts' + 'completion:Print a shell completion script' + ) + _arguments -s \\ + '--org[StackGuardian org for import]:org' \\ + '--export-dir[Payload/state output dir]:dir:_files -/' \\ + '--mapping[Project-segment -> group override map]:file:_files' \\ + '--concurrency[Max parallel jobs for convert/import]:n' \\ + '--no-create-groups[Require workflow groups to pre-exist]' \\ + '--no-variable-sets[Skip merging TFC Variable Sets]' \\ + '--no-vcs-triggers[Skip registering VCS triggers after import]' \\ + '--all[With clean: also remove config]' \\ + '(-v --verbose)'{-v,--verbose}'[Show full terraform/tool output]' \\ + '(-y --yes)'{-y,--yes}'[Skip the import confirmation prompt]' \\ + '(-h --help)'{-h,--help}'[Show help]' \\ + '(--native --local)'{--native,--local}'[Run natively instead of in Docker]' \\ + '--build[Rebuild the Docker image first]' \\ + '1:command:->cmd' \\ + '2:shell:->shell' + case "\$state" in + cmd) _describe -t commands 'command' cmds ;; + shell) [[ "\${words[CURRENT-1]}" == completion ]] && _values 'shell' bash zsh ;; + esac +} +compdef _sg_migrate sg-migrate.sh migrate.sh +ZSH + ;; + *) die "usage: $0 completion " ;; + esac +} + CMD="" while [ $# -gt 0 ]; do case "$1" in @@ -585,6 +679,10 @@ while [ $# -gt 0 ]; do exit 0 ;; init | apply | enrich | convert | validate | import | triggers | all | clean) CMD="$1" ;; + completion) + cmd_completion "${2:-}" + exit 0 + ;; *) echo "Unknown argument: $1" >&2 usage diff --git a/sg-migrate.sh b/sg-migrate.sh index c295768..e1138d6 100755 --- a/sg-migrate.sh +++ b/sg-migrate.sh @@ -27,9 +27,9 @@ for a in "$@"; do esac done -# 'clean' only touches the local filesystem — no container needed. +# 'clean' and 'completion' only touch the local shell/filesystem — no container. for a in ${ARGS[@]+"${ARGS[@]}"}; do - [ "$a" = "clean" ] && NATIVE=1 + case "$a" in clean | completion) NATIVE=1 ;; esac done if [ "$NATIVE" = "1" ] || ! command -v docker >/dev/null 2>&1; then From d740eb0314728df0354e426ff97ff22a75d48d2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= Date: Mon, 7 Sep 2026 07:59:13 +0200 Subject: [PATCH 08/30] docs: add CLAUDE.md Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01YKBeQrpJgR9DVFtR83myqX --- CLAUDE.md | 87 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 CLAUDE.md diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..ef472f3 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,87 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## What this is + +A migration tool that extracts workloads from other IaC platforms and transforms them into StackGuardian Workflow definitions (`sg-payload.json`), ready for bulk import via [sg-cli](https://github.com/StackGuardian/sg-cli). Currently the only implemented source is **Terraform Cloud / Enterprise (TFC/TFE)**. + +There is no application code to build — the "engine" is Terraform itself. The migrator is a Terraform root module that uses the `tfe` provider to read workspaces and the `local`/`null` providers to write the output payload and pull state files. + +## Architecture + +The migration is a user-driven pipeline, not a single program: + +1. **Extract + transform** — `transformer/terraform-cloud/` is a Terraform root module. `terraform apply` reads TFC/TFE workspaces and emits one `/sg-payload..json` per TFC project, a `migration-summary.md`/`.json` report, and per-workspace `.tfstate` files when `exportStateFiles` is true. +2. **Enrich (variable sets)** — `scripts/enrich_variable_sets.sh` merges TFC Variable Set variables into the payloads via the TFC API (the provider can't enumerate sets). Resolves global/project/workspace scope + TFC precedence per workspace; non-sensitive only. +3. **Tuning** — adjust per-workspace differences (integration IDs, VCS auth, runners, approvers, version) via the `workspaceOverrides` variable and re-apply; or hand-edit the payload files. `example_payload.jsonc` is the annotated field reference. +4. **HCL→JSON conversion** — `scripts/convert_hcl_to_json.sh` rewrites HCL-string variable values in each payload to real JSON. +5. **Validation** — `scripts/validate_payload.sh` checks each payload against `schema/sg-payload.schema.json` (downloads `yajsv`). +6. **Import** — `sg-cli workflow create --bulk`, run once per project file, each into its own workflow group. + +### Orchestration & tooling + +The five phases are wrapped by an orchestrator so users don't run them by hand: + +- `sg-migrate.sh` (host entrypoint, repo root) — runs `scripts/migrate.sh` **inside the Docker image** (`Dockerfile`), bind-mounting the repo at `/app` and forwarding `SG_API_TOKEN`/`SG_ORG`/`SG_BASE_URL`/`TFE_TOKEN`/`TF_TOKEN_*`. For TFC auth it prefers a long-lived `TFE_TOKEN` (env); otherwise it mounts `~/.terraform.d/credentials.tfrc.json` read-only. Runs natively instead when `--native`/`--local` is passed, `SG_NATIVE=1` is set, the command is `clean`, or Docker is absent. +- `scripts/migrate.sh` — the actual orchestrator. Subcommands `init|apply|enrich|convert|validate|import|triggers|all|clean|completion` (default `all`; `enrich` runs in `all` unless `--no-variable-sets`). Resolves each tool via `sg_resolve` (PATH first — the image installs them — else `sg_ensure_*` cache). Runs `convert` and `import` in parallel (`--concurrency`, default 4), raises `terraform apply -parallelism`, and retries `sg-cli` imports with backoff (`sg_retry`, `SG_RETRIES`). Import resolves each project's workflow group, checks existence via the SG API, shows a plan (`exists`/`create`), prompts (skip with `-y`), **creates the missing `tfc-` groups via the API**, then imports. sg-cli exits 0 even when individual workflows fail, so `do_import` parses its output: a workflow rejected as above SG's managed Terraform ceiling (1.5.7, last MPL/FOSS release) is re-imported with `SGDefaultTerraformVersion` (read from `terraform.tfvars`), the payload patched in place, and the case logged to `export/terraform-version-fallbacks.log` plus a printed notice; any other per-workflow failure fails the run. The trigger pass skips workflows that do not exist in SG, and `sg_api_post` returns 22 on 4xx so `sg_retry` (via `SG_NO_RETRY_RC`) does not retry definitive errors. `clean` removes local artifacts (`export/`, TF state, tool cache); `clean --all` also removes config. `completion bash|zsh` prints a completion script (`cmd_completion`; commands/options come from `SG_COMMANDS`/`SG_OPTIONS`, keep them in sync with the parser and the host flags); like `clean` it always runs natively. Output is concise by default — `apply` captures terraform's init/plan output and shows only progress + the final summary (or the full log on failure, `TF_IN_AUTOMATION=1`/`-no-color`); `-v`/`--verbose` (`SG_VERBOSE`) streams everything and un-gates the convert detail lines. Colored logging via `sg_step`/`sg_log`/`sg_success`/`sg_warn`/`sg_err`; paths shown relative via `sg_rel` (all auto-off when not a TTY / `NO_COLOR`). +- `scripts/tools.sh` — sourced by the other scripts (sets `SG_REPO_ROOT` to its parent dir). Provides `sg_resolve` (PATH-or-cache), `sg_ensure_{jq,hcl2json,yajsv,sgcli}` (pinned downloads into `.sg/cached/`; `sg-cli` is the Go binary), `sg_retry`, the color vars + log helpers. The Docker image bundles these tools at build time (`yajsv` built from source so arm64 works), so in-container runs never download. +- **Variable sets** — `scripts/enrich_variable_sets.sh ` (run by `cmd_enrich`, which reads `tfOrg`/`tfHostname` from `terraform.tfvars` via `hcl2json` — variable sets live in the **TFC** org, not `SG_ORG`). Lists all sets + vars via the TFC API (Bearer token from the `terraform login` creds file or `TFE_TOKEN`), computes per-workspace effective vars (scope: global/project/workspace; precedence: priority sets > workspace vars > non-priority sets), and merges them into each workflow (matched by workspace name parsed from `TfStateFilePath`). Sensitive set vars and key conflicts are reported. Skipped by `--no-variable-sets`. +- **Workflow groups** — each TFC project maps to an SG workflow group `tfc-`, created via `POST /api/v1/orgs/{org}/wfgrps/` (auth `Authorization: apikey `) if missing. (Both the API calls and sg-cli honor the undocumented `SG_BASE_URL` for non-prod targets.) Groups are addressed by name, so no generated ID is tracked. `.sg/workflow-groups.json` (gitignored, optional) overrides the target group per segment (`{"": ""}`); override groups are not auto-created. `--no-create-groups` requires all groups to pre-exist. + +### The transformer (`transformer/terraform-cloud/`) + +The whole transformation lives in `locals.tf` — there are no `outputs.tf`/`main.tf` business logic files; `main.tf` only pins provider versions. + +- `data.tf` — four data sources: `tfe_workspace_ids` (selects workspaces by name/tags), `tfe_workspace` (per-workspace details), `tfe_variables` (per-workspace variables), `tfe_projects` (project id→name, used to name per-project payload files). +- `locals.tf` — builds `local.workflowPayload` (workspace name → SG workflow object), groups it into `local.payloadByProject`, and assembles `local.summary`. This is the core mapping from TFC concepts to StackGuardian's payload schema. Key mappings: + - TFC `terraform` (non-sensitive) variables → `VCSConfig.iacInputData.data` (kept as strings here; `try(jsondecode(...), v.value)` only decodes values that are already valid JSON). + - TFC `env` (non-sensitive) variables → `EnvironmentVariables` as `PLAIN_TEXT`. + - `auto_apply` → inverted into `approvalPreApply` / gated `Approvers`. + - `terraform_version` → `TERRAFORM-` only for a pinned semver; otherwise `SGDefaultTerraformVersion`. + - `project_id` → `CLIConfiguration.WorkflowGroup.name` = `tfc-` (matches the per-project filename and the group the importer creates/targets). + - Sensitive variables (terraform + env) are skipped and recorded in the summary. + - Per-workspace `var.workspaceOverrides[]` fields take precedence over the `SGDefault*` values (resolved via `try(var.workspaceOverrides[name]., null) != null ? ... : `). + - `local.resourceNames` sanitizes workspace names to a valid SG `ResourceName` (≤100 chars, `^[-a-zA-Z0-9_]+$`, collision-disambiguated). For normal TFC names this is a no-op; any actual rename is reported in the summary. This is the single place to adjust naming rules. +- `resources.tf` — writes one `sg-payload..json` per project directly via `for_each` (no `mv`), plus `migration-summary.{md,json}`. When `exportStateFiles=true`, `null_resource.exportState` pulls each workspace's state **directly from the TFC/TFE API** (`GET /api/v2/workspaces/{id}/current-state-version` → `hosted-state-download-url`) via a `local-exec` `curl`/`jq` script — no `terraform init` or providers per workspace (avoids the plugin-cache concurrency bug and per-workspace provider downloads). The token is read at runtime from `~/.terraform.d/credentials.tfrc.json` (the `terraform login` file) or `TFE_TOKEN`, so it never enters TF state. Idempotent (keyed by workspace name/id; `forceStateRefresh` re-pulls), with per-workspace failures (no token / no state / download error) recorded in `state-export-failures.log` instead of aborting. `cmd_apply` ensures `jq`/`curl` are on PATH for the apply. +- `summary.tmpl` — renders `local.summary` to `migration-summary.md`. +- `variables.tf` / `terraform.tfvars.example` — inputs. `SGDefault*` variables supply global defaults baked into every workflow (deployment platform, VCS auth, repo prefix, source kind, approvers, Terraform version); `workspaceOverrides` overrides them per workspace; `forceStateRefresh` controls state re-pull. Requires Terraform `>= 1.3` (for `optional()` object attributes). + +### `scripts/convert_hcl_to_json.sh` + +Run from repo root, once per payload file (`for f in export/sg-payload.*.json; do ./scripts/convert_hcl_to_json.sh "$f"; done`). Rewrites the file **in place** (atomically — writes to a temp file, then `mv`s over the original). Downloads pinned `jq` and `hcl2json` binaries to a temp dir at runtime. For each workflow it walks `.VCSConfig.iacInputData.data` and, for any string value that looks like an HCL object/list (`{`/`[`), wraps it as `temp = ` and pipes through `hcl2json` to get real JSON. Plain scalar strings (ids, names) and already-converted objects pass through untouched. + +### `scripts/validate_payload.sh` + `schema/` + +`schema/sg-payload.schema.json` is a self-contained draft-07 schema for the generated payload array, with constraints (ResourceName length, `kind`/`sourceConfigDestKind` enums) derived from the StackGuardian OpenAPI spec (request body `#/components/schemas/Workflow`). The OpenAPI spec is **not committed** — download it from the [API Explorer](https://docs.stackguardian.io/api-reference/) if the schema needs regenerating. The schema is intentionally lenient (unknown fields allowed) — it checks the fields the transformer emits, not every optional API field. `scripts/validate_payload.sh` downloads `yajsv` and validates one or more payload files against it. + +## Common commands + +```shell +# Run the transformer +cd transformer/terraform-cloud +cp terraform.tfvars.example terraform.tfvars # then edit +terraform init +terraform apply -auto-approve -var-file=terraform.tfvars + +# Convert HCL-string vars to JSON, then validate (from repo root, per file) +for f in export/sg-payload.*.json; do ./scripts/convert_hcl_to_json.sh "$f"; done +./scripts/validate_payload.sh export/sg-payload.*.json + +# Bulk import (from the export dir, sg-cli fetched per README) — once per project file +export SG_API_TOKEN= +./sg-cli workflow create --bulk --workflow-group "" --org "" -- sg-payload..json +``` + +`terraform login` must be run first so the `tfe` provider can authenticate to TFC/TFE. + +## Conventions + +- Terraform variable/local naming is **camelCase** (e.g. `workflowNames`, `exportStateFiles`, `SGDefaultVCSAuthIntegrationID`) — not the usual snake_case. Match it. +- Output payload keys are PascalCase to match StackGuardian's API schema (`ResourceName`, `DeploymentPlatformConfig`, `VCSConfig`, ...). Keep `example_payload.jsonc` in sync when you change the generated shape. +- Run `terraform fmt` on `.tf` changes (see commit history). +- `export/`, `*.tfvars`, `*.tfstate`, and `.terraform/` are gitignored — they hold generated output and secrets. + +## Adding a new source platform + +The `transformer/` directory is the extension point: each platform is its own self-contained Terraform root module (mirror `terraform-cloud/`). The contract a transformer must satisfy is producing `sg-payload.*.json` arrays matching `example_payload.jsonc`; everything downstream (`scripts/convert_hcl_to_json.sh`, sg-cli import) is platform-agnostic. From d5e5c4e3a0706fb972eec466d0a67fe8f682d338 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= Date: Mon, 7 Sep 2026 08:18:32 +0200 Subject: [PATCH 09/30] feat: show help menu when no command is given Running sg-migrate.sh without a command used to start the whole pipeline; it now prints the help menu, locally, without touching Docker. Usage and hints show the name the user invoked (./sg-migrate.sh) instead of the in-container script path. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01YKBeQrpJgR9DVFtR83myqX --- CLAUDE.md | 4 ++-- README.md | 2 +- scripts/migrate.sh | 21 +++++++++++++-------- sg-migrate.sh | 15 ++++++++++++--- 4 files changed, 28 insertions(+), 14 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index ef472f3..e05407d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -23,8 +23,8 @@ The migration is a user-driven pipeline, not a single program: The five phases are wrapped by an orchestrator so users don't run them by hand: -- `sg-migrate.sh` (host entrypoint, repo root) — runs `scripts/migrate.sh` **inside the Docker image** (`Dockerfile`), bind-mounting the repo at `/app` and forwarding `SG_API_TOKEN`/`SG_ORG`/`SG_BASE_URL`/`TFE_TOKEN`/`TF_TOKEN_*`. For TFC auth it prefers a long-lived `TFE_TOKEN` (env); otherwise it mounts `~/.terraform.d/credentials.tfrc.json` read-only. Runs natively instead when `--native`/`--local` is passed, `SG_NATIVE=1` is set, the command is `clean`, or Docker is absent. -- `scripts/migrate.sh` — the actual orchestrator. Subcommands `init|apply|enrich|convert|validate|import|triggers|all|clean|completion` (default `all`; `enrich` runs in `all` unless `--no-variable-sets`). Resolves each tool via `sg_resolve` (PATH first — the image installs them — else `sg_ensure_*` cache). Runs `convert` and `import` in parallel (`--concurrency`, default 4), raises `terraform apply -parallelism`, and retries `sg-cli` imports with backoff (`sg_retry`, `SG_RETRIES`). Import resolves each project's workflow group, checks existence via the SG API, shows a plan (`exists`/`create`), prompts (skip with `-y`), **creates the missing `tfc-` groups via the API**, then imports. sg-cli exits 0 even when individual workflows fail, so `do_import` parses its output: a workflow rejected as above SG's managed Terraform ceiling (1.5.7, last MPL/FOSS release) is re-imported with `SGDefaultTerraformVersion` (read from `terraform.tfvars`), the payload patched in place, and the case logged to `export/terraform-version-fallbacks.log` plus a printed notice; any other per-workflow failure fails the run. The trigger pass skips workflows that do not exist in SG, and `sg_api_post` returns 22 on 4xx so `sg_retry` (via `SG_NO_RETRY_RC`) does not retry definitive errors. `clean` removes local artifacts (`export/`, TF state, tool cache); `clean --all` also removes config. `completion bash|zsh` prints a completion script (`cmd_completion`; commands/options come from `SG_COMMANDS`/`SG_OPTIONS`, keep them in sync with the parser and the host flags); like `clean` it always runs natively. Output is concise by default — `apply` captures terraform's init/plan output and shows only progress + the final summary (or the full log on failure, `TF_IN_AUTOMATION=1`/`-no-color`); `-v`/`--verbose` (`SG_VERBOSE`) streams everything and un-gates the convert detail lines. Colored logging via `sg_step`/`sg_log`/`sg_success`/`sg_warn`/`sg_err`; paths shown relative via `sg_rel` (all auto-off when not a TTY / `NO_COLOR`). +- `sg-migrate.sh` (host entrypoint, repo root) — runs `scripts/migrate.sh` **inside the Docker image** (`Dockerfile`), bind-mounting the repo at `/app` and forwarding `SG_API_TOKEN`/`SG_ORG`/`SG_BASE_URL`/`TFE_TOKEN`/`TF_TOKEN_*`. For TFC auth it prefers a long-lived `TFE_TOKEN` (env); otherwise it mounts `~/.terraform.d/credentials.tfrc.json` read-only. Runs natively instead when `--native`/`--local` is passed, `SG_NATIVE=1` is set, the command is `clean`/`completion`/help (or no command is given), or Docker is absent. Exports `SG_PROG` so `migrate.sh` shows `./sg-migrate.sh` in its usage/hints. +- `scripts/migrate.sh` — the actual orchestrator. Subcommands `init|apply|enrich|convert|validate|import|triggers|all|clean|completion` (no command prints the help menu; `enrich` runs in `all` unless `--no-variable-sets`). Resolves each tool via `sg_resolve` (PATH first — the image installs them — else `sg_ensure_*` cache). Runs `convert` and `import` in parallel (`--concurrency`, default 4), raises `terraform apply -parallelism`, and retries `sg-cli` imports with backoff (`sg_retry`, `SG_RETRIES`). Import resolves each project's workflow group, checks existence via the SG API, shows a plan (`exists`/`create`), prompts (skip with `-y`), **creates the missing `tfc-` groups via the API**, then imports. sg-cli exits 0 even when individual workflows fail, so `do_import` parses its output: a workflow rejected as above SG's managed Terraform ceiling (1.5.7, last MPL/FOSS release) is re-imported with `SGDefaultTerraformVersion` (read from `terraform.tfvars`), the payload patched in place, and the case logged to `export/terraform-version-fallbacks.log` plus a printed notice; any other per-workflow failure fails the run. The trigger pass skips workflows that do not exist in SG, and `sg_api_post` returns 22 on 4xx so `sg_retry` (via `SG_NO_RETRY_RC`) does not retry definitive errors. `clean` removes local artifacts (`export/`, TF state, tool cache); `clean --all` also removes config. `completion bash|zsh` prints a completion script (`cmd_completion`; commands/options come from `SG_COMMANDS`/`SG_OPTIONS`, keep them in sync with the parser and the host flags); like `clean` it always runs natively. Output is concise by default — `apply` captures terraform's init/plan output and shows only progress + the final summary (or the full log on failure, `TF_IN_AUTOMATION=1`/`-no-color`); `-v`/`--verbose` (`SG_VERBOSE`) streams everything and un-gates the convert detail lines. Colored logging via `sg_step`/`sg_log`/`sg_success`/`sg_warn`/`sg_err`; paths shown relative via `sg_rel` (all auto-off when not a TTY / `NO_COLOR`). - `scripts/tools.sh` — sourced by the other scripts (sets `SG_REPO_ROOT` to its parent dir). Provides `sg_resolve` (PATH-or-cache), `sg_ensure_{jq,hcl2json,yajsv,sgcli}` (pinned downloads into `.sg/cached/`; `sg-cli` is the Go binary), `sg_retry`, the color vars + log helpers. The Docker image bundles these tools at build time (`yajsv` built from source so arm64 works), so in-container runs never download. - **Variable sets** — `scripts/enrich_variable_sets.sh ` (run by `cmd_enrich`, which reads `tfOrg`/`tfHostname` from `terraform.tfvars` via `hcl2json` — variable sets live in the **TFC** org, not `SG_ORG`). Lists all sets + vars via the TFC API (Bearer token from the `terraform login` creds file or `TFE_TOKEN`), computes per-workspace effective vars (scope: global/project/workspace; precedence: priority sets > workspace vars > non-priority sets), and merges them into each workflow (matched by workspace name parsed from `TfStateFilePath`). Sensitive set vars and key conflicts are reported. Skipped by `--no-variable-sets`. - **Workflow groups** — each TFC project maps to an SG workflow group `tfc-`, created via `POST /api/v1/orgs/{org}/wfgrps/` (auth `Authorization: apikey `) if missing. (Both the API calls and sg-cli honor the undocumented `SG_BASE_URL` for non-prod targets.) Groups are addressed by name, so no generated ID is tracked. `.sg/workflow-groups.json` (gitignored, optional) overrides the target group per segment (`{"": ""}`); override groups are not auto-created. `--no-create-groups` requires all groups to pre-exist. diff --git a/README.md b/README.md index 4c0d291..d965523 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ export SG_ORG= That's it — no workflow-group mapping to fill in. Each TFC project is imported into an SG workflow group named `tfc-`, **created automatically via the API** if it doesn't exist. The import prompt shows each group as `exists` or `create` before anything is written. -- Single phase: `./sg-migrate.sh apply|enrich|convert|validate|import`. +- Single phase: `./sg-migrate.sh apply|enrich|convert|validate|import|triggers`. Running `./sg-migrate.sh` with no command prints the help menu. - TFC **Variable Set** variables are merged into the payloads automatically (the `enrich` phase, via the TFC API); skip it with `--no-variable-sets`. - `./sg-migrate.sh clean` removes local working artifacts (`export/`, Terraform state, tool cache) for a fresh start; add `--all` to also remove config. `clean` always runs locally. - **Override** a project's target group (to reuse an existing group) in `.sg/workflow-groups.json`: `{"": ""}`. Override groups must already exist (they're not auto-created). diff --git a/scripts/migrate.sh b/scripts/migrate.sh index c765e63..7d4e063 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -14,6 +14,7 @@ source "$SCRIPT_DIR/tools.sh" # root used for all repo-relative paths. TRANSFORMER_DIR="$SG_REPO_ROOT/transformer/terraform-cloud" TFVARS="$TRANSFORMER_DIR/terraform.tfvars" +PROG="${SG_PROG:-$0}" EXPORT_DIR="${SG_EXPORT_DIR:-$SG_REPO_ROOT/export}" MAPPING="${SG_WFGROUP_MAP:-$SG_REPO_ROOT/.sg/workflow-groups.json}" ORG="${SG_ORG:-}" @@ -32,7 +33,7 @@ PF=() usage() { cat >&2 < Commands: init Create terraform.tfvars from the template @@ -43,7 +44,7 @@ Commands: import Import each payload to StackGuardian (parallel, with confirmation), then register VCS triggers (unless --no-vcs-triggers) triggers Register VCS triggers for already-imported workflows (second pass) - all apply -> enrich -> convert -> validate -> import (default) + all apply -> enrich -> convert -> validate -> import clean Remove local working artifacts for a fresh start (export/, TF state, tool cache). Add --all to also remove config (terraform.tfvars, mapping). completion Print a shell completion script for the current session: @@ -261,7 +262,7 @@ set_triggers_pass() { if run_parallel do_set_triggers "$CONC" "${PF[@]}"; then sg_success "vcs triggers registered" else - sg_err "one or more VCS trigger registrations failed (re-run: $0 triggers)" + sg_err "one or more VCS trigger registrations failed (re-run: $PROG triggers)" return 1 fi } @@ -333,7 +334,7 @@ cmd_apply() { cmd_enrich() { sg_step "Phase: variable sets" - [ -f "$TFVARS" ] || die "Missing $(sg_rel "$TFVARS") (run: $0 init)." + [ -f "$TFVARS" ] || die "Missing $(sg_rel "$TFVARS") (run: $PROG init)." payload_files [ "${#PF[@]}" -gt 0 ] || die "No payload files in $(sg_rel "$EXPORT_DIR") (run 'apply' first)." # Variable sets belong to the TFC org (tfOrg in terraform.tfvars), not SG_ORG. @@ -613,7 +614,7 @@ _sg_migrate() { 'validate:Validate payloads against the SG schema' 'import:Import payloads to StackGuardian, then register VCS triggers' 'triggers:Register VCS triggers for already-imported workflows' - 'all:apply -> enrich -> convert -> validate -> import (default)' + 'all:apply -> enrich -> convert -> validate -> import' 'clean:Remove local working artifacts' 'completion:Print a shell completion script' ) @@ -641,7 +642,7 @@ _sg_migrate() { compdef _sg_migrate sg-migrate.sh migrate.sh ZSH ;; - *) die "usage: $0 completion " ;; + *) die "usage: $PROG completion " ;; esac } @@ -691,7 +692,11 @@ while [ $# -gt 0 ]; do esac shift done -CMD="${CMD:-all}" +# No command: show the help menu instead of running the whole pipeline. +if [ -z "$CMD" ]; then + usage + exit 0 +fi export SG_VERBOSE="$VERBOSE" case "$CMD" in @@ -706,7 +711,7 @@ triggers) cmd_triggers ;; all) if [ ! -f "$TFVARS" ]; then cmd_init - die "Edit $(sg_rel "$TFVARS"), then re-run '$0 all'." + die "Edit $(sg_rel "$TFVARS"), then re-run '$PROG all'." fi # Fail fast on import prerequisites before the (long) apply. [ -n "${SG_API_TOKEN:-}" ] || die "SG_API_TOKEN is not set (needed for import)." diff --git a/sg-migrate.sh b/sg-migrate.sh index e1138d6..14cd745 100755 --- a/sg-migrate.sh +++ b/sg-migrate.sh @@ -27,10 +27,19 @@ for a in "$@"; do esac done -# 'clean' and 'completion' only touch the local shell/filesystem — no container. +# Help, 'clean' and 'completion' only touch the local shell/filesystem — no +# container. With no command at all, migrate.sh prints the help menu. +HAS_CMD=0 for a in ${ARGS[@]+"${ARGS[@]}"}; do - case "$a" in clean | completion) NATIVE=1 ;; esac + case "$a" in + clean | completion | -h | --help) NATIVE=1; HAS_CMD=1 ;; + init | apply | enrich | convert | validate | import | triggers | all) HAS_CMD=1 ;; + esac done +[ "$HAS_CMD" -eq 1 ] || NATIVE=1 +# Let migrate.sh print the name the user actually invoked in its help/hints. +export SG_PROG="${0##*/}" +case "$0" in */*) SG_PROG="./${0##*/}" ;; esac if [ "$NATIVE" = "1" ] || ! command -v docker >/dev/null 2>&1; then [ "$NATIVE" = "1" ] || sg_warn "docker not found; running natively" @@ -45,7 +54,7 @@ fi DOCKER_ARGS=(--rm -i -v "$SCRIPT_DIR:/app" -w /app -e SG_API_TOKEN -e SG_ORG -e SG_BASE_URL -e SG_CONCURRENCY -e SG_RETRIES -e SG_TF_PARALLELISM - -e TFE_TOKEN) + -e TFE_TOKEN -e SG_PROG) # Interactive TTY only when attached to one (so the confirmation prompt works, # but CI/non-tty invocations still run — use -y there). From 784a66cf34bd5b8e6a756164f4425d8a03bb104c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= Date: Mon, 7 Sep 2026 08:23:42 +0200 Subject: [PATCH 10/30] fix: fail fast without TFC credentials and configure tfe provider host Without TFE_TOKEN or a terraform login credentials file, apply used to warn and then fail deep inside terraform with "Invalid provider configuration". Both the host entrypoint and migrate.sh now stop with a clear message before anything runs. Add an explicit provider "tfe" block so tfHostname also applies to the provider, not just the state export. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01YKBeQrpJgR9DVFtR83myqX --- scripts/migrate.sh | 18 ++++++++++++++++-- sg-migrate.sh | 6 +++++- transformer/terraform-cloud/main.tf | 7 +++++++ 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/scripts/migrate.sh b/scripts/migrate.sh index 7d4e063..b186a61 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -294,10 +294,23 @@ cmd_clean() { sg_success "clean complete" } +# require_tfc_auth — fail fast with a clear message when no TFC/TFE credential +# is available for the tfe provider. Accepted: TFE_TOKEN, any TF_TOKEN_* var +# (terraform's per-host token env), or the `terraform login` credentials file. +# Without this, terraform fails later with a confusing "Invalid provider +# configuration" error. +require_tfc_auth() { + [ -n "${TFE_TOKEN:-}" ] && return 0 + if env | grep -q '^TF_TOKEN_'; then return 0; fi + [ -f "$HOME/.terraform.d/credentials.tfrc.json" ] && return 0 + die "no Terraform Cloud/Enterprise credentials found. Set TFE_TOKEN= (recommended) or run 'terraform login' before '$PROG apply'." +} + cmd_apply() { sg_step "Phase: apply (terraform)" command -v terraform >/dev/null 2>&1 || die "terraform not found on PATH" - [ -f "$TFVARS" ] || die "Missing $(sg_rel "$TFVARS"). Run: $0 init (then edit it)." + [ -f "$TFVARS" ] || die "Missing $(sg_rel "$TFVARS"). Run: $PROG init (then edit it)." + require_tfc_auth # State export (TFC API) calls curl + jq from terraform's local-exec; make sure # both are on PATH for the apply (jq from cache if not already installed). command -v curl >/dev/null 2>&1 || die "curl is required for state export" @@ -713,7 +726,8 @@ all) cmd_init die "Edit $(sg_rel "$TFVARS"), then re-run '$PROG all'." fi - # Fail fast on import prerequisites before the (long) apply. + # Fail fast on prerequisites before the (long) apply. + require_tfc_auth [ -n "${SG_API_TOKEN:-}" ] || die "SG_API_TOKEN is not set (needed for import)." [ -n "$ORG" ] || die "StackGuardian org not set (use --org or SG_ORG)." cmd_apply diff --git a/sg-migrate.sh b/sg-migrate.sh index 14cd745..b43c84e 100755 --- a/sg-migrate.sh +++ b/sg-migrate.sh @@ -67,7 +67,11 @@ if [ -n "${TFE_TOKEN:-}" ]; then elif [ -f "$CREDS" ]; then DOCKER_ARGS+=(-v "$CREDS:/root/.terraform.d/credentials.tfrc.json:ro") else - sg_warn "no TFC auth found — set TFE_TOKEN (long-lived API token) or run 'terraform login'" + # Only apply/all need TFC auth; migrate.sh fails fast there with a clear message. + case " ${ARGS[*]-} " in *" apply "* | *" all "*) + sg_err "no Terraform Cloud/Enterprise credentials found. Set TFE_TOKEN= (recommended) or run 'terraform login' first." + exit 1 ;; + esac fi # Forward any TF_TOKEN_* env vars (alternative TFC/TFE auth) if present. diff --git a/transformer/terraform-cloud/main.tf b/transformer/terraform-cloud/main.tf index 9a19179..7e8dd58 100644 --- a/transformer/terraform-cloud/main.tf +++ b/transformer/terraform-cloud/main.tf @@ -16,3 +16,10 @@ terraform { } } } + +# Token comes from TFE_TOKEN / TF_TOKEN_ or the `terraform login` +# credentials file; only the hostname is configured here so TFE (self-hosted) +# installs work by setting tfHostname. +provider "tfe" { + hostname = var.tfHostname +} From 6a43e8a9975b0ccf562fb88ad00dc16816d917bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= Date: Mon, 7 Sep 2026 08:43:51 +0200 Subject: [PATCH 11/30] fix: verify TFC credentials before apply An expired 'terraform login' session (or a bad TFE_TOKEN) used to fail deep inside terraform. Resolve the token the tfe provider will use, in its own precedence, and check it against /api/v2/account/details first; stop with a clear message on 401/403 or when the host is unreachable. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01YKBeQrpJgR9DVFtR83myqX --- CLAUDE.md | 2 +- scripts/migrate.sh | 48 +++++++++++++++++++++++++++++++++++++--------- 2 files changed, 40 insertions(+), 10 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index e05407d..d972fd8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -24,7 +24,7 @@ The migration is a user-driven pipeline, not a single program: The five phases are wrapped by an orchestrator so users don't run them by hand: - `sg-migrate.sh` (host entrypoint, repo root) — runs `scripts/migrate.sh` **inside the Docker image** (`Dockerfile`), bind-mounting the repo at `/app` and forwarding `SG_API_TOKEN`/`SG_ORG`/`SG_BASE_URL`/`TFE_TOKEN`/`TF_TOKEN_*`. For TFC auth it prefers a long-lived `TFE_TOKEN` (env); otherwise it mounts `~/.terraform.d/credentials.tfrc.json` read-only. Runs natively instead when `--native`/`--local` is passed, `SG_NATIVE=1` is set, the command is `clean`/`completion`/help (or no command is given), or Docker is absent. Exports `SG_PROG` so `migrate.sh` shows `./sg-migrate.sh` in its usage/hints. -- `scripts/migrate.sh` — the actual orchestrator. Subcommands `init|apply|enrich|convert|validate|import|triggers|all|clean|completion` (no command prints the help menu; `enrich` runs in `all` unless `--no-variable-sets`). Resolves each tool via `sg_resolve` (PATH first — the image installs them — else `sg_ensure_*` cache). Runs `convert` and `import` in parallel (`--concurrency`, default 4), raises `terraform apply -parallelism`, and retries `sg-cli` imports with backoff (`sg_retry`, `SG_RETRIES`). Import resolves each project's workflow group, checks existence via the SG API, shows a plan (`exists`/`create`), prompts (skip with `-y`), **creates the missing `tfc-` groups via the API**, then imports. sg-cli exits 0 even when individual workflows fail, so `do_import` parses its output: a workflow rejected as above SG's managed Terraform ceiling (1.5.7, last MPL/FOSS release) is re-imported with `SGDefaultTerraformVersion` (read from `terraform.tfvars`), the payload patched in place, and the case logged to `export/terraform-version-fallbacks.log` plus a printed notice; any other per-workflow failure fails the run. The trigger pass skips workflows that do not exist in SG, and `sg_api_post` returns 22 on 4xx so `sg_retry` (via `SG_NO_RETRY_RC`) does not retry definitive errors. `clean` removes local artifacts (`export/`, TF state, tool cache); `clean --all` also removes config. `completion bash|zsh` prints a completion script (`cmd_completion`; commands/options come from `SG_COMMANDS`/`SG_OPTIONS`, keep them in sync with the parser and the host flags); like `clean` it always runs natively. Output is concise by default — `apply` captures terraform's init/plan output and shows only progress + the final summary (or the full log on failure, `TF_IN_AUTOMATION=1`/`-no-color`); `-v`/`--verbose` (`SG_VERBOSE`) streams everything and un-gates the convert detail lines. Colored logging via `sg_step`/`sg_log`/`sg_success`/`sg_warn`/`sg_err`; paths shown relative via `sg_rel` (all auto-off when not a TTY / `NO_COLOR`). +- `scripts/migrate.sh` — the actual orchestrator. Subcommands `init|apply|enrich|convert|validate|import|triggers|all|clean|completion` (no command prints the help menu; `enrich` runs in `all` unless `--no-variable-sets`). Resolves each tool via `sg_resolve` (PATH first — the image installs them — else `sg_ensure_*` cache). `apply`/`all` first run `require_tfc_auth`, which resolves the token the tfe provider will use (`TFE_TOKEN`, `TF_TOKEN_`, or the `terraform login` file for `tfHostname`) and verifies it with `GET /api/v2/account/details`, failing fast on a missing or rejected (expired) credential. Runs `convert` and `import` in parallel (`--concurrency`, default 4), raises `terraform apply -parallelism`, and retries `sg-cli` imports with backoff (`sg_retry`, `SG_RETRIES`). Import resolves each project's workflow group, checks existence via the SG API, shows a plan (`exists`/`create`), prompts (skip with `-y`), **creates the missing `tfc-` groups via the API**, then imports. sg-cli exits 0 even when individual workflows fail, so `do_import` parses its output: a workflow rejected as above SG's managed Terraform ceiling (1.5.7, last MPL/FOSS release) is re-imported with `SGDefaultTerraformVersion` (read from `terraform.tfvars`), the payload patched in place, and the case logged to `export/terraform-version-fallbacks.log` plus a printed notice; any other per-workflow failure fails the run. The trigger pass skips workflows that do not exist in SG, and `sg_api_post` returns 22 on 4xx so `sg_retry` (via `SG_NO_RETRY_RC`) does not retry definitive errors. `clean` removes local artifacts (`export/`, TF state, tool cache); `clean --all` also removes config. `completion bash|zsh` prints a completion script (`cmd_completion`; commands/options come from `SG_COMMANDS`/`SG_OPTIONS`, keep them in sync with the parser and the host flags); like `clean` it always runs natively. Output is concise by default — `apply` captures terraform's init/plan output and shows only progress + the final summary (or the full log on failure, `TF_IN_AUTOMATION=1`/`-no-color`); `-v`/`--verbose` (`SG_VERBOSE`) streams everything and un-gates the convert detail lines. Colored logging via `sg_step`/`sg_log`/`sg_success`/`sg_warn`/`sg_err`; paths shown relative via `sg_rel` (all auto-off when not a TTY / `NO_COLOR`). - `scripts/tools.sh` — sourced by the other scripts (sets `SG_REPO_ROOT` to its parent dir). Provides `sg_resolve` (PATH-or-cache), `sg_ensure_{jq,hcl2json,yajsv,sgcli}` (pinned downloads into `.sg/cached/`; `sg-cli` is the Go binary), `sg_retry`, the color vars + log helpers. The Docker image bundles these tools at build time (`yajsv` built from source so arm64 works), so in-container runs never download. - **Variable sets** — `scripts/enrich_variable_sets.sh ` (run by `cmd_enrich`, which reads `tfOrg`/`tfHostname` from `terraform.tfvars` via `hcl2json` — variable sets live in the **TFC** org, not `SG_ORG`). Lists all sets + vars via the TFC API (Bearer token from the `terraform login` creds file or `TFE_TOKEN`), computes per-workspace effective vars (scope: global/project/workspace; precedence: priority sets > workspace vars > non-priority sets), and merges them into each workflow (matched by workspace name parsed from `TfStateFilePath`). Sensitive set vars and key conflicts are reported. Skipped by `--no-variable-sets`. - **Workflow groups** — each TFC project maps to an SG workflow group `tfc-`, created via `POST /api/v1/orgs/{org}/wfgrps/` (auth `Authorization: apikey `) if missing. (Both the API calls and sg-cli honor the undocumented `SG_BASE_URL` for non-prod targets.) Groups are addressed by name, so no generated ID is tracked. `.sg/workflow-groups.json` (gitignored, optional) overrides the target group per segment (`{"": ""}`); override groups are not auto-created. `--no-create-groups` requires all groups to pre-exist. diff --git a/scripts/migrate.sh b/scripts/migrate.sh index b186a61..4d1f3a3 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -294,16 +294,46 @@ cmd_clean() { sg_success "clean complete" } -# require_tfc_auth — fail fast with a clear message when no TFC/TFE credential -# is available for the tfe provider. Accepted: TFE_TOKEN, any TF_TOKEN_* var -# (terraform's per-host token env), or the `terraform login` credentials file. -# Without this, terraform fails later with a confusing "Invalid provider -# configuration" error. +# tfc_hostname — TFC/TFE host from terraform.tfvars (tfHostname), default app.terraform.io. +tfc_hostname() { + local h="" + if [ -f "$TFVARS" ]; then + h="$("$(sg_resolve hcl2json sg_ensure_hcl2json)" "$TFVARS" 2>/dev/null | "$(sg_resolve jq sg_ensure_jq)" -r '.tfHostname // empty' 2>/dev/null || true)" + fi + echo "${h:-app.terraform.io}" +} + +# tfc_token — resolve the TFC/TFE token the tfe provider will use, in +# terraform's own precedence: TFE_TOKEN, TF_TOKEN_ ('.'->'_', '-'->'__'), +# then the `terraform login` credentials file. Prints nothing if none is set. +tfc_token() { + local host="$1" var creds + if [ -n "${TFE_TOKEN:-}" ]; then echo "$TFE_TOKEN"; return; fi + var="TF_TOKEN_$(echo "$host" | sed 's/-/__/g; s/\./_/g')" + if [ -n "${!var:-}" ]; then echo "${!var}"; return; fi + creds="$HOME/.terraform.d/credentials.tfrc.json" + [ -f "$creds" ] && "$(sg_resolve jq sg_ensure_jq)" -r --arg h "$host" '.credentials[$h].token // empty' "$creds" 2>/dev/null || true +} + +# require_tfc_auth — fail fast, before the (long) apply, when the tfe provider +# would not be able to authenticate: no credential at all, or a credential +# that TFC/TFE rejects (e.g. an expired `terraform login` session). Verified +# with GET /api/v2/account/details. Without this, terraform fails later with a +# confusing "Invalid provider configuration" error. require_tfc_auth() { - [ -n "${TFE_TOKEN:-}" ] && return 0 - if env | grep -q '^TF_TOKEN_'; then return 0; fi - [ -f "$HOME/.terraform.d/credentials.tfrc.json" ] && return 0 - die "no Terraform Cloud/Enterprise credentials found. Set TFE_TOKEN= (recommended) or run 'terraform login' before '$PROG apply'." + local host token code src + host="$(tfc_hostname)" + token="$(tfc_token "$host")" + if [ -n "${TFE_TOKEN:-}" ]; then src="TFE_TOKEN"; elif env | grep -q '^TF_TOKEN_'; then src="the TF_TOKEN_* variable"; else src="the 'terraform login' session (likely expired)"; fi + [ -n "$token" ] || die "no Terraform Cloud/Enterprise credentials found for $host. Set TFE_TOKEN= (recommended) or run 'terraform login' before '$PROG apply'." + command -v curl >/dev/null 2>&1 || return 0 + code="$(curl -sS -o /dev/null -w '%{http_code}' -H "Authorization: Bearer $token" "https://$host/api/v2/account/details" 2>/dev/null || echo 000)" + case "$code" in + 200) sg_log "TFC/TFE credentials verified ($host)" ;; + 401 | 403) die "Terraform Cloud/Enterprise rejected $src for $host (HTTP $code). Set TFE_TOKEN= or re-run 'terraform login'." ;; + 000) die "could not reach https://$host to verify TFC/TFE credentials (network/proxy?)" ;; + *) sg_warn "unexpected HTTP $code verifying TFC/TFE credentials at $host; continuing" ;; + esac } cmd_apply() { From 53a48e8fe5fadb899fb161c3529d9a212eead2b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= Date: Mon, 7 Sep 2026 08:50:49 +0200 Subject: [PATCH 12/30] feat: global runner constraints via SGDefaultRunnerConstraints Every workflow was hardcoded to shared runners unless overridden per workspace. Add SGDefaultRunnerConstraints (default shared) so all workflows can be put behind a private runner group in one place; workspaceOverrides[name].RunnerConstraints still wins per workspace. Validated: type is shared|private, and private requires names. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01YKBeQrpJgR9DVFtR83myqX --- CLAUDE.md | 2 +- README.md | 2 +- transformer/terraform-cloud/locals.tf | 2 +- .../terraform-cloud/terraform.tfvars.example | 5 +++++ transformer/terraform-cloud/variables.tf | 17 +++++++++++++++++ 5 files changed, 25 insertions(+), 3 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index d972fd8..ad9f762 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -45,7 +45,7 @@ The whole transformation lives in `locals.tf` — there are no `outputs.tf`/`mai - `local.resourceNames` sanitizes workspace names to a valid SG `ResourceName` (≤100 chars, `^[-a-zA-Z0-9_]+$`, collision-disambiguated). For normal TFC names this is a no-op; any actual rename is reported in the summary. This is the single place to adjust naming rules. - `resources.tf` — writes one `sg-payload..json` per project directly via `for_each` (no `mv`), plus `migration-summary.{md,json}`. When `exportStateFiles=true`, `null_resource.exportState` pulls each workspace's state **directly from the TFC/TFE API** (`GET /api/v2/workspaces/{id}/current-state-version` → `hosted-state-download-url`) via a `local-exec` `curl`/`jq` script — no `terraform init` or providers per workspace (avoids the plugin-cache concurrency bug and per-workspace provider downloads). The token is read at runtime from `~/.terraform.d/credentials.tfrc.json` (the `terraform login` file) or `TFE_TOKEN`, so it never enters TF state. Idempotent (keyed by workspace name/id; `forceStateRefresh` re-pulls), with per-workspace failures (no token / no state / download error) recorded in `state-export-failures.log` instead of aborting. `cmd_apply` ensures `jq`/`curl` are on PATH for the apply. - `summary.tmpl` — renders `local.summary` to `migration-summary.md`. -- `variables.tf` / `terraform.tfvars.example` — inputs. `SGDefault*` variables supply global defaults baked into every workflow (deployment platform, VCS auth, repo prefix, source kind, approvers, Terraform version); `workspaceOverrides` overrides them per workspace; `forceStateRefresh` controls state re-pull. Requires Terraform `>= 1.3` (for `optional()` object attributes). +- `variables.tf` / `terraform.tfvars.example` — inputs. `SGDefault*` variables supply global defaults baked into every workflow (deployment platform, VCS auth, repo prefix, source kind, approvers, Terraform version, runner constraints — `SGDefaultRunnerConstraints`, validated to `shared` or `private`+`names`); `workspaceOverrides` overrides them per workspace; `forceStateRefresh` controls state re-pull. Requires Terraform `>= 1.3` (for `optional()` object attributes). ### `scripts/convert_hcl_to_json.sh` diff --git a/README.md b/README.md index d965523..b47ab17 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ The manual, step-by-step flow below remains supported for fine-grained control a ## Prerequisites - An organization on [StackGuardian Platform](https://app.stackguardian.io) -- Optionally, pre-configure VCS, cloud integrations or private runners to use when importing into StackGuardian Platform. +- Optionally, pre-configure VCS, cloud integrations or private runners to use when importing into StackGuardian Platform. To run every workflow on a private runner group, set `SGDefaultRunnerConstraints = { type = "private", names = [""] }` in `terraform.tfvars` (per-workspace exceptions via `workspaceOverrides[].RunnerConstraints`). - Terraform - [sg-cli](https://github.com/StackGuardian/sg-cli) diff --git a/transformer/terraform-cloud/locals.tf b/transformer/terraform-cloud/locals.tf index 48a6ebc..d2ba8f0 100644 --- a/transformer/terraform-cloud/locals.tf +++ b/transformer/terraform-cloud/locals.tf @@ -77,7 +77,7 @@ locals { ) DeploymentPlatformConfig = try(var.workspaceOverrides[wsName].DeploymentPlatformConfig, null) != null ? var.workspaceOverrides[wsName].DeploymentPlatformConfig : var.SGDefaultDeploymentPlatformConfig - RunnerConstraints = try(var.workspaceOverrides[wsName].RunnerConstraints, null) != null ? var.workspaceOverrides[wsName].RunnerConstraints : { "type" : "shared" } + RunnerConstraints = try(var.workspaceOverrides[wsName].RunnerConstraints, null) != null ? var.workspaceOverrides[wsName].RunnerConstraints : { for k, v in var.SGDefaultRunnerConstraints : k => v if v != null } VCSConfig = { "iacVCSConfig" : { diff --git a/transformer/terraform-cloud/terraform.tfvars.example b/transformer/terraform-cloud/terraform.tfvars.example index 6b0024a..5b7143f 100644 --- a/transformer/terraform-cloud/terraform.tfvars.example +++ b/transformer/terraform-cloud/terraform.tfvars.example @@ -36,6 +36,11 @@ SGDefaultDeploymentPlatformConfig = [ } ] +# Runners for every workflow: SG-hosted shared runners (default), or put all +# workflows behind a private runner group: +# SGDefaultRunnerConstraints = { type = "private", names = ["sg-runner"] } +SGDefaultRunnerConstraints = { type = "shared" } + # Choose from: GITHUB_COM, BITBUCKET_ORG, GITLAB_COM, AZURE_DEVOPS, GIT_OTHER SGDefaultSourceConfigDestKind = "GITHUB_COM" diff --git a/transformer/terraform-cloud/variables.tf b/transformer/terraform-cloud/variables.tf index 3c3b960..26d8299 100644 --- a/transformer/terraform-cloud/variables.tf +++ b/transformer/terraform-cloud/variables.tf @@ -71,6 +71,23 @@ variable "SGDefaultDeploymentPlatformConfig" { type = list(any) } +variable "SGDefaultRunnerConstraints" { + default = { type = "shared" } + description = "Runner constraints applied to every workflow. Use { type = \"shared\" } for SG-hosted runners, or { type = \"private\", names = [\"\"] } to put every workflow behind a private runner group. Override per workspace via workspaceOverrides[name].RunnerConstraints." + type = object({ + type = string + names = optional(list(string)) + }) + validation { + condition = contains(["shared", "private"], var.SGDefaultRunnerConstraints.type) + error_message = "SGDefaultRunnerConstraints.type must be \"shared\" or \"private\"." + } + validation { + condition = var.SGDefaultRunnerConstraints.type != "private" || length(coalesce(var.SGDefaultRunnerConstraints.names, [])) > 0 + error_message = "SGDefaultRunnerConstraints.names must list at least one runner group when type is \"private\"." + } +} + variable "SGDefaultSourceConfigDestKind" { default = "GIT_OTHER" description = "Choose from: GITHUB_COM, BITBUCKET_ORG, GITLAB_COM, AZURE_DEVOPS, GIT_OTHER" From ed666e7dd29bfa55f2011d76b10af6e9ad4a5bb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= Date: Mon, 7 Sep 2026 09:14:40 +0200 Subject: [PATCH 13/30] feat: shared libs for prompts, tfvars, TFC and SG API access Move the TFC/SG API helpers out of migrate.sh into scripts/lib/ and add the building blocks the upcoming init wizard, preflight and checklist need: interactive prompt helpers (tty or scripted answers), cached tfvars reads, paginated TFC listing (orgs/projects/workspaces) and SG lookups for integrations, runner groups, workflows and secrets. The enrich script now shares the same token resolution as the provider. Also stop turning curl connection failures into "000000" status codes. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01YKBeQrpJgR9DVFtR83myqX --- .gitignore | 2 + scripts/enrich_variable_sets.sh | 38 +++------ scripts/lib/prompt.sh | 121 ++++++++++++++++++++++++++++ scripts/lib/sg_api.sh | 136 ++++++++++++++++++++++++++++++++ scripts/lib/tfc_api.sh | 105 ++++++++++++++++++++++++ scripts/lib/tfvars.sh | 36 +++++++++ scripts/migrate.sh | 122 ++++------------------------ 7 files changed, 426 insertions(+), 134 deletions(-) create mode 100644 scripts/lib/prompt.sh create mode 100644 scripts/lib/sg_api.sh create mode 100644 scripts/lib/tfc_api.sh create mode 100644 scripts/lib/tfvars.sh diff --git a/.gitignore b/.gitignore index 953654c..4e5a8c2 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,8 @@ eggs/ .eggs/ lib/ lib64/ +# The migrator's sourced shell libraries are not Python build output. +!scripts/lib/ parts/ sdist/ var/ diff --git a/scripts/enrich_variable_sets.sh b/scripts/enrich_variable_sets.sh index d7bbc6b..167b04a 100755 --- a/scripts/enrich_variable_sets.sh +++ b/scripts/enrich_variable_sets.sh @@ -15,6 +15,11 @@ set -uo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck source=tools.sh source "$SCRIPT_DIR/tools.sh" +# shellcheck source=lib/tfvars.sh +source "$SCRIPT_DIR/lib/tfvars.sh" +# shellcheck source=lib/tfc_api.sh +source "$SCRIPT_DIR/lib/tfc_api.sh" +TFVARS="${TFVARS:-$SG_REPO_ROOT/transformer/terraform-cloud/terraform.tfvars}" ORG="${1:-}" shift || true @@ -23,44 +28,25 @@ if [ -z "$ORG" ] || [ "$#" -eq 0 ]; then exit 1 fi -HOST="${SG_TFC_HOSTNAME:-app.terraform.io}" -API="https://$HOST/api/v2" +TFC_HOST="${SG_TFC_HOSTNAME:-$(tfc_hostname)}" +HOST="$TFC_HOST" command -v curl >/dev/null 2>&1 || { sg_err "curl is required for variable-set enrichment" exit 1 } JQ_BIN="$(sg_resolve jq sg_ensure_jq)" -# TFC token (same sources as state export): credentials file or TFE_TOKEN. -creds="$HOME/.terraform.d/credentials.tfrc.json" -token="" -[ -f "$creds" ] && token="$("$JQ_BIN" -r --arg h "$HOST" '.credentials[$h].token // empty' "$creds" 2>/dev/null || true)" -[ -z "$token" ] && token="${TFE_TOKEN:-}" -if [ -z "$token" ]; then +# TFC token: same resolution as the tfe provider / state export (lib/tfc_api.sh). +if [ -z "$(tfc_token "$HOST")" ]; then sg_warn "no TFC token (terraform login / TFE_TOKEN); skipping variable-set enrichment" exit 0 fi WORK="$(mktemp -d)" trap 'rm -rf "$WORK"' EXIT -AUTH=(-H "Authorization: Bearer $token") -# fetch_all — GET a paginated JSON:API collection, print the merged .data array. -fetch_all() { - local path="$1" page=1 next - : >"$WORK/acc.ndjson" - while :; do - if ! curl -fsS "${AUTH[@]}" "$API/$path?page%5Bsize%5D=100&page%5Bnumber%5D=$page" >"$WORK/page.json"; then - sg_err "TFC API request failed: $path" - return 1 - fi - "$JQ_BIN" -c '.data[]?' "$WORK/page.json" >>"$WORK/acc.ndjson" - next="$("$JQ_BIN" -r '.meta.pagination."next-page" // empty' "$WORK/page.json" 2>/dev/null || true)" - [ -z "$next" ] && break - page="$next" - done - "$JQ_BIN" -s '.' "$WORK/acc.ndjson" -} +# fetch_all — paginated GET, merged .data array (lib/tfc_api.sh). +fetch_all() { tfc_get_all "$1"; } sg_log "fetching workspaces and variable sets from $HOST (org: $ORG)..." @@ -74,7 +60,7 @@ fetch_all "organizations/$ORG/workspaces" | sets_raw="$(fetch_all "organizations/$ORG/varsets")" || exit 1 echo "$sets_raw" | "$JQ_BIN" -c '.[]' | while IFS= read -r s; do sid="$(echo "$s" | "$JQ_BIN" -r '.id')" - vars="$(curl -fsS "${AUTH[@]}" "$API/varsets/$sid/relationships/vars" 2>/dev/null | + vars="$(tfc_http "varsets/$sid/relationships/vars" 2>/dev/null | "$JQ_BIN" -c '[.data[]? | {key: .attributes.key, value: (.attributes.value // ""), category: .attributes.category, sensitive: (.attributes.sensitive // false), hcl: (.attributes.hcl // false)}]' 2>/dev/null || echo '[]')" echo "$s" | "$JQ_BIN" -c --argjson vars "$vars" '{ name: .attributes.name, diff --git a/scripts/lib/prompt.sh b/scripts/lib/prompt.sh new file mode 100644 index 0000000..3ca5540 --- /dev/null +++ b/scripts/lib/prompt.sh @@ -0,0 +1,121 @@ +#!/bin/bash +# Interactive prompt helpers for the migrator (sourced; needs tools.sh). +# +# Every prompt is written to stderr and read from the terminal; the answer is +# printed on stdout so callers capture it: v="$(sg_ask "Org" "demo")". +# Non-interactive runs (SG_NONINTERACTIVE=1, or no usable TTY) get the default +# answer, and fail when a prompt has none. Tests can script answers by pointing +# SG_ANSWERS_FILE at a file with one answer per line, consumed in order. + +# sg_interactive — true when we can talk to a terminal (and were not told not to). +sg_interactive() { + [ "${SG_NONINTERACTIVE:-0}" != "1" ] || return 1 + [ -n "${SG_ANSWERS_FILE:-}" ] && return 0 + (: /dev/null +} + +# The scripted-answers file is opened once, here, on fd 9: prompts run inside +# $(...) subshells, which inherit the descriptor and advance the shared offset, +# so answers are consumed in order across calls. +if [ -n "${SG_ANSWERS_FILE:-}" ]; then + exec 9<"$SG_ANSWERS_FILE" +fi + +# _sg_read — print the prompt, read one line (tty or answers +# file). Returns 1 on EOF so callers never loop on an exhausted input. +_sg_read() { + local ans + printf '%s' "$1" >&2 + if [ -n "${SG_ANSWERS_FILE:-}" ]; then + IFS= read -r ans <&9 || { printf '\n' >&2; return 1; } + printf '%s\n' "$ans" >&2 + else + IFS= read -r ans &2; return 1; } + fi + printf '%s' "$ans" +} + +# sg_ask [default] — free-text answer (default when empty). +sg_ask() { + local q="$1" def="${2-}" ans + if ! sg_interactive; then + [ -n "$def" ] || { sg_err "'$q' has no default and the run is non-interactive"; return 1; } + printf '%s' "$def" + return 0 + fi + if [ -n "$def" ]; then + ans="$(_sg_read "$(printf '%s? %s%s %s[%s]%s: ' "$C_BOLD" "$q" "$C_RESET" "$C_DIM" "$def" "$C_RESET")")" || { sg_err "input closed while asking '$q'"; return 1; } + else + ans="$(_sg_read "$(printf '%s? %s%s: ' "$C_BOLD" "$q" "$C_RESET")")" || { sg_err "input closed while asking '$q'"; return 1; } + fi + printf '%s' "${ans:-$def}" +} + +# sg_ask_required — like sg_ask, but re-prompts until non-empty. +sg_ask_required() { + local ans + while :; do + ans="$(sg_ask "$1")" || return 1 + [ -n "$ans" ] && { printf '%s' "$ans"; return 0; } + sg_warn "a value is required" + done +} + +# sg_confirm [Y|N] — exit 0 for yes, 1 for no. Default Y unless "N". +sg_confirm() { + local q="$1" def="${2:-Y}" ans hint + if ! sg_interactive; then + [ "$def" = "Y" ] && return 0 || return 1 + fi + [ "$def" = "Y" ] && hint="Y/n" || hint="y/N" + ans="$(_sg_read "$(printf '%s? %s%s %s[%s]%s ' "$C_BOLD" "$q" "$C_RESET" "$C_DIM" "$hint" "$C_RESET")")" || { sg_err "input closed while asking '$q'"; return 1; } + ans="${ans:-$def}" + case "$ans" in y | Y | yes | YES | Yes) return 0 ;; *) return 1 ;; esac +} + +# sg_select ... — numbered menu; prints the chosen item's value. +# Items are "value" or "value|description". The first item is the default. +# SG_SELECT_OTHER=1 adds an "Other (type a value)" entry for free text. +sg_select() { + local q="$1" + shift + local -a vals descs + local i n item ans + n=0 + for item in "$@"; do + vals[n]="${item%%|*}" + case "$item" in *"|"*) descs[n]="${item#*|}" ;; *) descs[n]="" ;; esac + n=$((n + 1)) + done + [ "$n" -gt 0 ] || { sg_err "sg_select: no items for '$q'"; return 1; } + if ! sg_interactive; then + printf '%s' "${vals[0]}" + return 0 + fi + printf '%s? %s%s\n' "$C_BOLD" "$q" "$C_RESET" >&2 + for ((i = 0; i < n; i++)); do + if [ -n "${descs[i]}" ]; then + printf ' %s%2d)%s %s %s%s%s\n' "$C_CYAN" "$((i + 1))" "$C_RESET" "${vals[i]}" "$C_DIM" "${descs[i]}" "$C_RESET" >&2 + else + printf ' %s%2d)%s %s\n' "$C_CYAN" "$((i + 1))" "$C_RESET" "${vals[i]}" >&2 + fi + done + [ "${SG_SELECT_OTHER:-0}" = "1" ] && printf ' %s%2d)%s Other (type a value)\n' "$C_CYAN" "$((n + 1))" "$C_RESET" >&2 + while :; do + ans="$(_sg_read "$(printf ' %schoice%s %s[1]%s: ' "$C_BOLD" "$C_RESET" "$C_DIM" "$C_RESET")")" || { sg_err "input closed while asking '$q'"; return 1; } + ans="${ans:-1}" + if [[ "$ans" =~ ^[0-9]+$ ]] && [ "$ans" -ge 1 ] && [ "$ans" -le "$n" ]; then + printf '%s' "${vals[ans - 1]}" + return 0 + fi + if [ "${SG_SELECT_OTHER:-0}" = "1" ] && [[ "$ans" =~ ^[0-9]+$ ]] && [ "$ans" -eq "$((n + 1))" ]; then + sg_ask_required "value" + return $? + fi + # Typing an item's value verbatim also works. + for ((i = 0; i < n; i++)); do + [ "$ans" = "${vals[i]}" ] && { printf '%s' "$ans"; return 0; } + done + sg_warn "enter a number between 1 and $n" + done +} diff --git a/scripts/lib/sg_api.sh b/scripts/lib/sg_api.sh new file mode 100644 index 0000000..9937a4b --- /dev/null +++ b/scripts/lib/sg_api.sh @@ -0,0 +1,136 @@ +#!/bin/bash +# StackGuardian API access (sourced; needs tools.sh). Expects SG_API_TOKEN, +# SG_BASE_URL and ORG to be set by the caller. Bodies go to stdout, logs to +# stderr. Return-code contract shared by every call: 0 on 2xx, 22 on a +# definitive 4xx (not retryable — pair with SG_NO_RETRY_RC=22), 1 on 5xx or a +# network error (retryable). SG_HTTP_CODE holds the last status. + +sg_org_url() { printf '%s/api/v1/orgs/%s' "$SG_BASE_URL" "$ORG"; } + +# sg_http_code — prints the HTTP status only (000 on network error). +sg_http_code() { + local code + code="$(curl -sS -o /dev/null -w '%{http_code}' -X "$1" -H "Authorization: apikey $SG_API_TOKEN" "$2" 2>/dev/null)" || true + sg_norm_code "$code" +} + +# sg_norm_code — curl prints "000" and exits non-zero on connection +# errors, so never append a fallback to its output; normalize instead. +sg_norm_code() { case "$1" in [0-9][0-9][0-9]) printf '%s' "$1" ;; *) printf '000' ;; esac; } + +# _sg_api [json-body] — shared request; body on stdout. +_sg_api() { + local method="$1" url="$2" body="${3-}" tmp code + tmp="$(mktemp)" + if [ -n "$body" ]; then + code="$(curl -sS -o "$tmp" -w '%{http_code}' -X "$method" \ + -H "Authorization: apikey $SG_API_TOKEN" -H "Content-Type: application/json" \ + -d "$body" "$url" 2>/dev/null)" || true + else + code="$(curl -sS -o "$tmp" -w '%{http_code}' -X "$method" \ + -H "Authorization: apikey $SG_API_TOKEN" "$url" 2>/dev/null)" || true + fi + code="$(sg_norm_code "$code")" + # shellcheck disable=SC2034 # read by callers + SG_HTTP_CODE="$code" + case "$code" in + 2*) + cat "$tmp" + rm -f "$tmp" + return 0 + ;; + 4*) + sg_err " HTTP $code from ${url#"$SG_BASE_URL"}: $(head -c 400 "$tmp")" + if declare -F explain_api_error >/dev/null; then explain_api_error "$(head -c 2000 "$tmp")"; fi + rm -f "$tmp" + return 22 + ;; + *) + sg_warn " HTTP $code from ${url#"$SG_BASE_URL"}" + rm -f "$tmp" + return 1 + ;; + esac +} + +sg_api_get() { _sg_api GET "$1"; } +sg_api_post() { _sg_api POST "$1" "$2" >/dev/null; } +sg_api_patch() { _sg_api PATCH "$1" "$2" >/dev/null; } + +# --- workflow groups ------------------------------------------------------- + +# wfgroup_http_code -> HTTP status of GET (200 exists, 404 missing). +wfgroup_http_code() { sg_http_code GET "$(sg_org_url)/wfgrps/$1/"; } + +# wfgroup_create — create a workflow group (idempotent at call sites). +wfgroup_create() { + local body + body="$("$(sg_resolve jq sg_ensure_jq)" -nc --arg n "$1" '{ResourceName:$n, Description:"Created by stackguardian-migrator (Terraform Cloud import)"}')" + SG_NO_RETRY_RC=22 sg_retry "$RETRIES" "$RETRY_BASE" -- sg_api_post "$(sg_org_url)/wfgrps/" "$body" +} + +# --- workflows ------------------------------------------------------------- + +wf_url() { printf '%s/wfgrps/%s/wfs/%s/' "$(sg_org_url)" "$1" "$2"; } +wf_triggers_endpoint() { printf '%swebhooks/vcs_triggers/' "$(wf_url "$1" "$2")"; } + +# sg_workflow_exists — exit 0 when the workflow exists. +sg_workflow_exists() { [ "$(sg_http_code GET "$(wf_url "$1" "$2")")" = "200" ]; } + +# sg_list_workflows — ["wf-name", ...] in the group ([] on 404). +sg_list_workflows() { + local body + if body="$(sg_api_get "$(sg_org_url)/wfgrps/$1/wfs/listall/" 2>/dev/null)"; then + printf '%s' "$body" | "$(sg_resolve jq sg_ensure_jq)" -c '[(.msg // .data // [])[] | .ResourceName] | map(select(. != null))' + else + echo '[]' + fi +} + +# sg_patch_workflow — PATCH a workflow. +sg_patch_workflow() { sg_api_patch "$(wf_url "$1" "$2")" "$3"; } + +# --- integrations (connectors) -------------------------------------------- + +# sg_list_integrations — [{name, type}, ...] for the org (fails on error). +sg_list_integrations() { + local body + body="$(sg_api_get "$(sg_org_url)/integrations/listall/")" || return $? + printf '%s' "$body" | "$(sg_resolve jq sg_ensure_jq)" -c '[(.msg // .data // [])[] | {name: .ResourceName, type: .ResourceType}] | map(select(.name != null))' +} + +# sg_integration_exists — exit 0 when it exists. +sg_integration_exists() { + local n="${1#/integrations/}" + [ "$(sg_http_code GET "$(sg_org_url)/integrations/$n/")" = "200" ] +} + +# --- runner groups --------------------------------------------------------- + +# sg_runnergroup_exists — exit 0 when the runner group exists. +sg_runnergroup_exists() { [ "$(sg_http_code GET "$(sg_org_url)/runnergroups/$1/")" = "200" ]; } + +# sg_list_runnergroups — ["name", ...]; empty output (exit 1) when the API has +# no list endpoint, so callers fall back to free text. +sg_list_runnergroups() { + local body + body="$(sg_api_get "$(sg_org_url)/runnergroups/listall/" 2>/dev/null)" || return 1 + printf '%s' "$body" | "$(sg_resolve jq sg_ensure_jq)" -c '[(.msg // .data // [])[] | .ResourceName] | map(select(. != null))' +} + +# --- secrets --------------------------------------------------------------- + +# sg_secret_exists — exit 0 when a secret with that name exists. +sg_secret_exists() { + local body + body="$(sg_api_get "$(sg_org_url)/secrets/listall/" 2>/dev/null)" || return 1 + printf '%s' "$body" | "$(sg_resolve jq sg_ensure_jq)" -e --arg n "$1" '[(.msg // .data // [])[] | .ResourceName] | index($n) != null' >/dev/null +} + +# sg_create_secret [description] +sg_create_secret() { + local body + body="$("$(sg_resolve jq sg_ensure_jq)" -nc --arg n "$1" --arg v "$2" --arg d "${3:-Created by stackguardian-migrator (placeholder — set the real value)}" \ + '{ResourceName:$n, ResourceType:"SECRET", Description:$d, Value:$v}')" + SG_NO_RETRY_RC=22 sg_retry "$RETRIES" "$RETRY_BASE" -- sg_api_post "$(sg_org_url)/secrets/" "$body" +} diff --git a/scripts/lib/tfc_api.sh b/scripts/lib/tfc_api.sh new file mode 100644 index 0000000..5a9c74e --- /dev/null +++ b/scripts/lib/tfc_api.sh @@ -0,0 +1,105 @@ +#!/bin/bash +# Terraform Cloud/Enterprise API access (sourced; needs tools.sh + tfvars.sh). +# +# Token resolution mirrors the tfe provider: TFE_TOKEN, then TF_TOKEN_, +# then the `terraform login` credentials file. All helpers print JSON on stdout +# and log on stderr. + +# tfc_hostname — TFC/TFE host from terraform.tfvars (tfHostname), default app.terraform.io. +tfc_hostname() { + local h + h="$(tfvars_get '.tfHostname')" + printf '%s' "${h:-app.terraform.io}" +} + +# tfc_token — the token the tfe provider will use, or nothing. +tfc_token() { + local host="$1" var creds + if [ -n "${TFE_TOKEN:-}" ]; then printf '%s' "$TFE_TOKEN"; return; fi + var="TF_TOKEN_$(echo "$host" | sed 's/-/__/g; s/\./_/g')" + if [ -n "${!var:-}" ]; then printf '%s' "${!var}"; return; fi + creds="$HOME/.terraform.d/credentials.tfrc.json" + [ -f "$creds" ] && "$(sg_resolve jq sg_ensure_jq)" -r --arg h "$host" '.credentials[$h].token // empty' "$creds" 2>/dev/null || true +} + +# tfc_token_source — human description of where the token came from. +tfc_token_source() { + if [ -n "${TFE_TOKEN:-}" ]; then printf 'TFE_TOKEN' + elif env | grep -q '^TF_TOKEN_'; then printf 'the TF_TOKEN_* variable' + else printf "the 'terraform login' session"; fi +} + +# tfc_http [curl-args...] — GET https:///api/v2/, body on +# stdout. Exit 0 on 2xx, 22 on 4xx, 1 on 5xx/network. Sets TFC_HTTP_CODE. +tfc_http() { + local path="$1" host token tmp code + shift + host="${TFC_HOST:-$(tfc_hostname)}" + token="$(tfc_token "$host")" + tmp="$(mktemp)" + code="$(curl -sS -o "$tmp" -w '%{http_code}' -H "Authorization: Bearer $token" \ + -H "Content-Type: application/vnd.api+json" "$@" "https://$host/api/v2/$path" 2>/dev/null)" || true + case "$code" in [0-9][0-9][0-9]) ;; *) code=000 ;; esac + # shellcheck disable=SC2034 # read by callers + TFC_HTTP_CODE="$code" + case "$code" in + 2*) cat "$tmp"; rm -f "$tmp"; return 0 ;; + 4*) rm -f "$tmp"; return 22 ;; + *) rm -f "$tmp"; return 1 ;; + esac +} + +# tfc_get_all — GET a paginated JSON:API collection; prints the merged +# .data array. Fails (non-zero) if any page fails. +tfc_get_all() { + local path="$1" page=1 next acc sep body jqb + jqb="$(sg_resolve jq sg_ensure_jq)" + acc="$(mktemp)" + : >"$acc" + case "$path" in *\?*) sep='&' ;; *) sep='?' ;; esac + while :; do + if ! body="$(tfc_http "${path}${sep}page%5Bsize%5D=100&page%5Bnumber%5D=$page")"; then + sg_err "TFC API request failed (HTTP ${TFC_HTTP_CODE:-000}): $path" + rm -f "$acc" + return 1 + fi + printf '%s' "$body" | "$jqb" -c '.data[]?' >>"$acc" + next="$(printf '%s' "$body" | "$jqb" -r '.meta.pagination."next-page" // empty' 2>/dev/null || true)" + [ -z "$next" ] && break + page="$next" + done + "$jqb" -s '.' "$acc" + rm -f "$acc" +} + +# tfc_list_orgs — ["org-name", ...] the token can see. +tfc_list_orgs() { tfc_get_all "organizations" | "$(sg_resolve jq sg_ensure_jq)" -c '[.[].id]'; } + +# tfc_list_projects — [{id, name}, ...] +tfc_list_projects() { tfc_get_all "organizations/$1/projects" | "$(sg_resolve jq sg_ensure_jq)" -c '[.[] | {id: .id, name: .attributes.name}]'; } + +# tfc_list_workspaces — [{name, id, project, tags, terraform_version, execution_mode}, ...] +tfc_list_workspaces() { + tfc_get_all "organizations/$1/workspaces" | "$(sg_resolve jq sg_ensure_jq)" -c \ + '[.[] | {name: .attributes.name, id: .id, project: (.relationships.project.data.id // ""), tags: (.attributes."tag-names" // []), terraform_version: .attributes."terraform-version", execution_mode: .attributes."execution-mode"}]' +} + +# require_tfc_auth — fail fast when the tfe provider would not be able to +# authenticate: no credential at all, or one that TFC/TFE rejects (e.g. an +# expired `terraform login` session). Verified with GET /account/details. +require_tfc_auth() { + local host token + host="$(tfc_hostname)" + token="$(tfc_token "$host")" + [ -n "$token" ] || die "no Terraform Cloud/Enterprise credentials found for $host. Set TFE_TOKEN= (recommended) or run 'terraform login' before '$PROG apply'." + command -v curl >/dev/null 2>&1 || return 0 + if tfc_http "account/details" >/dev/null; then + sg_log "TFC/TFE credentials verified ($host)" + return 0 + fi + case "$TFC_HTTP_CODE" in + 401 | 403) die "Terraform Cloud/Enterprise rejected $(tfc_token_source) for $host (HTTP $TFC_HTTP_CODE). Set TFE_TOKEN= or re-run 'terraform login'." ;; + 000) die "could not reach https://$host to verify TFC/TFE credentials (network/proxy?)" ;; + *) sg_warn "unexpected HTTP $TFC_HTTP_CODE verifying TFC/TFE credentials at $host; continuing" ;; + esac +} diff --git a/scripts/lib/tfvars.sh b/scripts/lib/tfvars.sh new file mode 100644 index 0000000..d6073e6 --- /dev/null +++ b/scripts/lib/tfvars.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# Read access to transformer/terraform-cloud/terraform.tfvars (sourced; needs +# tools.sh and TFVARS to be set). The file is converted once with hcl2json and +# cached for the life of the process. + +_TFVARS_JSON="" +_TFVARS_JSON_FOR="" + +# tfvars_json — the whole tfvars file as JSON (empty object when missing). +tfvars_json() { + if [ -z "$_TFVARS_JSON" ] || [ "$_TFVARS_JSON_FOR" != "$TFVARS" ]; then + if [ -f "$TFVARS" ]; then + _TFVARS_JSON="$("$(sg_resolve hcl2json sg_ensure_hcl2json)" "$TFVARS" 2>/dev/null || echo '{}')" + else + _TFVARS_JSON='{}' + fi + _TFVARS_JSON_FOR="$TFVARS" + fi + printf '%s' "$_TFVARS_JSON" +} + +# tfvars_get [default] — raw value of an expression over the tfvars +# JSON, e.g. tfvars_get '.tfOrg'. Prints the default when null/missing. +tfvars_get() { + local expr="$1" def="${2-}" v + v="$(tfvars_json | "$(sg_resolve jq sg_ensure_jq)" -r "$expr // empty" 2>/dev/null || true)" + printf '%s' "${v:-$def}" +} + +# tfvars_get_json — compact JSON value of an expression (or null). +tfvars_get_json() { + tfvars_json | "$(sg_resolve jq sg_ensure_jq)" -c "$1 // null" 2>/dev/null || echo null +} + +# tfvars_invalidate — forget the cached conversion (after writing the file). +tfvars_invalidate() { _TFVARS_JSON=""; } diff --git a/scripts/migrate.sh b/scripts/migrate.sh index 4d1f3a3..5f7439b 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -9,6 +9,14 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck source=tools.sh source "$SCRIPT_DIR/tools.sh" +# shellcheck source=lib/prompt.sh +source "$SCRIPT_DIR/lib/prompt.sh" +# shellcheck source=lib/tfvars.sh +source "$SCRIPT_DIR/lib/tfvars.sh" +# shellcheck source=lib/tfc_api.sh +source "$SCRIPT_DIR/lib/tfc_api.sh" +# shellcheck source=lib/sg_api.sh +source "$SCRIPT_DIR/lib/sg_api.sh" # SCRIPT_DIR holds the sibling scripts; SG_REPO_ROOT (from tools.sh) is the repo # root used for all repo-relative paths. @@ -148,7 +156,7 @@ completion_hint() { sg_log "tab completion for this shell session: source <(./sg-migrate.sh completion $sh)" } -# --- StackGuardian API helpers (workflow groups) --------------------------- +# --- workflow-group mapping (API helpers live in lib/sg_api.sh) ------------- # group_for -> the SG workflow group for a project segment: an entry # from the optional override map, else the default tfc-. @@ -160,28 +168,6 @@ group_for() { [ -n "$override" ] && echo "$override" || echo "tfc-$seg" } -# wfgroup_http_code -> HTTP status of GET (200 exists, 404 missing). -wfgroup_http_code() { - curl -sS -o /dev/null -w '%{http_code}' \ - -H "Authorization: apikey $SG_API_TOKEN" \ - "$SG_BASE_URL/api/v1/orgs/$ORG/wfgrps/$1/" -} - -# wfgroup_create — create a workflow group (idempotent at call sites). -wfgroup_create() { - local body - body="$("$JQ_BIN" -nc --arg n "$1" '{ResourceName:$n, Description:"Created by stackguardian-migrator (Terraform Cloud import)"}')" - sg_retry "$RETRIES" "$RETRY_BASE" -- \ - curl -fsS -X POST \ - -H "Authorization: apikey $SG_API_TOKEN" -H "Content-Type: application/json" \ - -d "$body" "$SG_BASE_URL/api/v1/orgs/$ORG/wfgrps/" >/dev/null -} - -# wf_triggers_endpoint -> the VCS-triggers webhook URL for a workflow. -wf_triggers_endpoint() { - echo "$SG_BASE_URL/api/v1/orgs/$ORG/wfgrps/$1/wfs/$2/webhooks/vcs_triggers/" -} - # do_set_triggers — for each workflow in the file with a non-null # VCSTriggers block, register its VCS triggers via the dedicated webhooks # endpoint (the bulk create API silently drops VCSTriggers; this second pass is @@ -200,7 +186,7 @@ do_set_triggers() { fi wf="$("$JQ_BIN" -r --argjson i "$i" '.[$i].ResourceName' "$f")" # A workflow that failed to import has nothing to attach triggers to. - if [ "$(sg_http_code GET "$SG_BASE_URL/api/v1/orgs/$ORG/wfgrps/$grp/wfs/$wf/")" != "200" ]; then + if ! sg_workflow_exists "$grp" "$wf"; then sg_warn " $grp/$wf does not exist in SG (import failed?) — skipping triggers" rc=1 continue @@ -218,37 +204,6 @@ do_set_triggers() { return "$rc" } -# sg_http_code — prints the HTTP status (000 on network error). -sg_http_code() { - curl -sS -o /dev/null -w '%{http_code}' -X "$1" -H "Authorization: apikey $SG_API_TOKEN" "$2" 2>/dev/null || echo 000 -} - -# sg_api_post — POST to the SG API. Exit 0 on 2xx, 22 on a -# definitive 4xx (not retryable; response echoed), 1 on 5xx/network (retryable). -sg_api_post() { - local url="$1" body="$2" tmp code - tmp="$(mktemp)" - code="$(curl -sS -o "$tmp" -w '%{http_code}' -X POST \ - -H "Authorization: apikey $SG_API_TOKEN" -H "Content-Type: application/json" \ - -d "$body" "$url" 2>/dev/null || echo 000)" - case "$code" in - 2*) - rm -f "$tmp" - return 0 - ;; - 4*) - sg_err " HTTP $code from ${url#"$SG_BASE_URL"}: $(head -c 400 "$tmp")" - rm -f "$tmp" - return 22 - ;; - *) - sg_warn " HTTP $code from ${url#"$SG_BASE_URL"}" - rm -f "$tmp" - return 1 - ;; - esac -} - # set_triggers_pass — run do_set_triggers over all payload files (assumes # JQ_BIN/SG_API_TOKEN/ORG are already set up by the caller). set_triggers_pass() { @@ -294,48 +249,6 @@ cmd_clean() { sg_success "clean complete" } -# tfc_hostname — TFC/TFE host from terraform.tfvars (tfHostname), default app.terraform.io. -tfc_hostname() { - local h="" - if [ -f "$TFVARS" ]; then - h="$("$(sg_resolve hcl2json sg_ensure_hcl2json)" "$TFVARS" 2>/dev/null | "$(sg_resolve jq sg_ensure_jq)" -r '.tfHostname // empty' 2>/dev/null || true)" - fi - echo "${h:-app.terraform.io}" -} - -# tfc_token — resolve the TFC/TFE token the tfe provider will use, in -# terraform's own precedence: TFE_TOKEN, TF_TOKEN_ ('.'->'_', '-'->'__'), -# then the `terraform login` credentials file. Prints nothing if none is set. -tfc_token() { - local host="$1" var creds - if [ -n "${TFE_TOKEN:-}" ]; then echo "$TFE_TOKEN"; return; fi - var="TF_TOKEN_$(echo "$host" | sed 's/-/__/g; s/\./_/g')" - if [ -n "${!var:-}" ]; then echo "${!var}"; return; fi - creds="$HOME/.terraform.d/credentials.tfrc.json" - [ -f "$creds" ] && "$(sg_resolve jq sg_ensure_jq)" -r --arg h "$host" '.credentials[$h].token // empty' "$creds" 2>/dev/null || true -} - -# require_tfc_auth — fail fast, before the (long) apply, when the tfe provider -# would not be able to authenticate: no credential at all, or a credential -# that TFC/TFE rejects (e.g. an expired `terraform login` session). Verified -# with GET /api/v2/account/details. Without this, terraform fails later with a -# confusing "Invalid provider configuration" error. -require_tfc_auth() { - local host token code src - host="$(tfc_hostname)" - token="$(tfc_token "$host")" - if [ -n "${TFE_TOKEN:-}" ]; then src="TFE_TOKEN"; elif env | grep -q '^TF_TOKEN_'; then src="the TF_TOKEN_* variable"; else src="the 'terraform login' session (likely expired)"; fi - [ -n "$token" ] || die "no Terraform Cloud/Enterprise credentials found for $host. Set TFE_TOKEN= (recommended) or run 'terraform login' before '$PROG apply'." - command -v curl >/dev/null 2>&1 || return 0 - code="$(curl -sS -o /dev/null -w '%{http_code}' -H "Authorization: Bearer $token" "https://$host/api/v2/account/details" 2>/dev/null || echo 000)" - case "$code" in - 200) sg_log "TFC/TFE credentials verified ($host)" ;; - 401 | 403) die "Terraform Cloud/Enterprise rejected $src for $host (HTTP $code). Set TFE_TOKEN= or re-run 'terraform login'." ;; - 000) die "could not reach https://$host to verify TFC/TFE credentials (network/proxy?)" ;; - *) sg_warn "unexpected HTTP $code verifying TFC/TFE credentials at $host; continuing" ;; - esac -} - cmd_apply() { sg_step "Phase: apply (terraform)" command -v terraform >/dev/null 2>&1 || die "terraform not found on PATH" @@ -381,13 +294,10 @@ cmd_enrich() { payload_files [ "${#PF[@]}" -gt 0 ] || die "No payload files in $(sg_rel "$EXPORT_DIR") (run 'apply' first)." # Variable sets belong to the TFC org (tfOrg in terraform.tfvars), not SG_ORG. - local jqb h2j tforg tfhost - jqb="$(sg_resolve jq sg_ensure_jq)" - h2j="$(sg_resolve hcl2json sg_ensure_hcl2json)" - tforg="$("$h2j" "$TFVARS" | "$jqb" -r '.tfOrg // empty')" + local tforg + tforg="$(tfvars_get '.tfOrg')" [ -n "$tforg" ] || die "tfOrg not found in $(sg_rel "$TFVARS")" - tfhost="$("$h2j" "$TFVARS" | "$jqb" -r '.tfHostname // empty')" - SG_TFC_HOSTNAME="${tfhost:-app.terraform.io}" "$SCRIPT_DIR/enrich_variable_sets.sh" "$tforg" "${PF[@]}" + SG_TFC_HOSTNAME="$(tfc_hostname)" "$SCRIPT_DIR/enrich_variable_sets.sh" "$tforg" "${PF[@]}" } do_convert() { "$SCRIPT_DIR/convert_hcl_to_json.sh" "$1"; } @@ -581,11 +491,7 @@ cmd_import() { SGCLI_BIN="$(sg_resolve sg-cli sg_ensure_sgcli)" # Fallback Terraform version for workflows the API rejects as above the # managed ceiling: SGDefaultTerraformVersion from terraform.tfvars, else 1.5.7. - SG_DEFAULT_TF_VERSION="${SG_DEFAULT_TF_VERSION:-}" - if [ -z "$SG_DEFAULT_TF_VERSION" ] && [ -f "$TFVARS" ]; then - SG_DEFAULT_TF_VERSION="$("$(sg_resolve hcl2json sg_ensure_hcl2json)" "$TFVARS" | "$JQ_BIN" -r '.SGDefaultTerraformVersion // empty')" - fi - SG_DEFAULT_TF_VERSION="${SG_DEFAULT_TF_VERSION:-TERRAFORM-1.5.7}" + SG_DEFAULT_TF_VERSION="${SG_DEFAULT_TF_VERSION:-$(tfvars_get '.SGDefaultTerraformVersion' TERRAFORM-1.5.7)}" rm -f "$EXPORT_DIR/terraform-version-fallbacks.log" sg_log "importing ${#PF[@]} payload(s), up to $CONC in parallel (retries: $RETRIES)" From 0d408c0cb1b12745ee0d888597c961d78ca39dd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= Date: Mon, 7 Sep 2026 09:18:23 +0200 Subject: [PATCH 14/30] feat: interactive init wizard with TFC and SG discovery 'init' now walks through the configuration instead of copying the example file: it lists the TFC organisations, projects and workspaces the token can see, the SG VCS and cloud connectors and runner groups, and derives the source kind and repo prefix from the chosen connector. Every step falls back to free text when a token is missing or an API call fails, and non-interactive runs keep the template behaviour. The generated terraform.tfvars keeps the example's order and comments; re-running the wizard prefills the current values and keeps a .bak. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01YKBeQrpJgR9DVFtR83myqX --- scripts/lib/tfvars.sh | 79 +++++++++++++++ scripts/lib/wizard.sh | 230 ++++++++++++++++++++++++++++++++++++++++++ scripts/migrate.sh | 24 ++++- sg-migrate.sh | 2 +- 4 files changed, 330 insertions(+), 5 deletions(-) create mode 100644 scripts/lib/wizard.sh diff --git a/scripts/lib/tfvars.sh b/scripts/lib/tfvars.sh index d6073e6..8bfc766 100644 --- a/scripts/lib/tfvars.sh +++ b/scripts/lib/tfvars.sh @@ -34,3 +34,82 @@ tfvars_get_json() { # tfvars_invalidate — forget the cached conversion (after writing the file). tfvars_invalidate() { _TFVARS_JSON=""; } + +# tfvars_write — render terraform.tfvars from W_* variables set by the +# wizard (lists/objects are passed as compact JSON, which HCL accepts). Keeps +# the same order and comments as terraform.tfvars.example so the file stays +# hand-editable afterwards. +# W_TFORG W_TFHOST W_WSNAMES_JSON W_TAGS_JSON W_IGNORE_TAGS_JSON W_EXPORT_STATE +# W_APPROVERS_JSON W_REPO_PREFIX W_VCS_INTEGRATION W_DPC_JSON W_RUNNER_JSON +# W_DEST_KIND W_TF_VERSION W_TRIGGERS +tfvars_write() { + local dest="$1" host_line="" + if [ "${W_TFHOST:-app.terraform.io}" != "app.terraform.io" ]; then + host_line="$(printf '\n# Terraform Enterprise hostname (omit for Terraform Cloud)\ntfHostname = "%s"\n' "$W_TFHOST")" + fi + cat >"$dest" <) +SGDefaultVCSAuthIntegrationID = "$W_VCS_INTEGRATION" + +# Cloud connector the workflows deploy with +SGDefaultDeploymentPlatformConfig = $W_DPC_JSON + +# Runners for every workflow: { type = "shared" } for SG-hosted runners, or +# { type = "private", names = [""] } for a private runner group. +SGDefaultRunnerConstraints = $W_RUNNER_JSON + +# Choose from: GITHUB_COM, BITBUCKET_ORG, GITLAB_COM, AZURE_DEVOPS, GIT_OTHER +SGDefaultSourceConfigDestKind = "$W_DEST_KIND" + +# SG Terraform version used when a workspace's version is not a pinned semver, or +# when the SG API rejects a pinned version as above the managed ceiling (1.5.7, +# the last MPL/FOSS release; newer versions are BSL and not bundled). +SGDefaultTerraformVersion = "$W_TF_VERSION" + +# Pre-configure VCS triggers on each workflow from the workspace's TFC settings +SGDefaultEnableVCSTriggers = $W_TRIGGERS + +# Re-pull state for every workspace on each apply (default: idempotent) +forceStateRefresh = false + +# Per-workspace overrides, keyed by workspace name. Any field set here wins over +# the SGDefault* value above, for that workspace only. See terraform.tfvars.example +# for every supported field. +# workspaceOverrides = { +# "prod-networking" = { +# RunnerConstraints = { "type" : "private", "names" : ["sg-runner"] } +# Approvers = ["lead@example.com"] +# terraformVersion = "TERRAFORM-1.5.7" +# } +# } +TFVARS + tfvars_invalidate +} diff --git a/scripts/lib/wizard.sh b/scripts/lib/wizard.sh new file mode 100644 index 0000000..5096092 --- /dev/null +++ b/scripts/lib/wizard.sh @@ -0,0 +1,230 @@ +#!/bin/bash +# Interactive 'init' wizard (sourced; needs tools.sh, prompt.sh, tfvars.sh, +# tfc_api.sh, sg_api.sh). Discovers TFC orgs/workspaces and SG integrations / +# runner groups with the tokens already in the environment and writes +# terraform.tfvars. Every step degrades to free-text entry when a token is +# missing or an API call fails, so the wizard always completes. + +# _w_default — current tfvars value (re-run) or fallback. +_w_default() { local v; v="$(tfvars_get "$1")"; printf '%s' "${v:-$2}"; } + +# _w_csv_json — "a, b" -> ["a","b"]; empty -> []. +_w_csv_json() { + local jqb + jqb="$(sg_resolve jq sg_ensure_jq)" + printf '%s' "$1" | tr ',' '\n' | sed 's/^ *//; s/ *$//' | grep -v '^$' | "$jqb" -R . | "$jqb" -sc . 2>/dev/null || echo '[]' +} + +# _w_repo_prefix_for — proposed repo URL prefix. +_w_repo_prefix_for() { + case "$1" in + GITHUB_COM) printf 'https://github.com' ;; + GITLAB_COM) printf 'https://gitlab.com' ;; + BITBUCKET_ORG) printf 'https://bitbucket.org' ;; + AZURE_DEVOPS) printf 'https://dev.azure.com' ;; + *) printf 'https://VCS_PROVIDER_DOMAIN' ;; + esac +} + +# --- step 1: Terraform Cloud ------------------------------------------------ +wizard_tfc() { + local host token orgs n ws projects wsn prn scope tags alltags + local jqb + jqb="$(sg_resolve jq sg_ensure_jq)" + sg_step "1/4 Terraform Cloud / Enterprise" + host="$(sg_ask "TFC/TFE hostname" "$(_w_default .tfHostname app.terraform.io)")" || return 1 + W_TFHOST="$host" + # shellcheck disable=SC2034 # consumed by tfc_http (lib/tfc_api.sh) + TFC_HOST="$host" + token="$(tfc_token "$host")" + W_TFC_DISCOVERY=0 + if [ -z "$token" ]; then + sg_warn "no TFC credentials found for $host (TFE_TOKEN or 'terraform login'); organisations and workspaces cannot be listed" + elif ! tfc_http "account/details" >/dev/null; then + sg_warn "TFC rejected $(tfc_token_source) for $host (HTTP $TFC_HTTP_CODE); continuing without discovery" + else + W_TFC_DISCOVERY=1 + fi + + if [ "$W_TFC_DISCOVERY" -eq 1 ] && orgs="$(tfc_list_orgs 2>/dev/null)" && [ "$(printf '%s' "$orgs" | "$jqb" 'length')" -gt 0 ]; then + n="$(printf '%s' "$orgs" | "$jqb" 'length')" + if [ "$n" -eq 1 ]; then + W_TFORG="$(printf '%s' "$orgs" | "$jqb" -r '.[0]')" + sg_log "organisation: $W_TFORG (the only one this token can see)" + else + # shellcheck disable=SC2046 + W_TFORG="$(SG_SELECT_OTHER=1 sg_select "Which TFC organisation do you want to migrate?" $(printf '%s' "$orgs" | "$jqb" -r '.[]'))" || return 1 + fi + else + W_TFORG="$(sg_ask "TFC organisation name" "$(_w_default .tfOrg '')")" || return 1 + [ -n "$W_TFORG" ] || W_TFORG="$(sg_ask_required "TFC organisation name")" || return 1 + fi + + W_WSNAMES_JSON='["*"]' + W_TAGS_JSON=null + W_IGNORE_TAGS_JSON=null + if [ "$W_TFC_DISCOVERY" -eq 1 ] && ws="$(tfc_list_workspaces "$W_TFORG" 2>/dev/null)"; then + wsn="$(printf '%s' "$ws" | "$jqb" 'length')" + projects="$(tfc_list_projects "$W_TFORG" 2>/dev/null || echo '[]')" + prn="$(printf '%s' "$projects" | "$jqb" 'length')" + sg_log "found $wsn workspace(s) in $prn project(s); each project becomes SG workflow group tfc-" + W_WS_COUNT="$wsn" + W_WS_ABOVE_CEILING="$(printf '%s' "$ws" | "$jqb" '[.[] | select((.terraform_version // "") | test("^[0-9]+\\.[0-9]+\\.[0-9]+$")) | select(((.terraform_version | split(".") | map(tonumber)) as $v | ($v[0] > 1) or ($v[0] == 1 and $v[1] > 5) or ($v[0] == 1 and $v[1] == 5 and $v[2] > 7)))] | length')" + alltags="$(printf '%s' "$ws" | "$jqb" -r '[.[].tags[]?] | unique | join(", ")')" + else + alltags="" + fi + scope="$(sg_select "Which workspaces should be migrated?" \ + "all|every workspace in the organisation" \ + "tags|only workspaces carrying certain tags" \ + "exclude|all workspaces except those carrying certain tags" \ + "names|specific workspace names")" || return 1 + case "$scope" in + tags) + [ -n "$alltags" ] && sg_dim "tags in use: $alltags" + tags="$(sg_ask_required "Tags to include (comma-separated)")" || return 1 + W_TAGS_JSON="$(_w_csv_json "$tags")" + ;; + exclude) + [ -n "$alltags" ] && sg_dim "tags in use: $alltags" + tags="$(sg_ask_required "Tags to exclude (comma-separated)")" || return 1 + W_IGNORE_TAGS_JSON="$(_w_csv_json "$tags")" + ;; + names) + tags="$(sg_ask_required "Workspace names (comma-separated, * wildcards allowed)")" || return 1 + W_WSNAMES_JSON="$(_w_csv_json "$tags")" + ;; + esac +} + +# --- step 2: StackGuardian --------------------------------------------------- +wizard_sg() { + local ints vcs cloud pick kind name runner groups jqb + jqb="$(sg_resolve jq sg_ensure_jq)" + sg_step "2/4 StackGuardian" + if [ -z "$ORG" ]; then + ORG="$(sg_ask_required "StackGuardian organisation")" || return 1 + else + sg_log "organisation: $ORG (from --org / SG_ORG)" + fi + W_SG_DISCOVERY=0 + if [ -z "${SG_API_TOKEN:-}" ]; then + sg_warn "SG_API_TOKEN is not set; connectors and runner groups cannot be listed (export it and re-run 'init' to pick from a list)" + elif ! ints="$(sg_list_integrations)"; then + case "$SG_HTTP_CODE" in + 401 | 403) sg_warn "StackGuardian rejected SG_API_TOKEN for org '$ORG' (HTTP $SG_HTTP_CODE); continuing without discovery" ;; + 404) sg_warn "StackGuardian org '$ORG' not found at $SG_BASE_URL (HTTP 404); continuing without discovery" ;; + *) sg_warn "could not list connectors (HTTP $SG_HTTP_CODE); continuing without discovery" ;; + esac + else + W_SG_DISCOVERY=1 + fi + + # VCS connector -> integration id, source kind, repo prefix. + vcs="" + [ "$W_SG_DISCOVERY" -eq 1 ] && vcs="$(printf '%s' "$ints" | "$jqb" -r '.[] | select(.type | test("^(GITHUB_COM|GITHUB_APP_CUSTOM|GITLAB_COM|BITBUCKET_ORG|AZURE_DEVOPS|GIT_OTHER)$")) | "\(.name)|\(.type)"')" + if [ -n "$vcs" ]; then + # shellcheck disable=SC2046 + pick="$(SG_SELECT_OTHER=1 sg_select "Which VCS connector should clone the repositories?" $(printf '%s\n' "$vcs" | tr '\n' ' '))" || return 1 + kind="$(printf '%s' "$ints" | "$jqb" -r --arg n "$pick" '.[] | select(.name == $n) | .type' | head -1)" + else + pick="$(sg_ask "VCS connector name (as in StackGuardian, e.g. github_com)" "$(_w_default .SGDefaultVCSAuthIntegrationID '' | sed 's#^/integrations/##')")" || return 1 + [ -n "$pick" ] || pick="$(sg_ask_required "VCS connector name")" || return 1 + kind="" + fi + W_VCS_INTEGRATION="/integrations/${pick#/integrations/}" + case "$kind" in + GITHUB_APP_CUSTOM) W_DEST_KIND=GITHUB_COM ;; + GITHUB_COM | GITLAB_COM | BITBUCKET_ORG | AZURE_DEVOPS | GIT_OTHER) W_DEST_KIND="$kind" ;; + *) W_DEST_KIND="$(sg_select "VCS provider kind" GITHUB_COM GITLAB_COM BITBUCKET_ORG AZURE_DEVOPS GIT_OTHER)" || return 1 ;; + esac + W_REPO_PREFIX="$(sg_ask "Repository URL prefix" "$(_w_default .SGDefaultIACVCSRepoPrefix "$(_w_repo_prefix_for "$W_DEST_KIND")")")" || return 1 + + # Cloud connector -> DeploymentPlatformConfig. + cloud="" + [ "$W_SG_DISCOVERY" -eq 1 ] && cloud="$(printf '%s' "$ints" | "$jqb" -r '.[] | select(.type | test("^(AWS|AZURE|GCP)_")) | "\(.name)|\(.type)"')" + if [ -n "$cloud" ]; then + # shellcheck disable=SC2046 + pick="$(SG_SELECT_OTHER=1 sg_select "Which cloud connector should the workflows deploy with?" $(printf '%s\n' "$cloud" | tr '\n' ' ') "skip|decide later (leaves a placeholder to edit)")" || return 1 + kind="$(printf '%s' "$ints" | "$jqb" -r --arg n "$pick" '.[] | select(.name == $n) | .type' | head -1)" + else + pick="$(sg_ask "Cloud connector name (as in StackGuardian; empty to decide later)" "$(_w_default .SGDefaultDeploymentPlatformConfig[0].config.integrationId '' | sed 's#^/integrations/##')")" || return 1 + kind="" + fi + if [ -z "$pick" ] || [ "$pick" = "skip" ]; then + W_DPC_JSON='[{"kind":"AWS_RBAC","config":{"integrationId":"/integrations/CHANGE_ME","profileName":"default"}}]' + W_DPC_PLACEHOLDER=1 + else + [ -n "$kind" ] || kind="$(sg_select "Connector kind" AWS_RBAC AWS_STATIC AWS_OIDC AZURE_STATIC AZURE_OIDC AZURE_MANAGED_ID_OIDC GCP_STATIC GCP_OIDC)" || return 1 + name="$(sg_ask "Profile name" "$(_w_default .SGDefaultDeploymentPlatformConfig[0].config.profileName default)")" || return 1 + W_DPC_JSON="$("$jqb" -nc --arg k "$kind" --arg i "/integrations/${pick#/integrations/}" --arg p "$name" '[{kind:$k, config:{integrationId:$i, profileName:$p}}]')" + W_DPC_PLACEHOLDER=0 + fi + + # Runner constraints. + runner="$(sg_select "Where should the workflows run?" \ + "shared|StackGuardian-hosted shared runners" \ + "private|a private runner group in your own network")" || return 1 + if [ "$runner" = "private" ]; then + groups="" + [ "$W_SG_DISCOVERY" -eq 1 ] && groups="$(sg_list_runnergroups 2>/dev/null | "$jqb" -r '.[]' 2>/dev/null || true)" + if [ -n "$groups" ]; then + # shellcheck disable=SC2046 + name="$(SG_SELECT_OTHER=1 sg_select "Which runner group?" $(printf '%s\n' "$groups" | tr '\n' ' '))" || return 1 + else + name="$(sg_ask_required "Runner group name")" || return 1 + fi + if [ "$W_SG_DISCOVERY" -eq 1 ] && ! sg_runnergroup_exists "$name"; then + sg_warn "runner group '$name' was not found in org '$ORG' — preflight will fail until it exists" + fi + W_RUNNER_JSON="$("$jqb" -nc --arg n "$name" '{type:"private", names:[$n]}')" + else + W_RUNNER_JSON='{"type":"shared"}' + fi +} + +# --- step 3: policy ------------------------------------------------------------ +wizard_policy() { + local approvers + sg_step "3/4 Workflow defaults" + approvers="$(sg_ask "Approver emails for plans (comma-separated, empty for none)" "$(tfvars_get_json .SGDefaultWfApprovers | "$(sg_resolve jq sg_ensure_jq)" -r 'if . == null then "" else join(", ") end')")" || return 1 + W_APPROVERS_JSON="$(_w_csv_json "$approvers")" + if sg_confirm "Export Terraform state for each workspace?" "$([ "$(_w_default .exportStateFiles true)" = "false" ] && echo N || echo Y)"; then W_EXPORT_STATE=true; else W_EXPORT_STATE=false; fi + if sg_confirm "Pre-configure VCS triggers (push / pull-request runs) from the TFC settings?" "$([ "$(_w_default .SGDefaultEnableVCSTriggers true)" = "false" ] && echo N || echo Y)"; then W_TRIGGERS=true; else W_TRIGGERS=false; fi + sg_dim "StackGuardian bundles managed Terraform only up to 1.5.7 (last MPL/FOSS release);" + sg_dim "workspaces pinned above it are imported with the fallback version below." + while :; do + W_TF_VERSION="$(sg_ask "Fallback Terraform version" "$(_w_default .SGDefaultTerraformVersion TERRAFORM-1.5.7)")" || return 1 + [[ "$W_TF_VERSION" =~ ^TERRAFORM-[0-9]+\.[0-9]+\.[0-9]+$ ]] && break + sg_warn "use the SG format, e.g. TERRAFORM-1.5.7" + done +} + +# --- step 4: review + write ---------------------------------------------------- +wizard_review() { + sg_step "4/4 Review" + row() { printf ' %s%-28s%s %s\n' "$C_BOLD" "$1" "$C_RESET" "$2" >&2; } + row "TFC host / org" "$W_TFHOST / $W_TFORG" + row "Workspaces" "$W_WSNAMES_JSON tags=$W_TAGS_JSON ignore=$W_IGNORE_TAGS_JSON${W_WS_COUNT:+ ($W_WS_COUNT found)}" + row "SG org" "$ORG" + row "VCS connector" "$W_VCS_INTEGRATION ($W_DEST_KIND, $W_REPO_PREFIX)" + row "Cloud connector" "$W_DPC_JSON" + row "Runners" "$W_RUNNER_JSON" + row "Approvers" "$W_APPROVERS_JSON" + row "State export / triggers" "$W_EXPORT_STATE / $W_TRIGGERS" + row "Fallback Terraform" "$W_TF_VERSION${W_WS_ABOVE_CEILING:+ ($W_WS_ABOVE_CEILING workspace(s) above 1.5.7 will use it)}" + [ "${W_DPC_PLACEHOLDER:-0}" -eq 1 ] && sg_warn "cloud connector left as a placeholder — edit SGDefaultDeploymentPlatformConfig in $(sg_rel "$TFVARS") before 'apply'" + sg_confirm "Write $(sg_rel "$TFVARS")?" Y +} + +# wizard_run — the whole flow; returns non-zero when aborted. +wizard_run() { + wizard_tfc && wizard_sg && wizard_policy || { sg_err "init aborted"; return 1; } + wizard_review || { sg_log "nothing written"; return 1; } + if [ -f "$TFVARS" ]; then + cp "$TFVARS" "$TFVARS.bak" + sg_log "previous file kept as $(sg_rel "$TFVARS.bak")" + fi + tfvars_write "$TFVARS" + sg_success "wrote $(sg_rel "$TFVARS")" +} diff --git a/scripts/migrate.sh b/scripts/migrate.sh index 5f7439b..5cb69a7 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -17,6 +17,8 @@ source "$SCRIPT_DIR/lib/tfvars.sh" source "$SCRIPT_DIR/lib/tfc_api.sh" # shellcheck source=lib/sg_api.sh source "$SCRIPT_DIR/lib/sg_api.sh" +# shellcheck source=lib/wizard.sh +source "$SCRIPT_DIR/lib/wizard.sh" # SCRIPT_DIR holds the sibling scripts; SG_REPO_ROOT (from tools.sh) is the repo # root used for all repo-relative paths. @@ -136,13 +138,27 @@ seg_of() { cmd_init() { sg_step "Phase: init" mkdir -p "$SG_REPO_ROOT/.sg" "$SG_CACHE_BIN" - if [ ! -f "$TFVARS" ]; then - cp "$TRANSFORMER_DIR/terraform.tfvars.example" "$TFVARS" - sg_log "created $(sg_rel "$TFVARS") — edit it before 'apply'" + if ! sg_interactive; then + # Non-interactive (CI, no TTY, -y): fall back to the template. + if [ ! -f "$TFVARS" ]; then + cp "$TRANSFORMER_DIR/terraform.tfvars.example" "$TFVARS" + sg_log "created $(sg_rel "$TFVARS") from the template — edit it before 'apply' (run 'init' in a terminal for the guided setup)" + else + sg_log "$(sg_rel "$TFVARS") already exists" + fi else - sg_log "$(sg_rel "$TFVARS") already exists" + if [ -f "$TFVARS" ]; then + case "$(sg_select "$(sg_rel "$TFVARS") already exists" "keep|leave it unchanged" "rerun|run the wizard again (current values become the defaults; a .bak copy is kept)")" in + keep) sg_log "keeping $(sg_rel "$TFVARS")" ;; + rerun) wizard_run || return 1 ;; + esac + else + sg_log "answer a few questions to generate $(sg_rel "$TFVARS"); tokens are read from the environment and never written to disk" + wizard_run || return 1 + fi fi sg_log "workflow groups are created automatically as tfc-; no mapping needed" + sg_log "next: ./sg-migrate.sh all" completion_hint sg_success "init complete" } diff --git a/sg-migrate.sh b/sg-migrate.sh index b43c84e..7d4a1f1 100755 --- a/sg-migrate.sh +++ b/sg-migrate.sh @@ -54,7 +54,7 @@ fi DOCKER_ARGS=(--rm -i -v "$SCRIPT_DIR:/app" -w /app -e SG_API_TOKEN -e SG_ORG -e SG_BASE_URL -e SG_CONCURRENCY -e SG_RETRIES -e SG_TF_PARALLELISM - -e TFE_TOKEN -e SG_PROG) + -e TFE_TOKEN -e SG_PROG -e SG_NONINTERACTIVE -e SG_UI_URL) # Interactive TTY only when attached to one (so the confirmation prompt works, # but CI/non-tty invocations still run — use -y there). From 1e9e264e7ccd6037028aa84ea40e44752313433f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= Date: Mon, 7 Sep 2026 09:21:19 +0200 Subject: [PATCH 15/30] feat: preflight checks before apply and import Verify up front what used to fail minutes later inside terraform or as API 400s: TFC token and org, that the workspace selection matches anything, SG token and org, every connector / secret / runner group referenced in terraform.tfvars (defaults and workspaceOverrides), SGDefaultSourceConfigDestKind and SGDefaultTerraformVersion format, the exportPath vs export-dir mismatch, and override keys that match no workspace. Runs automatically for apply, import and all; also available as 'preflight'; --skip-preflight bypasses it. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01YKBeQrpJgR9DVFtR83myqX --- scripts/lib/preflight.sh | 221 +++++++++++++++++++++++++++++++++++++++ scripts/migrate.sh | 38 +++++-- sg-migrate.sh | 2 +- 3 files changed, 249 insertions(+), 12 deletions(-) create mode 100644 scripts/lib/preflight.sh diff --git a/scripts/lib/preflight.sh b/scripts/lib/preflight.sh new file mode 100644 index 0000000..c03e0cc --- /dev/null +++ b/scripts/lib/preflight.sh @@ -0,0 +1,221 @@ +#!/bin/bash +# Preflight checks (sourced; needs tools.sh, tfvars.sh, tfc_api.sh, sg_api.sh). +# +# Verifies, before the long terraform apply or a bulk import, that the tokens +# work and that everything terraform.tfvars refers to actually exists in TFC +# and StackGuardian. Prints one line per check (✓ ok, ! warning, ✗ failure) and +# fails the run when any check fails. Skipped with --skip-preflight. + +PF_FAIL=0 +PF_WARN=0 +pf_ok() { printf ' %s✓%s %s\n' "$C_GREEN" "$C_RESET" "$*" >&2; } +pf_warn() { printf ' %s!%s %s\n' "$C_YELLOW" "$C_RESET" "$*" >&2; PF_WARN=$((PF_WARN + 1)); } +pf_fail() { printf ' %s✗%s %s\n' "$C_RED$C_BOLD" "$C_RESET" "$*" >&2; PF_FAIL=$((PF_FAIL + 1)); } + +# --- Terraform Cloud ----------------------------------------------------------- +preflight_tfc() { + local host token org body n names_json tags_json ignore_json jqb + jqb="$(sg_resolve jq sg_ensure_jq)" + host="$(tfc_hostname)" + org="$(tfvars_get .tfOrg)" + [ -n "$org" ] || { pf_fail "tfOrg is not set in $(sg_rel "$TFVARS")"; return 0; } + token="$(tfc_token "$host")" + if [ -z "$token" ]; then + pf_fail "no TFC/TFE credentials for $host — set TFE_TOKEN (recommended) or run 'terraform login'" + return 0 + fi + if ! tfc_http "account/details" >/dev/null; then + case "$TFC_HTTP_CODE" in + 401 | 403) pf_fail "TFC rejected $(tfc_token_source) for $host (HTTP $TFC_HTTP_CODE) — set TFE_TOKEN or re-run 'terraform login'" ;; + 000) pf_fail "cannot reach https://$host (network/proxy?)" ;; + *) pf_warn "unexpected HTTP $TFC_HTTP_CODE from $host while verifying credentials" ;; + esac + return 0 + fi + pf_ok "TFC credentials valid ($host, via $(tfc_token_source))" + if tfc_http "organizations/$org" >/dev/null; then + pf_ok "TFC organisation '$org' accessible" + else + case "$TFC_HTTP_CODE" in + 404 | 403) pf_fail "TFC organisation '$org' not found or not accessible with this token (HTTP $TFC_HTTP_CODE) — check tfOrg" ;; + *) pf_warn "could not verify TFC organisation '$org' (HTTP $TFC_HTTP_CODE)" ;; + esac + return 0 + fi + # Workspace selection: mirror the module's filters (names glob, include/exclude tags). + if body="$(tfc_list_workspaces "$org" 2>/dev/null)"; then + names_json="$(tfvars_get_json .workspacenames)" + tags_json="$(tfvars_get_json .tfWorkspaceTags)" + ignore_json="$(tfvars_get_json .tfWorkspaceIgnoreTags)" + n="$(printf '%s' "$body" | "$jqb" --argjson names "${names_json:-null}" --argjson tags "${tags_json:-null}" --argjson ignore "${ignore_json:-null}" ' + def glob($p): ("^" + ($p | gsub("\\*"; ".*")) + "$"); + [ .[] + | select(($names == null) or ($names == ["*"]) or ([.name] | inside([]) | not) and ([$names[] as $p | (.name | test(glob($p)))] | any)) + | select(($tags == null) or (($tags | length) == 0) or ([.tags[]?] | inside($tags) | not) or (([.tags[]?] | map(select(. as $t | $tags | index($t) != null)) | length) > 0)) + | select(($ignore == null) or (($ignore | length) == 0) or (([.tags[]?] | map(select(. as $t | $ignore | index($t) != null)) | length) == 0)) + ] | length')" + if [ "$n" -gt 0 ]; then + pf_ok "$n workspace(s) match the selection (of $(printf '%s' "$body" | "$jqb" 'length') in the org)" + else + pf_warn "no workspace matches workspacenames/tfWorkspaceTags/tfWorkspaceIgnoreTags — apply would export nothing" + fi + PF_TFC_WORKSPACES="$body" + else + pf_warn "could not list workspaces for '$org' (HTTP $TFC_HTTP_CODE)" + fi +} + +# --- StackGuardian ------------------------------------------------------------ +# preflight_sg +preflight_sg() { + local required="$1" ints names id n jqb rg + jqb="$(sg_resolve jq sg_ensure_jq)" + if [ -z "${SG_API_TOKEN:-}" ] || [ -z "$ORG" ]; then + if [ "$required" -eq 1 ]; then + [ -n "${SG_API_TOKEN:-}" ] || pf_fail "SG_API_TOKEN is not set" + [ -n "$ORG" ] || pf_fail "StackGuardian org not set (use --org or SG_ORG)" + else + pf_warn "SG_API_TOKEN/SG_ORG not set — StackGuardian references are not verified (they will be at import)" + fi + return 0 + fi + if ! ints="$(sg_list_integrations 2>/dev/null)"; then + case "$SG_HTTP_CODE" in + 401 | 403) pf_fail "StackGuardian rejected SG_API_TOKEN for org '$ORG' (HTTP $SG_HTTP_CODE)" ;; + 404) pf_fail "StackGuardian org '$ORG' not found at $SG_BASE_URL (HTTP 404) — check --org / SG_ORG" ;; + 000) pf_fail "cannot reach $SG_BASE_URL (network/proxy? SG_BASE_URL?)" ;; + *) pf_fail "could not list StackGuardian connectors (HTTP $SG_HTTP_CODE)" ;; + esac + return 0 + fi + pf_ok "StackGuardian credentials valid (org '$ORG', $SG_BASE_URL)" + + # Every integration id referenced anywhere in tfvars must exist. + names="$(printf '%s' "$ints" | "$jqb" -c '[.[].name]')" + while IFS= read -r id; do + [ -n "$id" ] || continue + case "$id" in + *CHANGE_ME* | *INTEGRATION_ID*) pf_fail "placeholder connector id '$id' in $(sg_rel "$TFVARS") — run '$PROG init' or edit the file" ;; + /integrations/*) + if printf '%s' "$names" | "$jqb" -e --arg n "${id#/integrations/}" 'index($n) != null' >/dev/null; then + pf_ok "connector $id exists" + else + pf_fail "connector $id not found in org '$ORG' (available: $(printf '%s' "$names" | "$jqb" -r 'join(", ")'))" + fi + ;; + /secrets/*) + if sg_secret_exists "${id#/secrets/}"; then pf_ok "secret $id exists"; else pf_fail "secret $id not found in org '$ORG'"; fi + ;; + *) pf_warn "unrecognised integration id '$id' (expected /integrations/ or /secrets/)" ;; + esac + done < <(tfvars_json | "$jqb" -r ' + [ .SGDefaultVCSAuthIntegrationID, + (.SGDefaultDeploymentPlatformConfig // [])[]?.config.integrationId, + ((.workspaceOverrides // {}) | to_entries[]? | .value | (.vcsAuthIntegrationID, ((.DeploymentPlatformConfig // [])[]?.config.integrationId))) ] + | map(select(. != null and . != "")) | unique | .[]') + + # Every private runner group must exist. + while IFS= read -r rg; do + [ -n "$rg" ] || continue + if sg_runnergroup_exists "$rg"; then pf_ok "runner group '$rg' exists"; else pf_fail "runner group '$rg' not found in org '$ORG' — check SGDefaultRunnerConstraints / workspaceOverrides"; fi + done < <(tfvars_json | "$jqb" -r ' + [ (.SGDefaultRunnerConstraints // {} | select(.type == "private") | .names[]?), + ((.workspaceOverrides // {}) | to_entries[]? | .value.RunnerConstraints // {} | select(.type == "private") | .names[]?) ] + | unique | .[]') + n="$(tfvars_json | "$jqb" -r '(.SGDefaultRunnerConstraints // {}).type // "shared"')" + [ "$n" = "shared" ] && pf_ok "runners: StackGuardian shared runners" +} + +# --- static config -------------------------------------------------------------- +preflight_config() { + local v ceiling="1.5.7" export_path want jqb + jqb="$(sg_resolve jq sg_ensure_jq)" + v="$(tfvars_get .SGDefaultSourceConfigDestKind GIT_OTHER)" + case "$v" in + GITHUB_COM | GITHUB_APP_CUSTOM | GIT_OTHER | INLINE | BITBUCKET_ORG | GITLAB_COM | AZURE_DEVOPS) pf_ok "SGDefaultSourceConfigDestKind = $v" ;; + *) pf_fail "SGDefaultSourceConfigDestKind '$v' is not one of GITHUB_COM, GITLAB_COM, BITBUCKET_ORG, AZURE_DEVOPS, GIT_OTHER" ;; + esac + v="$(tfvars_get .SGDefaultTerraformVersion TERRAFORM-1.5.7)" + if [[ "$v" =~ ^TERRAFORM-([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then + if [ "$(printf '%03d%03d%03d' "${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}" "${BASH_REMATCH[3]}")" -gt "001005007" ]; then + pf_warn "SGDefaultTerraformVersion $v is above SG's managed ceiling ($ceiling); fallbacks would fail too unless a private runner ships that binary" + else + pf_ok "SGDefaultTerraformVersion = $v" + fi + else + pf_fail "SGDefaultTerraformVersion '$v' must look like TERRAFORM-1.5.7" + fi + export_path="$(tfvars_get .exportPath export)" + case "$export_path" in /*) want="$export_path" ;; *) want="$SG_REPO_ROOT/$export_path" ;; esac + if [ "$(cd "$(dirname "$want")" 2>/dev/null && pwd)/$(basename "$want")" = "$(cd "$(dirname "$EXPORT_DIR")" 2>/dev/null && pwd)/$(basename "$EXPORT_DIR")" ]; then + pf_ok "export directory: $(sg_rel "$EXPORT_DIR")" + else + pf_warn "exportPath in tfvars ($export_path) differs from the orchestrator's export dir ($(sg_rel "$EXPORT_DIR")) — payloads would be written where later phases don't look" + fi + if [ -n "${PF_TFC_WORKSPACES:-}" ]; then + while IFS= read -r v; do + [ -n "$v" ] || continue + if printf '%s' "$PF_TFC_WORKSPACES" | "$jqb" -e --arg n "$v" '[.[].name] | index($n) != null' >/dev/null; then + pf_ok "workspaceOverrides['$v'] matches a workspace" + else + pf_warn "workspaceOverrides['$v'] does not match any workspace in the org (typo?)" + fi + done < <(tfvars_json | "$jqb" -r '(.workspaceOverrides // {}) | keys[]') + fi +} + +# --- import inputs -------------------------------------------------------------- +preflight_import_inputs() { + payload_files + if [ "${#PF[@]}" -gt 0 ]; then + pf_ok "${#PF[@]} payload file(s) in $(sg_rel "$EXPORT_DIR")" + else + pf_fail "no payload files in $(sg_rel "$EXPORT_DIR") — run '$PROG apply' first" + fi + if sg_resolve sg-cli sg_ensure_sgcli >/dev/null 2>&1; then pf_ok "sg-cli available"; else pf_fail "sg-cli not found and could not be downloaded"; fi +} + +# preflight_run — run the checks for a context; dies on +# failures. Runs at most once per process (PREFLIGHT_DONE). +preflight_run() { + local ctx="$1" + [ "${SKIP_PREFLIGHT:-0}" -eq 1 ] && { sg_warn "preflight skipped (--skip-preflight)"; return 0; } + [ "${PREFLIGHT_DONE:-0}" -eq 1 ] && return 0 + sg_step "Preflight ($ctx)" + [ -f "$TFVARS" ] || die "Missing $(sg_rel "$TFVARS"). Run: $PROG init" + PF_FAIL=0 + PF_WARN=0 + PF_TFC_WORKSPACES="" + case "$ctx" in + apply) + preflight_tfc + preflight_sg 0 + preflight_config + ;; + import) + preflight_sg 1 + preflight_config + preflight_import_inputs + ;; + all) + preflight_tfc + preflight_sg 1 + preflight_config + ;; + *) die "preflight: unknown context '$ctx'" ;; + esac + PREFLIGHT_DONE=1 + if [ "$PF_FAIL" -gt 0 ]; then + die "preflight found $PF_FAIL problem(s); fix them (or re-run '$PROG init') and try again. Use --skip-preflight to bypass." + fi + if [ "$PF_WARN" -gt 0 ]; then + sg_warn "preflight passed with $PF_WARN warning(s)" + else + sg_success "preflight passed" + fi +} + +cmd_preflight() { + PREFLIGHT_DONE=0 + preflight_run all +} diff --git a/scripts/migrate.sh b/scripts/migrate.sh index 5cb69a7..3c90ef1 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -19,6 +19,8 @@ source "$SCRIPT_DIR/lib/tfc_api.sh" source "$SCRIPT_DIR/lib/sg_api.sh" # shellcheck source=lib/wizard.sh source "$SCRIPT_DIR/lib/wizard.sh" +# shellcheck source=lib/preflight.sh +source "$SCRIPT_DIR/lib/preflight.sh" # SCRIPT_DIR holds the sibling scripts; SG_REPO_ROOT (from tools.sh) is the repo # root used for all repo-relative paths. @@ -34,6 +36,10 @@ PURGE=0 CREATE_GROUPS=1 ENRICH_VARSETS=1 VCS_TRIGGERS=1 +# shellcheck disable=SC2034 # consumed by lib/preflight.sh +SKIP_PREFLIGHT=0 +# shellcheck disable=SC2034 +PREFLIGHT_DONE=0 VERBOSE="${SG_VERBOSE:-0}" CONC="${SG_CONCURRENCY:-4}" TF_PARALLELISM="${SG_TF_PARALLELISM:-20}" @@ -46,7 +52,9 @@ usage() { Usage: $PROG [options] Commands: - init Create terraform.tfvars from the template + init Guided setup: discovers TFC/SG resources and writes terraform.tfvars + preflight Verify tokens and every connector/runner/org referenced in tfvars + (runs automatically before apply, import and all) apply Run the transformer (terraform apply) to generate payloads + state enrich Merge TFC Variable Set variables into the payloads (via the TFC API) convert Convert HCL-string variables to JSON in each payload (parallel) @@ -72,6 +80,7 @@ Options: --no-create-groups Do not create missing workflow groups; require them to exist --no-variable-sets Skip merging TFC Variable Set variables in the 'all' flow --no-vcs-triggers Skip registering VCS triggers after import + --skip-preflight Skip the preflight checks (not recommended) --all With 'clean': also remove config (terraform.tfvars, mapping, .sg) -v, --verbose Show full terraform/tool output (default: concise) -y, --yes Skip the import confirmation prompt @@ -268,8 +277,8 @@ cmd_clean() { cmd_apply() { sg_step "Phase: apply (terraform)" command -v terraform >/dev/null 2>&1 || die "terraform not found on PATH" - [ -f "$TFVARS" ] || die "Missing $(sg_rel "$TFVARS"). Run: $PROG init (then edit it)." - require_tfc_auth + [ -f "$TFVARS" ] || die "Missing $(sg_rel "$TFVARS"). Run: $PROG init" + preflight_run apply # State export (TFC API) calls curl + jq from terraform's local-exec; make sure # both are on PATH for the apply (jq from cache if not already installed). command -v curl >/dev/null 2>&1 || die "curl is required for state export" @@ -445,6 +454,7 @@ cmd_import() { # so the sg-cli child process inherits the same target. export SG_API_TOKEN SG_BASE_URL JQ_BIN="$(sg_resolve jq sg_ensure_jq)" + preflight_run import payload_files [ "${#PF[@]}" -gt 0 ] || die "No payload files in $(sg_rel "$EXPORT_DIR")." @@ -530,8 +540,8 @@ cmd_import() { # Single source of truth for shell completion (keep in sync with the parser below # and the host-only flags in sg-migrate.sh). -SG_COMMANDS="init apply enrich convert validate import triggers all clean completion" -SG_OPTIONS="--org --export-dir --mapping --concurrency --no-create-groups --no-variable-sets --no-vcs-triggers --all -v --verbose -y --yes -h --help --native --local --build" +SG_COMMANDS="init preflight apply enrich convert validate import triggers all clean completion" +SG_OPTIONS="--org --export-dir --mapping --concurrency --no-create-groups --no-variable-sets --no-vcs-triggers --skip-preflight --all -v --verbose -y --yes -h --help --native --local --build" # cmd_completion — print a completion script for sg-migrate.sh / # migrate.sh to stdout. Both shells fall back to the basename when the command @@ -572,7 +582,8 @@ BASH _sg_migrate() { local -a cmds cmds=( - 'init:Create terraform.tfvars from the template' + 'init:Guided setup — generates terraform.tfvars' + 'preflight:Verify tokens and every id in terraform.tfvars before running' 'apply:Run the transformer (terraform apply)' 'enrich:Merge TFC Variable Set variables into the payloads' 'convert:Convert HCL-string variables to JSON' @@ -591,6 +602,7 @@ _sg_migrate() { '--no-create-groups[Require workflow groups to pre-exist]' \\ '--no-variable-sets[Skip merging TFC Variable Sets]' \\ '--no-vcs-triggers[Skip registering VCS triggers after import]' \\ + '--skip-preflight[Skip the preflight checks]' \\ '--all[With clean: also remove config]' \\ '(-v --verbose)'{-v,--verbose}'[Show full terraform/tool output]' \\ '(-y --yes)'{-y,--yes}'[Skip the import confirmation prompt]' \\ @@ -638,13 +650,14 @@ while [ $# -gt 0 ]; do --no-create-groups) CREATE_GROUPS=0 ;; --no-variable-sets) ENRICH_VARSETS=0 ;; --no-vcs-triggers) VCS_TRIGGERS=0 ;; + --skip-preflight) export SKIP_PREFLIGHT=1 ;; -v | --verbose) VERBOSE=1 ;; --all) PURGE=1 ;; -h | --help) usage exit 0 ;; - init | apply | enrich | convert | validate | import | triggers | all | clean) CMD="$1" ;; + init | apply | enrich | convert | validate | import | triggers | all | clean | preflight) CMD="$1" ;; completion) cmd_completion "${2:-}" exit 0 @@ -673,15 +686,18 @@ convert) cmd_convert ;; validate) cmd_validate ;; import) cmd_import ;; triggers) cmd_triggers ;; +preflight) + export SG_API_TOKEN SG_BASE_URL + cmd_preflight + ;; all) if [ ! -f "$TFVARS" ]; then cmd_init die "Edit $(sg_rel "$TFVARS"), then re-run '$PROG all'." fi - # Fail fast on prerequisites before the (long) apply. - require_tfc_auth - [ -n "${SG_API_TOKEN:-}" ] || die "SG_API_TOKEN is not set (needed for import)." - [ -n "$ORG" ] || die "StackGuardian org not set (use --org or SG_ORG)." + # Fail fast on everything the whole pipeline needs, before the (long) apply. + export SG_API_TOKEN SG_BASE_URL + preflight_run all cmd_apply if [ "$ENRICH_VARSETS" -eq 1 ]; then cmd_enrich; fi cmd_convert diff --git a/sg-migrate.sh b/sg-migrate.sh index 7d4a1f1..5a56ba4 100755 --- a/sg-migrate.sh +++ b/sg-migrate.sh @@ -33,7 +33,7 @@ HAS_CMD=0 for a in ${ARGS[@]+"${ARGS[@]}"}; do case "$a" in clean | completion | -h | --help) NATIVE=1; HAS_CMD=1 ;; - init | apply | enrich | convert | validate | import | triggers | all) HAS_CMD=1 ;; + init | preflight | apply | enrich | convert | validate | import | triggers | all) HAS_CMD=1 ;; esac done [ "$HAS_CMD" -eq 1 ] || NATIVE=1 From 54e18764fb5a325d68c0bf886af9e13ea438528c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= Date: Mon, 7 Sep 2026 09:24:32 +0200 Subject: [PATCH 16/30] feat: resumable runs and project/workspace filters 'all' now records each phase's inputs in .sg/state.json and skips phases whose inputs have not changed, so a failed run resumes instead of re-running terraform apply. Import records per payload file which workflows landed and which failed; unchanged, fully imported files are skipped and files with failures are re-imported (sg-cli updates existing workflows in place). --fresh discards the state. --project and --workspace restrict every phase to a subset; apply maps --workspace to the workspacenames variable. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01YKBeQrpJgR9DVFtR83myqX --- scripts/lib/preflight.sh | 6 +- scripts/lib/state.sh | 65 +++++++++++++++++ scripts/migrate.sh | 146 +++++++++++++++++++++++++++++++++++---- 3 files changed, 203 insertions(+), 14 deletions(-) create mode 100644 scripts/lib/state.sh diff --git a/scripts/lib/preflight.sh b/scripts/lib/preflight.sh index c03e0cc..c0bb2ad 100644 --- a/scripts/lib/preflight.sh +++ b/scripts/lib/preflight.sh @@ -63,6 +63,7 @@ preflight_tfc() { else pf_warn "could not list workspaces for '$org' (HTTP $TFC_HTTP_CODE)" fi + return 0 } # --- StackGuardian ------------------------------------------------------------ @@ -123,7 +124,8 @@ preflight_sg() { ((.workspaceOverrides // {}) | to_entries[]? | .value.RunnerConstraints // {} | select(.type == "private") | .names[]?) ] | unique | .[]') n="$(tfvars_json | "$jqb" -r '(.SGDefaultRunnerConstraints // {}).type // "shared"')" - [ "$n" = "shared" ] && pf_ok "runners: StackGuardian shared runners" + if [ "$n" = "shared" ]; then pf_ok "runners: StackGuardian shared runners"; fi + return 0 } # --- static config -------------------------------------------------------------- @@ -162,6 +164,7 @@ preflight_config() { fi done < <(tfvars_json | "$jqb" -r '(.workspaceOverrides // {}) | keys[]') fi + return 0 } # --- import inputs -------------------------------------------------------------- @@ -173,6 +176,7 @@ preflight_import_inputs() { pf_fail "no payload files in $(sg_rel "$EXPORT_DIR") — run '$PROG apply' first" fi if sg_resolve sg-cli sg_ensure_sgcli >/dev/null 2>&1; then pf_ok "sg-cli available"; else pf_fail "sg-cli not found and could not be downloaded"; fi + return 0 } # preflight_run — run the checks for a context; dies on diff --git a/scripts/lib/state.sh b/scripts/lib/state.sh new file mode 100644 index 0000000..c69040d --- /dev/null +++ b/scripts/lib/state.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# Run state for resumable 'all' runs (sourced; needs tools.sh). +# +# .sg/state.json records, per phase, a hash of the inputs it last ran with and +# when. 'all' skips a phase whose inputs are unchanged; --fresh wipes the file. +# Import results are recorded per payload file (hash, imported/failed +# workflows) so a re-run only touches files that changed or had failures, and +# the post-import checklist knows what actually landed. +# +# Layout: { "phases": { "": {"at": iso, "input_sha": sha} }, +# "import": { "": {"at": iso, "payload_sha": sha, "group": g, +# "imported": [...], "failed": [...]} } } + +STATE_FILE="${SG_STATE_FILE:-$SG_REPO_ROOT/.sg/state.json}" + +# sg_sha — short sha256 of a string. +sg_sha() { + if command -v shasum >/dev/null 2>&1; then printf '%s' "$1" | shasum -a 256 | cut -c1-16 + else printf '%s' "$1" | sha256sum | cut -c1-16; fi +} + +# sg_sha_files ... — sha256 over the contents of the given files (sorted). +sg_sha_files() { + local f + { + for f in "$@"; do [ -f "$f" ] && cat "$f"; done + } | if command -v shasum >/dev/null 2>&1; then shasum -a 256; else sha256sum; fi | cut -c1-16 +} + +state_read() { [ -f "$STATE_FILE" ] && cat "$STATE_FILE" || echo '{}'; } + +# state_update [jq args...] — apply a filter to the state document. +state_update() { + local filter="$1" tmp + shift + mkdir -p "$(dirname "$STATE_FILE")" + tmp="$(mktemp)" + state_read | "$(sg_resolve jq sg_ensure_jq)" "$@" "$filter" >"$tmp" && mv -f "$tmp" "$STATE_FILE" +} + +state_now() { date -u +%Y-%m-%dT%H:%M:%SZ; } + +# state_phase_done — exit 0 when the phase last completed +# with exactly these inputs. Prints the completion time on stdout. +state_phase_done() { + local at + at="$(state_read | "$(sg_resolve jq sg_ensure_jq)" -r --arg p "$1" --arg s "$2" '.phases[$p] | select(.input_sha == $s) | .at // empty')" + [ -n "$at" ] && printf '%s' "$at" +} + +# state_mark_phase +state_mark_phase() { state_update '.phases[$p] = {at: $at, input_sha: $s}' --arg p "$1" --arg s "$2" --arg at "$(state_now)"; } + +# state_import_done — exit 0 when this payload file was +# fully imported (no failures) with exactly this content. +state_import_done() { + state_read | "$(sg_resolve jq sg_ensure_jq)" -e --arg s "$1" --arg sha "$2" \ + '.import[$s] | select(.payload_sha == $sha and ((.failed // []) | length) == 0)' >/dev/null 2>&1 +} + +# state_record_import — merge a do_import result +# ({group, payload_sha, imported, failed}) for a payload file. +state_record_import() { state_update '.import[$s] = ($r + {at: $at})' --arg s "$1" --argjson r "$2" --arg at "$(state_now)"; } + +state_reset() { rm -f "$STATE_FILE"; } diff --git a/scripts/migrate.sh b/scripts/migrate.sh index 3c90ef1..dd78765 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -21,6 +21,8 @@ source "$SCRIPT_DIR/lib/sg_api.sh" source "$SCRIPT_DIR/lib/wizard.sh" # shellcheck source=lib/preflight.sh source "$SCRIPT_DIR/lib/preflight.sh" +# shellcheck source=lib/state.sh +source "$SCRIPT_DIR/lib/state.sh" # SCRIPT_DIR holds the sibling scripts; SG_REPO_ROOT (from tools.sh) is the repo # root used for all repo-relative paths. @@ -40,6 +42,9 @@ VCS_TRIGGERS=1 SKIP_PREFLIGHT=0 # shellcheck disable=SC2034 PREFLIGHT_DONE=0 +FRESH=0 +PROJECT_FILTER=() +WS_FILTER=() VERBOSE="${SG_VERBOSE:-0}" CONC="${SG_CONCURRENCY:-4}" TF_PARALLELISM="${SG_TF_PARALLELISM:-20}" @@ -81,6 +86,9 @@ Options: --no-variable-sets Skip merging TFC Variable Set variables in the 'all' flow --no-vcs-triggers Skip registering VCS triggers after import --skip-preflight Skip the preflight checks (not recommended) + --fresh Ignore the saved run state: redo every phase and re-import everything + --project SEG Only handle this TFC project (repeatable; matches sg-payload..json) + --workspace NAME Only handle this workspace (repeatable; apply exports only it) --all With 'clean': also remove config (terraform.tfvars, mapping, .sg) -v, --verbose Show full terraform/tool output (default: concise) -y, --yes Skip the import confirmation prompt @@ -130,11 +138,54 @@ run_parallel() { return "$rc" } +# payload_sha — hash of the current payload files (the input of enrich/convert/validate). +payload_sha() { + payload_files + sg_sha_files ${PF[@]+"${PF[@]}"} +} + +# run_phase — in 'all', skip a phase that already ran +# with identical inputs; otherwise run it and record the inputs it ran with. +# Phases that rewrite the payloads (enrich, convert) record the post-run hash, +# so an unchanged export is recognised on the next run. +run_phase() { + local name="$1" sha="$2" fn="$3" at + if at="$(state_phase_done "$name" "$sha")" && [ -n "$at" ]; then + sg_log "skipping $name — unchanged since $at (--fresh to redo)" + return 0 + fi + "$fn" || return $? + case "$name" in + apply) state_mark_phase "$name" "$sha" ;; + *) state_mark_phase "$name" "$(payload_sha)" ;; + esac +} + # Populate PF with the generated payload files. payload_files() { + local f seg keep shopt -s nullglob PF=("$EXPORT_DIR"/sg-payload.*.json) shopt -u nullglob + if [ "${#PROJECT_FILTER[@]}" -gt 0 ]; then + keep=() + for f in "${PF[@]}"; do + seg="$(seg_of "$f")" + case " ${PROJECT_FILTER[*]} " in *" $seg "*) keep+=("$f") ;; esac + done + PF=(${keep[@]+"${keep[@]}"}) + fi +} + +# ws_filter_json — the --workspace names as a JSON array (empty array = no filter). +ws_filter_json() { + if [ "${#WS_FILTER[@]}" -eq 0 ]; then echo '[]'; else names_json "${WS_FILTER[@]}"; fi +} + +# ws_selected — exit 0 when no --workspace filter is set or it lists . +ws_selected() { + [ "${#WS_FILTER[@]}" -eq 0 ] && return 0 + case " ${WS_FILTER[*]} " in *" $1 "*) return 0 ;; *) return 1 ;; esac } seg_of() { @@ -210,6 +261,7 @@ do_set_triggers() { continue fi wf="$("$JQ_BIN" -r --argjson i "$i" '.[$i].ResourceName' "$f")" + ws_selected "$wf" || continue # A workflow that failed to import has nothing to attach triggers to. if ! sg_workflow_exists "$grp" "$wf"; then sg_warn " $grp/$wf does not exist in SG (import failed?) — skipping triggers" @@ -266,6 +318,7 @@ cmd_clean() { rm -rf "$TRANSFORMER_DIR/.terraform" "$TRANSFORMER_DIR/.terraform.lock.hcl" \ "$TRANSFORMER_DIR/terraform.tfstate" "$TRANSFORMER_DIR/terraform.tfstate.backup" rm -rf "$SG_CACHE_DIR" + state_reset if [ "$PURGE" -eq 1 ]; then rm -f "$TFVARS" "$MAPPING" rm -rf "$SG_REPO_ROOT/.sg" @@ -283,12 +336,18 @@ cmd_apply() { # both are on PATH for the apply (jq from cache if not already installed). command -v curl >/dev/null 2>&1 || die "curl is required for state export" local jqdir tflog rc=0 + local -a tfvar_args=() jqdir="$(dirname "$(sg_resolve jq sg_ensure_jq)")" + if [ "${#WS_FILTER[@]}" -gt 0 ]; then + JQ_BIN="${JQ_BIN:-$(sg_resolve jq sg_ensure_jq)}" + tfvar_args=(-var "workspacenames=$(ws_filter_json)") + sg_log "limiting apply to workspace(s): ${WS_FILTER[*]}" + fi if [ "$VERBOSE" -eq 1 ]; then (cd "$TRANSFORMER_DIR" && export PATH="$jqdir:$PATH" TF_IN_AUTOMATION=1 && terraform init -input=false && - terraform apply -auto-approve -compact-warnings -parallelism="$TF_PARALLELISM" -var-file=terraform.tfvars) || rc=$? + terraform apply -auto-approve -compact-warnings -parallelism="$TF_PARALLELISM" -var-file=terraform.tfvars "${tfvar_args[@]}") || rc=$? else # Quiet: capture terraform's verbose plan/output; surface only progress, the # final summary, and (on failure) the captured log. @@ -298,7 +357,7 @@ cmd_apply() { if [ "$rc" -eq 0 ]; then sg_log "reading workspaces, generating payloads, exporting state..." (cd "$TRANSFORMER_DIR" && export PATH="$jqdir:$PATH" TF_IN_AUTOMATION=1 && - terraform apply -auto-approve -compact-warnings -no-color -parallelism="$TF_PARALLELISM" -var-file=terraform.tfvars) >"$tflog" 2>&1 || rc=$? + terraform apply -auto-approve -compact-warnings -no-color -parallelism="$TF_PARALLELISM" -var-file=terraform.tfvars "${tfvar_args[@]}") >"$tflog" 2>&1 || rc=$? fi if [ "$rc" -ne 0 ]; then sg_err "terraform failed (rc=$rc):" @@ -374,12 +433,24 @@ names_json() { printf '%s\n' "$@" | "$JQ_BIN" -R . | "$JQ_BIN" -s .; } # the trigger pass see what was actually imported); each fallback is appended to # terraform-version-fallbacks.log. Any other per-workflow failure fails the file. do_import() { - local f="$1" seg grp out rc=0 ceiling="" failed=() fb=() name line tmp names + local f="$1" seg grp out rc=0 ceiling="" failed=() fb=() name line tmp names work all_names seg="$(seg_of "$f")" grp="$(group_for "$seg")" + # With --workspace, import only the selected workflows (a filtered copy). + work="$f" + if [ "${#WS_FILTER[@]}" -gt 0 ]; then + work="$(mktemp "$EXPORT_DIR/.subset.$seg.XXXXXX")" + "$JQ_BIN" --argjson names "$(ws_filter_json)" 'map(select(.ResourceName as $n | $names | index($n) != null))' "$f" >"$work" + if [ "$("$JQ_BIN" 'length' "$work")" -eq 0 ]; then + sg_log "$(basename "$f"): no selected workflows — skipped" + rm -f "$work" + return 0 + fi + fi sg_log "importing $(basename "$f") -> $grp" out="$(mktemp)" - sg_retry "$RETRIES" "$RETRY_BASE" -- sgcli_bulk "$grp" "$f" "$out" || rc=1 + sg_retry "$RETRIES" "$RETRY_BASE" -- sgcli_bulk "$grp" "$work" "$out" || rc=1 + all_names="$("$JQ_BIN" -c '[.[].ResourceName]' "$work")" while IFS= read -r line; do if [[ "$line" =~ $TF_CEILING_RE ]]; then @@ -419,6 +490,16 @@ do_import() { rm -f "$tmp" fi + [ "$work" != "$f" ] && rm -f "$work" + + # Per-file result for the state file and the post-import checklist + # (run_parallel jobs run in subshells, so the caller merges these). + "$JQ_BIN" -nc --arg g "$grp" --arg sha "$(sg_sha_files "$f")" --argjson all "$all_names" \ + --argjson failed "$([ "${#failed[@]}" -gt 0 ] && names_json "${failed[@]}" || echo '[]')" \ + --argjson fallback "$([ "${#fb[@]}" -gt 0 ] && names_json "${fb[@]}" || echo '[]')" \ + '{group: $g, payload_sha: $sha, imported: ($all - $failed), failed: $failed, tf_fallback: ($fallback - $failed)}' \ + >"$EXPORT_DIR/.import-result.$seg.json" + if [ "${#failed[@]}" -gt 0 ]; then sg_err "$(basename "$f"): ${#failed[@]} workflow(s) failed to import: ${failed[*]}" return 1 @@ -520,12 +601,35 @@ cmd_import() { SG_DEFAULT_TF_VERSION="${SG_DEFAULT_TF_VERSION:-$(tfvars_get '.SGDefaultTerraformVersion' TERRAFORM-1.5.7)}" rm -f "$EXPORT_DIR/terraform-version-fallbacks.log" - sg_log "importing ${#PF[@]} payload(s), up to $CONC in parallel (retries: $RETRIES)" + # Resume: skip payload files already imported in full with identical content + # (a changed payload or a previous failure re-imports the whole file; + # sg-cli updates existing workflows in place). + local -a todo=() + local skipped=0 seg + for f in "${PF[@]}"; do + seg="$(seg_of "$f")" + if [ "$FRESH" -eq 0 ] && [ "${#WS_FILTER[@]}" -eq 0 ] && state_import_done "$seg" "$(sg_sha_files "$f")"; then + skipped=$((skipped + 1)) + continue + fi + todo+=("$f") + done + [ "$skipped" -gt 0 ] && sg_log "skipping $skipped payload(s) already imported and unchanged (use --fresh to re-import)" local import_rc=0 - run_parallel do_import "$CONC" "${PF[@]}" || import_rc=1 + if [ "${#todo[@]}" -gt 0 ]; then + sg_log "importing ${#todo[@]} payload(s), up to $CONC in parallel (retries: $RETRIES)" + run_parallel do_import "$CONC" "${todo[@]}" || import_rc=1 + for f in "${todo[@]}"; do + seg="$(seg_of "$f")" + if [ -f "$EXPORT_DIR/.import-result.$seg.json" ]; then + state_record_import "$seg" "$(cat "$EXPORT_DIR/.import-result.$seg.json")" + rm -f "$EXPORT_DIR/.import-result.$seg.json" + fi + done + fi tf_fallback_notice if [ "$import_rc" -eq 0 ]; then - sg_success "import complete (${#PF[@]} payload(s))" + sg_success "import complete (${#todo[@]} payload(s) imported, $skipped skipped)" else sg_err "one or more workflows failed to import (see above); VCS triggers are still registered for the ones that succeeded" fi @@ -541,7 +645,7 @@ cmd_import() { # Single source of truth for shell completion (keep in sync with the parser below # and the host-only flags in sg-migrate.sh). SG_COMMANDS="init preflight apply enrich convert validate import triggers all clean completion" -SG_OPTIONS="--org --export-dir --mapping --concurrency --no-create-groups --no-variable-sets --no-vcs-triggers --skip-preflight --all -v --verbose -y --yes -h --help --native --local --build" +SG_OPTIONS="--org --export-dir --mapping --concurrency --no-create-groups --no-variable-sets --no-vcs-triggers --skip-preflight --fresh --project --workspace --all -v --verbose -y --yes -h --help --native --local --build" # cmd_completion — print a completion script for sg-migrate.sh / # migrate.sh to stdout. Both shells fall back to the basename when the command @@ -560,7 +664,7 @@ _sg_migrate() { case "\$prev" in --export-dir) COMPREPLY=(\$(compgen -d -- "\$cur")); return ;; --mapping) COMPREPLY=(\$(compgen -f -- "\$cur")); return ;; - --org | --concurrency) COMPREPLY=(); return ;; + --org | --concurrency | --project | --workspace) COMPREPLY=(); return ;; completion) COMPREPLY=(\$(compgen -W "bash zsh" -- "\$cur")); return ;; esac for w in "\${COMP_WORDS[@]:1:COMP_CWORD-1}"; do @@ -603,6 +707,9 @@ _sg_migrate() { '--no-variable-sets[Skip merging TFC Variable Sets]' \\ '--no-vcs-triggers[Skip registering VCS triggers after import]' \\ '--skip-preflight[Skip the preflight checks]' \\ + '--fresh[Ignore saved run state: redo every phase]' \\ + '*--project[Only this TFC project segment]:segment' \\ + '*--workspace[Only this workspace]:name' \\ '--all[With clean: also remove config]' \\ '(-v --verbose)'{-v,--verbose}'[Show full terraform/tool output]' \\ '(-y --yes)'{-y,--yes}'[Skip the import confirmation prompt]' \\ @@ -651,6 +758,17 @@ while [ $# -gt 0 ]; do --no-variable-sets) ENRICH_VARSETS=0 ;; --no-vcs-triggers) VCS_TRIGGERS=0 ;; --skip-preflight) export SKIP_PREFLIGHT=1 ;; + --fresh) FRESH=1 ;; + --project) + PROJECT_FILTER+=("$2") + shift + ;; + --project=*) PROJECT_FILTER+=("${1#*=}") ;; + --workspace) + WS_FILTER+=("$2") + shift + ;; + --workspace=*) WS_FILTER+=("${1#*=}") ;; -v | --verbose) VERBOSE=1 ;; --all) PURGE=1 ;; -h | --help) @@ -698,10 +816,12 @@ all) # Fail fast on everything the whole pipeline needs, before the (long) apply. export SG_API_TOKEN SG_BASE_URL preflight_run all - cmd_apply - if [ "$ENRICH_VARSETS" -eq 1 ]; then cmd_enrich; fi - cmd_convert - cmd_validate + [ "$FRESH" -eq 1 ] && { state_reset; sg_log "--fresh: previous run state discarded"; } + JQ_BIN="$(sg_resolve jq sg_ensure_jq)" + run_phase apply "$(sg_sha "$(sg_sha_files "$TFVARS")|$(ws_filter_json)")" cmd_apply + if [ "$ENRICH_VARSETS" -eq 1 ]; then run_phase enrich "$(payload_sha)" cmd_enrich; fi + run_phase convert "$(payload_sha)" cmd_convert + run_phase validate "$(payload_sha)" cmd_validate cmd_import ;; esac From 921e369e4b97ec45063d380cfc4a483d06a09e99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= Date: Mon, 7 Sep 2026 09:26:39 +0200 Subject: [PATCH 17/30] feat: show the migration summary after apply and an import plan apply now prints a condensed migration summary (workflows per project, sensitive variables skipped, unpinned Terraform versions, non-remote execution modes, renames, failed state exports) instead of leaving it in a file nobody opens. import shows a per-workflow table before the confirmation prompt: create/update, Terraform version with the 1.5.7 fallback marked, runner, triggers, variable and secret counts. 'import --dry-run' stops after the table without touching anything. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01YKBeQrpJgR9DVFtR83myqX --- scripts/lib/report.sh | 89 +++++++++++++++++++++++++++++++++++++++++++ scripts/migrate.sh | 21 ++++++++-- 2 files changed, 106 insertions(+), 4 deletions(-) create mode 100644 scripts/lib/report.sh diff --git a/scripts/lib/report.sh b/scripts/lib/report.sh new file mode 100644 index 0000000..a670516 --- /dev/null +++ b/scripts/lib/report.sh @@ -0,0 +1,89 @@ +#!/bin/bash +# Reporting helpers (sourced; needs tools.sh, sg_api.sh): the migration +# summary shown after apply, and the per-workflow import plan shown before the +# confirmation prompt (and by 'import --dry-run'). + +# show_migration_summary — condensed view of export/migration-summary.json plus +# state-export-failures.log, so nobody has to know to open the files. +show_migration_summary() { + local f="$EXPORT_DIR/migration-summary.json" jqb n + [ -f "$f" ] || return 0 + jqb="$(sg_resolve jq sg_ensure_jq)" + sg_step "Migration summary" + printf ' %s%-30s%s %s\n' "$C_BOLD" "TFC organisation" "$C_RESET" "$("$jqb" -r '.organization' "$f")" >&2 + printf ' %s%-30s%s %s\n' "$C_BOLD" "Workspaces exported" "$C_RESET" "$("$jqb" -r '.workspaceCount' "$f")" >&2 + "$jqb" -r '.projectWorkspaceCounts | to_entries[] | " tfc-\(.key | ascii_downcase | gsub("[^a-z0-9-]+"; "-")): \(.value) workflow(s)"' "$f" >&2 + + _summary_section "$f" '.skippedSensitiveVars' "Sensitive variables skipped (TFC never exposes them)" \ + 'to_entries[] | "\(.key): \(.value | join(", "))"' "recreated as SG secrets after import" + _summary_section "$f" '.terraformVersionFallbacks' "Terraform version not pinned (SGDefaultTerraformVersion used)" \ + 'to_entries[] | "\(.key): \"\(.value)\""' "" + _summary_section "$f" '.nonRemoteExecutionModes' "Non-remote execution mode (state may not be in TFC)" \ + 'to_entries[] | "\(.key): \(.value)"' "" + _summary_section "$f" '.renamedWorkspaces' "Renamed to a valid SG workflow name" \ + 'to_entries[] | "\(.key) -> \(.value)"' "" + if [ -s "$EXPORT_DIR/state-export-failures.log" ]; then + n="$(wc -l <"$EXPORT_DIR/state-export-failures.log" | tr -d ' ')" + printf ' %s!%s %s\n' "$C_YELLOW" "$C_RESET" "state could not be exported for $n workspace(s):" >&2 + sed 's/^/ /' "$EXPORT_DIR/state-export-failures.log" | head -10 >&2 + [ "$n" -gt 10 ] && sg_dim " ... see $(sg_rel "$EXPORT_DIR")/state-export-failures.log" + fi + sg_dim "full report: $(sg_rel "$EXPORT_DIR")/migration-summary.md" +} + +# _summary_section <jq-line-filter> <hint> +_summary_section() { + local f="$1" path="$2" title="$3" lines="$4" hint="$5" jqb n + jqb="$(sg_resolve jq sg_ensure_jq)" + n="$("$jqb" -r "$path | length" "$f")" + [ "$n" -gt 0 ] || return 0 + printf ' %s!%s %s (%s)%s\n' "$C_YELLOW" "$C_RESET" "$title" "$n" "${hint:+ — $hint}" >&2 + "$jqb" -r "$path | $lines" "$f" | head -10 | sed 's/^/ /' >&2 + [ "$n" -gt 10 ] && sg_dim " ... $((n - 10)) more in migration-summary.md" + return 0 +} + +# tf_version_above_ceiling <TERRAFORM-x.y.z> — exit 0 when above 1.5.7. +tf_version_above_ceiling() { + [[ "$1" =~ ^TERRAFORM-([0-9]+)\.([0-9]+)\.([0-9]+)$ ]] || return 1 + [ "$(printf '%03d%03d%03d' "${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}" "${BASH_REMATCH[3]}")" -gt "001005007" ] +} + +# show_import_plan <payload>... — per-workflow table: what will be created or +# updated, with the Terraform version (and fallback), runner, triggers and the +# number of variables / skipped secrets. Needs JQ_BIN, ORG, SG_API_TOKEN. +show_import_plan() { + local f seg grp existing summary rows + summary="$EXPORT_DIR/migration-summary.json" + [ -f "$summary" ] || summary="" + printf '\n %s%-28s %-24s %-7s %-28s %-8s %-8s %-5s %s%s\n' "$C_BOLD" "WORKFLOW" "GROUP" "ACTION" "TERRAFORM" "RUNNER" "TRIGGERS" "VARS" "SECRETS" "$C_RESET" >&2 + for f in "$@"; do + seg="$(seg_of "$f")" + grp="$(group_for "$seg")" + existing="$(sg_list_workflows "$grp")" + rows="$("$JQ_BIN" -r --argjson ex "$existing" --arg def "$SG_DEFAULT_TF_VERSION" --argjson ws "$(ws_filter_json)" \ + --slurpfile sum "${summary:-/dev/null}" ' + ($sum[0] // {}) as $S + | .[] + | select(($ws | length) == 0 or (.ResourceName as $n | $ws | index($n) != null)) + | ((.CLIConfiguration.TfStateFilePath // "") | sub(".*/"; "") | sub("\\.tfstate$"; "")) as $wsName + | .ResourceName as $n + | [ $n, + (if ($ex | index($n)) != null then "update" else "create" end), + (.TerraformConfig.terraformVersion // "-"), + ((.RunnerConstraints // {}) | if .type == "private" then "private" else "shared" end), + (if (.VCSTriggers // null) != null then "yes" else "no" end), + ((.VCSConfig.iacInputData.data // {}) | length), + (($S.skippedSensitiveVars // {})[$wsName] // [] | length) + ] | @tsv' "$f")" + while IFS=$'\t' read -r name action tfv runner trig vars secrets; do + [ -n "$name" ] || continue + if tf_version_above_ceiling "$tfv"; then tfv="${tfv#TERRAFORM-} -> ${SG_DEFAULT_TF_VERSION#TERRAFORM-} (fallback)"; else tfv="${tfv#TERRAFORM-}"; fi + [ "$secrets" = "0" ] && secrets="-" + case "$action" in create) action="${C_GREEN}create ${C_RESET}" ;; update) action="${C_YELLOW}update ${C_RESET}" ;; esac + printf ' %-28s %-24s %s %-28s %-8s %-8s %-5s %s\n' "$name" "$grp" "$action" "$tfv" "$runner" "$trig" "$vars" "$secrets" >&2 + done <<<"$rows" + done + echo >&2 + sg_dim "TERRAFORM '-> fallback' = pinned above SG's managed ceiling (1.5.7, last FOSS release); SECRETS = sensitive vars to recreate" +} diff --git a/scripts/migrate.sh b/scripts/migrate.sh index dd78765..0fb52ee 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -23,6 +23,8 @@ source "$SCRIPT_DIR/lib/wizard.sh" source "$SCRIPT_DIR/lib/preflight.sh" # shellcheck source=lib/state.sh source "$SCRIPT_DIR/lib/state.sh" +# shellcheck source=lib/report.sh +source "$SCRIPT_DIR/lib/report.sh" # SCRIPT_DIR holds the sibling scripts; SG_REPO_ROOT (from tools.sh) is the repo # root used for all repo-relative paths. @@ -43,6 +45,7 @@ SKIP_PREFLIGHT=0 # shellcheck disable=SC2034 PREFLIGHT_DONE=0 FRESH=0 +DRY_RUN=0 PROJECT_FILTER=() WS_FILTER=() VERBOSE="${SG_VERBOSE:-0}" @@ -86,6 +89,7 @@ Options: --no-variable-sets Skip merging TFC Variable Set variables in the 'all' flow --no-vcs-triggers Skip registering VCS triggers after import --skip-preflight Skip the preflight checks (not recommended) + --dry-run With 'import': show the per-workflow plan and stop (nothing is created) --fresh Ignore the saved run state: redo every phase and re-import everything --project SEG Only handle this TFC project (repeatable; matches sg-payload.<SEG>.json) --workspace NAME Only handle this workspace (repeatable; apply exports only it) @@ -370,6 +374,7 @@ cmd_apply() { [ "$rc" -eq 0 ] || return "$rc" sg_success "apply complete — payloads in $(sg_rel "$EXPORT_DIR")" + show_migration_summary } cmd_enrich() { @@ -583,6 +588,15 @@ cmd_import() { die "some groups are missing (override groups are not auto-created; create them or remove the override)." fi + # Fallback Terraform version for workflows the API rejects as above the + # managed ceiling: SGDefaultTerraformVersion from terraform.tfvars, else 1.5.7. + SG_DEFAULT_TF_VERSION="${SG_DEFAULT_TF_VERSION:-$(tfvars_get '.SGDefaultTerraformVersion' TERRAFORM-1.5.7)}" + show_import_plan "${PF[@]}" + if [ "$DRY_RUN" -eq 1 ]; then + sg_success "dry run — nothing was created or changed" + return 0 + fi + if [ "$ASSUME_YES" -ne 1 ]; then printf '%sProceed?%s This imports to %s and creates any "create" groups. [y/N] ' "$C_BOLD$C_YELLOW" "$C_RESET" "$ORG" >&2 read -r ans || ans="" @@ -596,9 +610,6 @@ cmd_import() { done SGCLI_BIN="$(sg_resolve sg-cli sg_ensure_sgcli)" - # Fallback Terraform version for workflows the API rejects as above the - # managed ceiling: SGDefaultTerraformVersion from terraform.tfvars, else 1.5.7. - SG_DEFAULT_TF_VERSION="${SG_DEFAULT_TF_VERSION:-$(tfvars_get '.SGDefaultTerraformVersion' TERRAFORM-1.5.7)}" rm -f "$EXPORT_DIR/terraform-version-fallbacks.log" # Resume: skip payload files already imported in full with identical content @@ -645,7 +656,7 @@ cmd_import() { # Single source of truth for shell completion (keep in sync with the parser below # and the host-only flags in sg-migrate.sh). SG_COMMANDS="init preflight apply enrich convert validate import triggers all clean completion" -SG_OPTIONS="--org --export-dir --mapping --concurrency --no-create-groups --no-variable-sets --no-vcs-triggers --skip-preflight --fresh --project --workspace --all -v --verbose -y --yes -h --help --native --local --build" +SG_OPTIONS="--org --export-dir --mapping --concurrency --no-create-groups --no-variable-sets --no-vcs-triggers --skip-preflight --dry-run --fresh --project --workspace --all -v --verbose -y --yes -h --help --native --local --build" # cmd_completion <bash|zsh> — print a completion script for sg-migrate.sh / # migrate.sh to stdout. Both shells fall back to the basename when the command @@ -707,6 +718,7 @@ _sg_migrate() { '--no-variable-sets[Skip merging TFC Variable Sets]' \\ '--no-vcs-triggers[Skip registering VCS triggers after import]' \\ '--skip-preflight[Skip the preflight checks]' \\ + '--dry-run[With import: show the plan and stop]' \\ '--fresh[Ignore saved run state: redo every phase]' \\ '*--project[Only this TFC project segment]:segment' \\ '*--workspace[Only this workspace]:name' \\ @@ -759,6 +771,7 @@ while [ $# -gt 0 ]; do --no-vcs-triggers) VCS_TRIGGERS=0 ;; --skip-preflight) export SKIP_PREFLIGHT=1 ;; --fresh) FRESH=1 ;; + --dry-run) DRY_RUN=1 ;; --project) PROJECT_FILTER+=("$2") shift From ec461e01d1dc5ac92d50dd0250231e0c49ff0793 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 7 Sep 2026 09:29:47 +0200 Subject: [PATCH 18/30] feat: explain known StackGuardian API errors Map the API messages users actually hit (unknown connector, missing runner group, repository not reachable, bad approvers, invalid names or VCS kind, rejected token, Terraform ceiling) to a one-line hint naming the terraform.tfvars field to fix. Shown once per distinct hint, both for direct API calls and for sg-cli's per-workflow failures; unknown messages still show the raw response. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01YKBeQrpJgR9DVFtR83myqX --- scripts/lib/errors.sh | 42 ++++++++++++++++++++++++++++++++++++++++++ scripts/migrate.sh | 5 +++++ 2 files changed, 47 insertions(+) create mode 100644 scripts/lib/errors.sh diff --git a/scripts/lib/errors.sh b/scripts/lib/errors.sh new file mode 100644 index 0000000..bed4439 --- /dev/null +++ b/scripts/lib/errors.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# Friendly explanations for StackGuardian API errors (sourced; needs tools.sh). +# +# explain_api_error <response-body-or-message> prints, at most once per +# distinct hint per run, what the error usually means and which +# terraform.tfvars field to change. Unknown messages print nothing (the raw +# response is already shown by the caller). Patterns are case-insensitive +# extended regexes matched against the message text. + +_SG_HINTS_SHOWN=" " + +# Table: "<regex> => <hint>" — keep hints to one line each. +SG_API_HINTS=( + 'above the highest managed version => Terraform version above SG'"'"'s managed ceiling (1.5.7, last MPL/FOSS release). The importer retries with SGDefaultTerraformVersion automatically; to keep the newer version use a private runner binary path or a custom runtime template (workspaceOverrides[<ws>].terraformVersion).' + 'integration.*(not found|does not exist|invalid)|(not found|does not exist|invalid).*integration => Connector/integration id not found in this org. Check SGDefaultVCSAuthIntegrationID and SGDefaultDeploymentPlatformConfig[].config.integrationId (or the workspaceOverrides equivalents); '"'"'preflight'"'"' lists the ids that exist.' + 'runner ?group.*(not found|does not exist|invalid)|(not found|does not exist|invalid).*runner => Runner group not found. Check SGDefaultRunnerConstraints.names / workspaceOverrides[<ws>].RunnerConstraints against the runner groups in the SG org.' + '(repo|repository).*(not found|access|permission|denied|unable|could not|cannot)|(clone|checkout).*(fail|denied|unable) => The VCS connector cannot reach the repository. Check SGDefaultIACVCSRepoPrefix + the workspace repo path, and that the connector (SGDefaultVCSAuthIntegrationID) has access to that repository.' + 'approver => Approvers must be e-mail addresses of existing SG users. Check SGDefaultWfApprovers / workspaceOverrides[<ws>].Approvers.' + 'already exists => A resource with that name already exists. Re-running the import updates workflows in place; for workflow groups this is harmless.' + 'ResourceName => Invalid workflow name: 1-100 chars, letters/digits/-/_ only. The transformer sanitizes names (see renamedWorkspaces in migration-summary.md); adjust local.resourceNames if a rule is missing.' + 'sourceConfigDestKind => Invalid VCS kind. SGDefaultSourceConfigDestKind must be one of GITHUB_COM, GITLAB_COM, BITBUCKET_ORG, AZURE_DEVOPS, GIT_OTHER.' + '(unauthori[sz]ed|forbidden|invalid token|authentication) => SG_API_TOKEN was rejected or lacks permission for this org. Regenerate it under Org settings -> API keys and re-export SG_API_TOKEN.' +) + +explain_api_error() { + local msg="$1" entry re hint + # Prefer the API's "msg" field when the body is JSON. + if printf '%s' "$msg" | grep -q '^{'; then + msg="$(printf '%s' "$msg" | "$(sg_resolve jq sg_ensure_jq)" -r '.msg // .message // .error // .' 2>/dev/null || printf '%s' "$msg")" + fi + for entry in "${SG_API_HINTS[@]}"; do + re="${entry%% => *}" + hint="${entry#* => }" + if printf '%s' "$msg" | grep -Eiq -- "$re"; then + case "$_SG_HINTS_SHOWN" in *" $re "*) return 0 ;; esac + _SG_HINTS_SHOWN="$_SG_HINTS_SHOWN$re " + printf ' %s→ %s%s\n' "$C_YELLOW" "$hint" "$C_RESET" >&2 + return 0 + fi + done + return 0 +} diff --git a/scripts/migrate.sh b/scripts/migrate.sh index 0fb52ee..0527d28 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -25,6 +25,8 @@ source "$SCRIPT_DIR/lib/preflight.sh" source "$SCRIPT_DIR/lib/state.sh" # shellcheck source=lib/report.sh source "$SCRIPT_DIR/lib/report.sh" +# shellcheck source=lib/errors.sh +source "$SCRIPT_DIR/lib/errors.sh" # SCRIPT_DIR holds the sibling scripts; SG_REPO_ROOT (from tools.sh) is the repo # root used for all repo-relative paths. @@ -461,6 +463,9 @@ do_import() { if [[ "$line" =~ $TF_CEILING_RE ]]; then fb+=("${BASH_REMATCH[1]}") ceiling="${BASH_REMATCH[2]}" + elif [[ "$line" =~ Failed\ to\ create\ ([^:]+):\ [0-9]+:\ (.*)$ ]]; then + failed+=("${BASH_REMATCH[1]}") + explain_api_error "${BASH_REMATCH[2]}" elif [[ "$line" =~ Failed\ to\ create\ ([^:]+): ]]; then failed+=("${BASH_REMATCH[1]}") fi From 53c7d6b8a22fa7f98270cfd0a7bfe69597c22e97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 7 Sep 2026 09:31:17 +0200 Subject: [PATCH 19/30] feat: post-import checklist and placeholder secrets After import (or via 'checklist'), write export/post-import-checklist.md listing everything the migration could not do by itself: secrets to fill in, workflows that failed to import, Terraform version fallbacks to verify, VCS triggers that failed, state that could not be exported, non-remote execution modes and renamed workflows, with deep links into the SG UI (SG_UI_URL). For every sensitive variable TFC would not expose, create an SG secret with the value CHANGE_ME, reference it from the workflow (env var or IaC input, ${secret::<name>}) and patch the payload to match; --no-secret-stubs opts out. Trigger registration results are now recorded in the run state. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01YKBeQrpJgR9DVFtR83myqX --- scripts/lib/checklist.sh | 162 +++++++++++++++++++++++++++++++++++++++ scripts/migrate.sh | 40 ++++++++-- sg-migrate.sh | 2 +- 3 files changed, 198 insertions(+), 6 deletions(-) create mode 100644 scripts/lib/checklist.sh diff --git a/scripts/lib/checklist.sh b/scripts/lib/checklist.sh new file mode 100644 index 0000000..748683c --- /dev/null +++ b/scripts/lib/checklist.sh @@ -0,0 +1,162 @@ +#!/bin/bash +# Post-import checklist and secret stubs (sourced; needs tools.sh, sg_api.sh, +# state.sh). Turns everything the migration could not do automatically into +# export/post-import-checklist.md, and creates placeholder SG secrets for the +# sensitive variables TFC never exposes so the workflows are wired up and the +# user only has to fill in values. + +SG_UI_URL="${SG_UI_URL:-https://app.stackguardian.io}" +wf_ui_url() { printf '%s/orchestrator/orgs/%s/wfgrps/%s/wfs/%s' "$SG_UI_URL" "$ORG" "$1" "$2"; } +secrets_ui_url() { printf '%s/orchestrator/orgs/%s/settings?tab=secrets' "$SG_UI_URL" "$ORG"; } + +# secret_name_for <workflow> <var> — SG secret name for a stubbed variable. +secret_name_for() { printf 'tfc-%s-%s' "$1" "$2" | tr -c 'A-Za-z0-9_-\n' '-' | cut -c1-100; } + +# workflow_for_workspace <tfc-workspace-name> — "<seg>\t<group>\t<ResourceName>" +# for the payload entry whose TfStateFilePath basename is the workspace. +workflow_for_workspace() { + local f seg wf + for f in "${PF[@]}"; do + wf="$("$JQ_BIN" -r --arg ws "$1" '.[] | select(((.CLIConfiguration.TfStateFilePath // "") | sub(".*/"; "") | sub("\\.tfstate$"; "")) == $ws) | .ResourceName' "$f" | head -1)" + if [ -n "$wf" ]; then + seg="$(seg_of "$f")" + printf '%s\t%s\t%s\n' "$seg" "$(group_for "$seg")" "$wf" + return 0 + fi + done + return 1 +} + +# create_secret_stubs — for every skipped sensitive variable: create the SG +# secret (placeholder value) if missing, reference it from the workflow (env +# var or IaC input), patch the payload file to match, and record it in state. +create_secret_stubs() { + local summary="$EXPORT_DIR/migration-summary.json" ws entry cat var f seg grp wf name ref created=0 reused=0 patched=0 skipped=0 body imported + [ -f "$summary" ] || return 0 + [ "$("$JQ_BIN" '.skippedSensitiveVars | length' "$summary")" -gt 0 ] || return 0 + sg_log "creating placeholder secrets for sensitive variables (value: CHANGE_ME)..." + while IFS=$'\t' read -r ws entry; do + [ -n "$ws" ] || continue + cat="${entry%%:*}" + var="${entry#*:}" + if ! IFS=$'\t' read -r seg grp wf < <(workflow_for_workspace "$ws"); then + sg_warn " $ws: no payload entry found for $entry — listed in the checklist only" + skipped=$((skipped + 1)) + continue + fi + ws_selected "$wf" || continue + imported="$(state_read | "$JQ_BIN" -r --arg s "$seg" --arg w "$wf" '.import[$s].imported // [] | index($w) != null')" + if [ "$imported" != "true" ] || ! sg_workflow_exists "$grp" "$wf"; then + sg_warn " $grp/$wf is not in StackGuardian (import failed?) — $entry listed in the checklist only" + skipped=$((skipped + 1)) + continue + fi + name="$(secret_name_for "$wf" "$var")" + ref="\${secret::$name}" + if sg_secret_exists "$name"; then + reused=$((reused + 1)) + elif sg_create_secret "$name" "CHANGE_ME" "Placeholder for TFC sensitive variable $entry of workspace $ws — set the real value"; then + created=$((created + 1)) + else + sg_warn " could not create secret $name — $entry listed in the checklist only" + skipped=$((skipped + 1)) + continue + fi + # Reference the secret from the workflow: env var or IaC input value. + f="$EXPORT_DIR/sg-payload.$seg.json" + if [ "$cat" = "env" ]; then + "$JQ_BIN" --arg w "$wf" --arg v "$var" --arg r "$ref" 'map(if .ResourceName == $w then + .EnvironmentVariables = ((.EnvironmentVariables // []) | map(select(.config.varName != $v)) + [{kind:"PLAIN_TEXT", config:{varName:$v, textValue:$r}}]) else . end)' "$f" >"$f.tmp" && mv -f "$f.tmp" "$f" + body="$("$JQ_BIN" -c --arg w "$wf" '.[] | select(.ResourceName == $w) | {EnvironmentVariables}' "$f")" + else + "$JQ_BIN" --arg w "$wf" --arg v "$var" --arg r "$ref" 'map(if .ResourceName == $w then .VCSConfig.iacInputData.data[$v] = $r else . end)' "$f" >"$f.tmp" && mv -f "$f.tmp" "$f" + body="$("$JQ_BIN" -c --arg w "$wf" '.[] | select(.ResourceName == $w) | {VCSConfig}' "$f")" + fi + if SG_NO_RETRY_RC=22 sg_retry "$RETRIES" "$RETRY_BASE" -- sg_patch_workflow "$grp" "$wf" "$body"; then + patched=$((patched + 1)) + else + sg_warn " secret $name created but $grp/$wf could not be updated to reference it" + fi + state_update '.secrets[$n] = {workflow: $w, group: $g, var: $v, category: $c, workspace: $ws, at: $at}' \ + --arg n "$name" --arg w "$wf" --arg g "$grp" --arg v "$var" --arg c "$cat" --arg ws "$ws" --arg at "$(state_now)" + done < <("$JQ_BIN" -r '.skippedSensitiveVars | to_entries[] | .key as $ws | .value[] | "\($ws)\t\(.)"' "$summary") + sg_log "secret stubs: $created created, $reused already existed, $patched workflow(s) now reference them, $skipped skipped" + return 0 +} + +# write_checklist — render export/post-import-checklist.md and print it. +write_checklist() { + local out="$EXPORT_DIR/post-import-checklist.md" summary="$EXPORT_DIR/migration-summary.json" st items + st="$(state_read)" + { + printf '# Post-import checklist — StackGuardian org %s\n\n' "$ORG" + printf 'Generated %s by stackguardian-migrator. Tick items as you complete them.\n\n' "$(state_now)" + + # 1. secrets + printf '## 1. Set the real values of the placeholder secrets\n\n' + printf 'TFC never exposes sensitive variable values, so each one was recreated as an SG secret with the value `CHANGE_ME` and referenced from its workflow as `${secret::<name>}`. Set the real values under [Org settings → Secrets](%s).\n\n' "$(secrets_ui_url)" + items="$(printf '%s' "$st" | "$JQ_BIN" -r --arg ui "$SG_UI_URL" --arg org "$ORG" '.secrets // {} | to_entries[] | "- [ ] `\(.key)` — \(.value.category) var `\(.value.var)` of [\(.value.group)/\(.value.workflow)](\($ui)/orchestrator/orgs/\($org)/wfgrps/\(.value.group)/wfs/\(.value.workflow))"')" + if [ -n "$items" ]; then printf '%s\n\n' "$items"; else printf -- '- None.\n\n'; fi + if [ -f "$summary" ]; then + items="$("$JQ_BIN" -r --argjson stubbed "$(printf '%s' "$st" | "$JQ_BIN" -c '[.secrets // {} | .[] | "\(.workspace)|\(.category):\(.var)"]')" \ + '.skippedSensitiveVars | to_entries[] | .key as $ws | .value[] | select(($ws + "|" + .) as $k | $stubbed | index($k) == null) | "- [ ] `\(.)` of workspace `\($ws)` — no stub created (workflow missing or --no-secret-stubs); create the secret and reference it by hand"' "$summary")" + [ -n "$items" ] && printf '%s\n\n' "$items" + fi + + # 2. failed imports + printf '## 2. Workflows that failed to import\n\n' + items="$(printf '%s' "$st" | "$JQ_BIN" -r '.import // {} | to_entries[] | .value.group as $g | .value.failed[]? | "- [ ] `\($g)/\(.)` — see the import output for the API error; fix terraform.tfvars (or workspaceOverrides) and re-run `./sg-migrate.sh import`"')" + if [ -n "$items" ]; then printf '%s\n\n' "$items"; else printf -- '- None.\n\n'; fi + + # 3. terraform version fallbacks + printf '## 3. Verify workflows moved to a different Terraform version\n\n' + printf 'StackGuardian bundles managed Terraform only up to 1.5.7 (the last MPL/FOSS release). These workflows were pinned higher in TFC and now run the fallback version; run a plan and check for incompatibilities. To keep the newer version, point `workspaceOverrides[<ws>].terraformVersion` at a binary on a private runner and re-import.\n\n' + items="" + [ -s "$EXPORT_DIR/terraform-version-fallbacks.log" ] && items="$(sed 's/^/- [ ] /' "$EXPORT_DIR/terraform-version-fallbacks.log")" + if [ -f "$summary" ]; then + items="$items$(printf '%s' "$items" | grep -q . && echo)$("$JQ_BIN" -r '.terraformVersionFallbacks | to_entries[] | "- [ ] `\(.key)` was not pinned in TFC (\"\(.value)\"); SGDefaultTerraformVersion was used — confirm it is compatible"' "$summary")" + fi + if [ -n "$items" ]; then printf '%s\n\n' "$items"; else printf -- '- None.\n\n'; fi + + # 4. VCS triggers + printf '## 4. VCS triggers\n\n' + items="$(printf '%s' "$st" | "$JQ_BIN" -r '.triggers // {} | to_entries[] | .value.group as $g | (.value.failed[]? | "- [ ] `\($g)/\(.)` — trigger registration failed; check the connector has admin/webhook rights on the repository, then re-run `./sg-migrate.sh triggers`"), (.value.missing[]? | "- [ ] `\($g)/\(.)` — workflow was not imported, so no trigger was registered")')" + if [ -n "$items" ]; then printf '%s\n\n' "$items"; else printf -- '- None failed. Push to a tracked branch or open a pull request to confirm the webhooks fire.\n\n'; fi + + # 5. state + printf '## 5. Terraform state\n\n' + if [ -s "$EXPORT_DIR/state-export-failures.log" ]; then + printf 'State could not be pulled from TFC for these workspaces; upload it manually (Workflow → Settings → State) or run an import in the new workflow.\n\n' + sed 's/^/- [ ] /' "$EXPORT_DIR/state-export-failures.log"; echo + else + printf -- '- All selected workspaces had their state exported.\n\n' + fi + if [ -f "$summary" ]; then + items="$("$JQ_BIN" -r '.nonRemoteExecutionModes | to_entries[] | "- [ ] `\(.key)` used `\(.value)` execution in TFC — its state may live outside TFC; verify the exported state is current"' "$summary")" + [ -n "$items" ] && printf '%s\n\n' "$items" + fi + + # 6. renames + if [ -f "$summary" ] && [ "$("$JQ_BIN" '.renamedWorkspaces | length' "$summary")" -gt 0 ]; then + printf '## 6. Renamed workflows\n\nThese TFC workspace names were not valid StackGuardian workflow names and were adjusted:\n\n' + "$JQ_BIN" -r '.renamedWorkspaces | to_entries[] | "- `\(.key)` → `\(.value)`"' "$summary"; echo + fi + + printf '## Finally\n\n- [ ] Run a plan on one workflow per project and compare with the last TFC run.\n- [ ] Disable auto-apply / triggers on the TFC workspaces once StackGuardian owns the deployments.\n' + } >"$out" + sg_step "Post-import checklist" + sed 's/^/ /' "$out" >&2 + sg_dim "saved to $(sg_rel "$out")" +} + +cmd_checklist() { + sg_step "Phase: checklist" + [ -n "${SG_API_TOKEN:-}" ] || die "SG_API_TOKEN is not set." + [ -n "$ORG" ] || die "StackGuardian org not set (use --org or SG_ORG)." + export SG_API_TOKEN SG_BASE_URL + JQ_BIN="$(sg_resolve jq sg_ensure_jq)" + payload_files + [ "${#PF[@]}" -gt 0 ] || die "No payload files in $(sg_rel "$EXPORT_DIR")." + [ "$SECRET_STUBS" -eq 1 ] && create_secret_stubs + write_checklist +} diff --git a/scripts/migrate.sh b/scripts/migrate.sh index 0527d28..2ad0bf1 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -27,6 +27,8 @@ source "$SCRIPT_DIR/lib/state.sh" source "$SCRIPT_DIR/lib/report.sh" # shellcheck source=lib/errors.sh source "$SCRIPT_DIR/lib/errors.sh" +# shellcheck source=lib/checklist.sh +source "$SCRIPT_DIR/lib/checklist.sh" # SCRIPT_DIR holds the sibling scripts; SG_REPO_ROOT (from tools.sh) is the repo # root used for all repo-relative paths. @@ -48,6 +50,7 @@ SKIP_PREFLIGHT=0 PREFLIGHT_DONE=0 FRESH=0 DRY_RUN=0 +SECRET_STUBS=1 PROJECT_FILTER=() WS_FILTER=() VERBOSE="${SG_VERBOSE:-0}" @@ -72,6 +75,8 @@ Commands: import Import each payload to StackGuardian (parallel, with confirmation), then register VCS triggers (unless --no-vcs-triggers) triggers Register VCS triggers for already-imported workflows (second pass) + checklist Create placeholder secrets for skipped sensitive variables and write + export/post-import-checklist.md (runs automatically after import) all apply -> enrich -> convert -> validate -> import clean Remove local working artifacts for a fresh start (export/, TF state, tool cache). Add --all to also remove config (terraform.tfvars, mapping). @@ -92,6 +97,7 @@ Options: --no-vcs-triggers Skip registering VCS triggers after import --skip-preflight Skip the preflight checks (not recommended) --dry-run With 'import': show the per-workflow plan and stop (nothing is created) + --no-secret-stubs Do not create placeholder SG secrets for sensitive variables --fresh Ignore the saved run state: redo every phase and re-import everything --project SEG Only handle this TFC project (repeatable; matches sg-payload.<SEG>.json) --workspace NAME Only handle this workspace (repeatable; apply exports only it) @@ -105,6 +111,7 @@ Environment: SG_ORG StackGuardian org (alternative to --org) SG_RETRIES Import retry attempts on failure (default: 4) SG_TF_PARALLELISM terraform apply -parallelism (default: 20) + SG_UI_URL StackGuardian UI base for checklist links (default: https://app.stackguardian.io) EOF } @@ -257,7 +264,7 @@ group_for() { # straight from the (converted) payload. Per-workflow failures are surfaced but # do not abort the rest of the file. do_set_triggers() { - local f="$1" seg grp n i wf body rc=0 set=0 skip=0 + local f="$1" seg grp n i wf body rc=0 set=0 skip=0 ok=() failed=() missing=() seg="$(seg_of "$f")" grp="$(group_for "$seg")" n="$("$JQ_BIN" 'length' "$f")" @@ -271,6 +278,7 @@ do_set_triggers() { # A workflow that failed to import has nothing to attach triggers to. if ! sg_workflow_exists "$grp" "$wf"; then sg_warn " $grp/$wf does not exist in SG (import failed?) — skipping triggers" + missing+=("$wf") rc=1 continue fi @@ -278,12 +286,19 @@ do_set_triggers() { if SG_NO_RETRY_RC=22 sg_retry "$RETRIES" "$RETRY_BASE" -- \ sg_api_post "$(wf_triggers_endpoint "$grp" "$wf")" "$body"; then set=$((set + 1)) + ok+=("$wf") else sg_warn " vcs triggers failed: $grp/$wf" + failed+=("$wf") rc=1 fi done sg_log "$(basename "$f"): set triggers on $set workflow(s) (skipped $skip without triggers)" + "$JQ_BIN" -nc --arg g "$grp" \ + --argjson ok "$([ "${#ok[@]}" -gt 0 ] && names_json "${ok[@]}" || echo '[]')" \ + --argjson failed "$([ "${#failed[@]}" -gt 0 ] && names_json "${failed[@]}" || echo '[]')" \ + --argjson missing "$([ "${#missing[@]}" -gt 0 ] && names_json "${missing[@]}" || echo '[]')" \ + '{group: $g, set: $ok, failed: $failed, missing: $missing}' >"$EXPORT_DIR/.triggers-result.$seg.json" return "$rc" } @@ -297,7 +312,16 @@ set_triggers_pass() { return 0 fi sg_log "registering VCS triggers for $total workflow(s), up to $CONC in parallel (retries: $RETRIES)" - if run_parallel do_set_triggers "$CONC" "${PF[@]}"; then + local rc=0 f seg + run_parallel do_set_triggers "$CONC" "${PF[@]}" || rc=1 + for f in "${PF[@]}"; do + seg="$(seg_of "$f")" + if [ -f "$EXPORT_DIR/.triggers-result.$seg.json" ]; then + state_update '.triggers[$s] = ($r + {at: $at})' --arg s "$seg" --argjson r "$(cat "$EXPORT_DIR/.triggers-result.$seg.json")" --arg at "$(state_now)" + rm -f "$EXPORT_DIR/.triggers-result.$seg.json" + fi + done + if [ "$rc" -eq 0 ]; then sg_success "vcs triggers registered" else sg_err "one or more VCS trigger registrations failed (re-run: $PROG triggers)" @@ -655,13 +679,15 @@ cmd_import() { if [ "$VCS_TRIGGERS" -eq 1 ]; then set_triggers_pass || import_rc=1 fi + [ "$SECRET_STUBS" -eq 1 ] && create_secret_stubs + write_checklist return "$import_rc" } # Single source of truth for shell completion (keep in sync with the parser below # and the host-only flags in sg-migrate.sh). -SG_COMMANDS="init preflight apply enrich convert validate import triggers all clean completion" -SG_OPTIONS="--org --export-dir --mapping --concurrency --no-create-groups --no-variable-sets --no-vcs-triggers --skip-preflight --dry-run --fresh --project --workspace --all -v --verbose -y --yes -h --help --native --local --build" +SG_COMMANDS="init preflight apply enrich convert validate import triggers checklist all clean completion" +SG_OPTIONS="--org --export-dir --mapping --concurrency --no-create-groups --no-variable-sets --no-vcs-triggers --skip-preflight --dry-run --no-secret-stubs --fresh --project --workspace --all -v --verbose -y --yes -h --help --native --local --build" # cmd_completion <bash|zsh> — print a completion script for sg-migrate.sh / # migrate.sh to stdout. Both shells fall back to the basename when the command @@ -710,6 +736,7 @@ _sg_migrate() { 'validate:Validate payloads against the SG schema' 'import:Import payloads to StackGuardian, then register VCS triggers' 'triggers:Register VCS triggers for already-imported workflows' + 'checklist:Write the post-import checklist (and create secret stubs)' 'all:apply -> enrich -> convert -> validate -> import' 'clean:Remove local working artifacts' 'completion:Print a shell completion script' @@ -724,6 +751,7 @@ _sg_migrate() { '--no-vcs-triggers[Skip registering VCS triggers after import]' \\ '--skip-preflight[Skip the preflight checks]' \\ '--dry-run[With import: show the plan and stop]' \\ + '--no-secret-stubs[Do not create placeholder SG secrets for sensitive vars]' \\ '--fresh[Ignore saved run state: redo every phase]' \\ '*--project[Only this TFC project segment]:segment' \\ '*--workspace[Only this workspace]:name' \\ @@ -777,6 +805,7 @@ while [ $# -gt 0 ]; do --skip-preflight) export SKIP_PREFLIGHT=1 ;; --fresh) FRESH=1 ;; --dry-run) DRY_RUN=1 ;; + --no-secret-stubs) SECRET_STUBS=0 ;; --project) PROJECT_FILTER+=("$2") shift @@ -793,7 +822,7 @@ while [ $# -gt 0 ]; do usage exit 0 ;; - init | apply | enrich | convert | validate | import | triggers | all | clean | preflight) CMD="$1" ;; + init | apply | enrich | convert | validate | import | triggers | all | clean | preflight | checklist) CMD="$1" ;; completion) cmd_completion "${2:-}" exit 0 @@ -826,6 +855,7 @@ preflight) export SG_API_TOKEN SG_BASE_URL cmd_preflight ;; +checklist) cmd_checklist ;; all) if [ ! -f "$TFVARS" ]; then cmd_init diff --git a/sg-migrate.sh b/sg-migrate.sh index 5a56ba4..0d96281 100755 --- a/sg-migrate.sh +++ b/sg-migrate.sh @@ -33,7 +33,7 @@ HAS_CMD=0 for a in ${ARGS[@]+"${ARGS[@]}"}; do case "$a" in clean | completion | -h | --help) NATIVE=1; HAS_CMD=1 ;; - init | preflight | apply | enrich | convert | validate | import | triggers | all) HAS_CMD=1 ;; + init | preflight | apply | enrich | convert | validate | import | triggers | checklist | all) HAS_CMD=1 ;; esac done [ "$HAS_CMD" -eq 1 ] || NATIVE=1 From 23371e1c3b87a62c2d40f7ef52518d1a5b794809 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 7 Sep 2026 09:32:26 +0200 Subject: [PATCH 20/30] docs: describe the guided init, preflight, resume, dry-run and checklist Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01YKBeQrpJgR9DVFtR83myqX --- CLAUDE.md | 3 ++- README.md | 30 +++++++++++++++++------------- scripts/migrate.sh | 5 +++-- 3 files changed, 22 insertions(+), 16 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index ad9f762..ed6fc39 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -24,7 +24,8 @@ The migration is a user-driven pipeline, not a single program: The five phases are wrapped by an orchestrator so users don't run them by hand: - `sg-migrate.sh` (host entrypoint, repo root) — runs `scripts/migrate.sh` **inside the Docker image** (`Dockerfile`), bind-mounting the repo at `/app` and forwarding `SG_API_TOKEN`/`SG_ORG`/`SG_BASE_URL`/`TFE_TOKEN`/`TF_TOKEN_*`. For TFC auth it prefers a long-lived `TFE_TOKEN` (env); otherwise it mounts `~/.terraform.d/credentials.tfrc.json` read-only. Runs natively instead when `--native`/`--local` is passed, `SG_NATIVE=1` is set, the command is `clean`/`completion`/help (or no command is given), or Docker is absent. Exports `SG_PROG` so `migrate.sh` shows `./sg-migrate.sh` in its usage/hints. -- `scripts/migrate.sh` — the actual orchestrator. Subcommands `init|apply|enrich|convert|validate|import|triggers|all|clean|completion` (no command prints the help menu; `enrich` runs in `all` unless `--no-variable-sets`). Resolves each tool via `sg_resolve` (PATH first — the image installs them — else `sg_ensure_*` cache). `apply`/`all` first run `require_tfc_auth`, which resolves the token the tfe provider will use (`TFE_TOKEN`, `TF_TOKEN_<host>`, or the `terraform login` file for `tfHostname`) and verifies it with `GET /api/v2/account/details`, failing fast on a missing or rejected (expired) credential. Runs `convert` and `import` in parallel (`--concurrency`, default 4), raises `terraform apply -parallelism`, and retries `sg-cli` imports with backoff (`sg_retry`, `SG_RETRIES`). Import resolves each project's workflow group, checks existence via the SG API, shows a plan (`exists`/`create`), prompts (skip with `-y`), **creates the missing `tfc-<project>` groups via the API**, then imports. sg-cli exits 0 even when individual workflows fail, so `do_import` parses its output: a workflow rejected as above SG's managed Terraform ceiling (1.5.7, last MPL/FOSS release) is re-imported with `SGDefaultTerraformVersion` (read from `terraform.tfvars`), the payload patched in place, and the case logged to `export/terraform-version-fallbacks.log` plus a printed notice; any other per-workflow failure fails the run. The trigger pass skips workflows that do not exist in SG, and `sg_api_post` returns 22 on 4xx so `sg_retry` (via `SG_NO_RETRY_RC`) does not retry definitive errors. `clean` removes local artifacts (`export/`, TF state, tool cache); `clean --all` also removes config. `completion bash|zsh` prints a completion script (`cmd_completion`; commands/options come from `SG_COMMANDS`/`SG_OPTIONS`, keep them in sync with the parser and the host flags); like `clean` it always runs natively. Output is concise by default — `apply` captures terraform's init/plan output and shows only progress + the final summary (or the full log on failure, `TF_IN_AUTOMATION=1`/`-no-color`); `-v`/`--verbose` (`SG_VERBOSE`) streams everything and un-gates the convert detail lines. Colored logging via `sg_step`/`sg_log`/`sg_success`/`sg_warn`/`sg_err`; paths shown relative via `sg_rel` (all auto-off when not a TTY / `NO_COLOR`). +- `scripts/migrate.sh` — the actual orchestrator (sources `scripts/lib/*.sh`, see below). Subcommands `init|preflight|apply|enrich|convert|validate|import|triggers|checklist|all|clean|completion` (no command prints the help menu; `enrich` runs in `all` unless `--no-variable-sets`). Flags beyond the basics: `--dry-run` (import: plan only), `--fresh` (ignore run state), `--project SEG` / `--workspace NAME` (repeatable filters; apply maps `--workspace` to `-var workspacenames=[...]`, import works on a jq-filtered temp copy), `--skip-preflight`, `--no-secret-stubs`. Resolves each tool via `sg_resolve` (PATH first — the image installs them — else `sg_ensure_*` cache). `apply`/`all` first run `require_tfc_auth`, which resolves the token the tfe provider will use (`TFE_TOKEN`, `TF_TOKEN_<host>`, or the `terraform login` file for `tfHostname`) and verifies it with `GET /api/v2/account/details`, failing fast on a missing or rejected (expired) credential. Runs `convert` and `import` in parallel (`--concurrency`, default 4), raises `terraform apply -parallelism`, and retries `sg-cli` imports with backoff (`sg_retry`, `SG_RETRIES`). Import resolves each project's workflow group, checks existence via the SG API, shows a plan (`exists`/`create`), prompts (skip with `-y`), **creates the missing `tfc-<project>` groups via the API**, then imports. sg-cli exits 0 even when individual workflows fail, so `do_import` parses its output: a workflow rejected as above SG's managed Terraform ceiling (1.5.7, last MPL/FOSS release) is re-imported with `SGDefaultTerraformVersion` (read from `terraform.tfvars`), the payload patched in place, and the case logged to `export/terraform-version-fallbacks.log` plus a printed notice; any other per-workflow failure fails the run. The trigger pass skips workflows that do not exist in SG, and `sg_api_post` returns 22 on 4xx so `sg_retry` (via `SG_NO_RETRY_RC`) does not retry definitive errors. `clean` removes local artifacts (`export/`, TF state, tool cache); `clean --all` also removes config. `completion bash|zsh` prints a completion script (`cmd_completion`; commands/options come from `SG_COMMANDS`/`SG_OPTIONS`, keep them in sync with the parser and the host flags); like `clean` it always runs natively. Output is concise by default — `apply` captures terraform's init/plan output and shows only progress + the final summary (or the full log on failure, `TF_IN_AUTOMATION=1`/`-no-color`); `-v`/`--verbose` (`SG_VERBOSE`) streams everything and un-gates the convert detail lines. Colored logging via `sg_step`/`sg_log`/`sg_success`/`sg_warn`/`sg_err`; paths shown relative via `sg_rel` (all auto-off when not a TTY / `NO_COLOR`). +- `scripts/lib/` — sourced libraries (a `lib/` gitignore rule is negated for this dir): `prompt.sh` (`sg_ask`/`sg_confirm`/`sg_select`; tty or `SG_ANSWERS_FILE` for tests; `SG_NONINTERACTIVE=1`/`-y`/no TTY → defaults), `tfvars.sh` (`tfvars_get` cached hcl2json reads, `tfvars_write` renders the wizard's `W_*` vars), `tfc_api.sh` (`tfc_http`/`tfc_get_all` paginated, `tfc_list_{orgs,projects,workspaces}`, `tfc_token` with provider precedence, `require_tfc_auth`), `sg_api.sh` (`_sg_api` 0/22/1 contract + `SG_HTTP_CODE`, `sg_list_integrations`, `sg_*_exists`, `sg_list_workflows`, `sg_create_secret`, `sg_patch_workflow`, `wfgroup_*`), `wizard.sh` (`wizard_run`: 4 steps TFC → SG → defaults → review), `preflight.sh` (`preflight_run <apply|import|all>`, once per process), `state.sh` (`.sg/state.json`: `phases.<name>.input_sha`, `import.<seg>.{imported,failed,tf_fallback}`, `triggers.<seg>`, `secrets.<name>`; `run_phase` in migrate.sh skips unchanged phases), `report.sh` (`show_migration_summary` after apply, `show_import_plan` per-workflow table), `errors.sh` (`explain_api_error` regex→hint table, `SG_API_HINTS`), `checklist.sh` (`create_secret_stubs` → SG secret `tfc-<wf>-<VAR>` = `CHANGE_ME`, referenced as `${secret::<name>}` via PATCH + payload patch; `write_checklist` → `export/post-import-checklist.md`). Parallel jobs (`run_parallel`) run in subshells, so `do_import`/`do_set_triggers` write `.import-result.<seg>.json` / `.triggers-result.<seg>.json` into the export dir and the caller merges them into state. - `scripts/tools.sh` — sourced by the other scripts (sets `SG_REPO_ROOT` to its parent dir). Provides `sg_resolve` (PATH-or-cache), `sg_ensure_{jq,hcl2json,yajsv,sgcli}` (pinned downloads into `.sg/cached/`; `sg-cli` is the Go binary), `sg_retry`, the color vars + log helpers. The Docker image bundles these tools at build time (`yajsv` built from source so arm64 works), so in-container runs never download. - **Variable sets** — `scripts/enrich_variable_sets.sh <tfc-org> <payload...>` (run by `cmd_enrich`, which reads `tfOrg`/`tfHostname` from `terraform.tfvars` via `hcl2json` — variable sets live in the **TFC** org, not `SG_ORG`). Lists all sets + vars via the TFC API (Bearer token from the `terraform login` creds file or `TFE_TOKEN`), computes per-workspace effective vars (scope: global/project/workspace; precedence: priority sets > workspace vars > non-priority sets), and merges them into each workflow (matched by workspace name parsed from `TfStateFilePath`). Sensitive set vars and key conflicts are reported. Skipped by `--no-variable-sets`. - **Workflow groups** — each TFC project maps to an SG workflow group `tfc-<project-segment>`, created via `POST /api/v1/orgs/{org}/wfgrps/` (auth `Authorization: apikey <token>`) if missing. (Both the API calls and sg-cli honor the undocumented `SG_BASE_URL` for non-prod targets.) Groups are addressed by name, so no generated ID is tracked. `.sg/workflow-groups.json` (gitignored, optional) overrides the target group per segment (`{"<segment>": "<existing-group>"}`); override groups are not auto-created. `--no-create-groups` requires all groups to pre-exist. diff --git a/README.md b/README.md index b47ab17..d823216 100644 --- a/README.md +++ b/README.md @@ -21,23 +21,25 @@ export TFE_TOKEN=<TFC/TFE token> # long-lived API token (User/Team/Org token export SG_API_TOKEN=<your SG token> export SG_ORG=<your SG org> -./sg-migrate.sh init # scaffolds terraform.tfvars -# edit transformer/terraform-cloud/terraform.tfvars (org, integrations, workspaceOverrides) - -./sg-migrate.sh all # apply -> enrich -> convert -> validate -> import (prompts before importing) +./sg-migrate.sh init # guided setup: picks TFC org/workspaces, SG connectors, runners -> terraform.tfvars +./sg-migrate.sh all # preflight -> apply -> enrich -> convert -> validate -> import (shows a plan, asks before importing) ``` -That's it — no workflow-group mapping to fill in. Each TFC project is imported into an SG workflow group named `tfc-<project>`, **created automatically via the API** if it doesn't exist. The import prompt shows each group as `exists` or `create` before anything is written. +That's it — no IDs to look up and no workflow-group mapping to fill in. `init` lists what the tokens can see (TFC organisations and workspaces, SG VCS/cloud connectors and runner groups) and writes `terraform.tfvars` from your picks; `all` verifies every reference **before** running terraform (preflight), prints a migration summary after the export, shows a per-workflow import plan (create/update, Terraform version, runner, triggers, secrets), and ends with a **post-import checklist** of what still needs a human. Each TFC project is imported into an SG workflow group named `tfc-<project>`, **created automatically via the API** if it doesn't exist. -- Single phase: `./sg-migrate.sh apply|enrich|convert|validate|import|triggers`. Running `./sg-migrate.sh` with no command prints the help menu. +- Single phase: `./sg-migrate.sh preflight|apply|enrich|convert|validate|import|triggers|checklist`. Running `./sg-migrate.sh` with no command prints the help menu. +- **Resume.** `all` remembers what it completed (`.sg/state.json`) and skips phases whose inputs have not changed, so after a failure you just re-run it; files already imported in full are skipped and files with failures are retried. `--fresh` redoes everything. +- **Scope.** `--project <segment>` and `--workspace <name>` (repeatable) limit every phase to a subset — migrate one team first, then the rest. +- **Dry run.** `./sg-migrate.sh import --dry-run` prints the per-workflow plan and stops; nothing is created. +- **Sensitive variables** (which TFC never exposes) are recreated as SG secrets with the value `CHANGE_ME` and referenced from the workflows as `${secret::<name>}`; the checklist lists each one to fill in. Opt out with `--no-secret-stubs`. - TFC **Variable Set** variables are merged into the payloads automatically (the `enrich` phase, via the TFC API); skip it with `--no-variable-sets`. -- `./sg-migrate.sh clean` removes local working artifacts (`export/`, Terraform state, tool cache) for a fresh start; add `--all` to also remove config. `clean` always runs locally. +- `./sg-migrate.sh clean` removes local working artifacts (`export/`, Terraform state, run state, tool cache) for a fresh start; add `--all` to also remove config. `clean` always runs locally. - **Override** a project's target group (to reuse an existing group) in `.sg/workflow-groups.json`: `{"<project-segment>": "<existing-group>"}`. Override groups must already exist (they're not auto-created). -- Output is concise by default (terraform's plan/init noise is hidden; shown on error). Add `-v`/`--verbose` for full output. -- Flags: `-y` skip the import prompt (CI), `--concurrency N` parallel jobs, `--org NAME`, `--no-create-groups` require groups to pre-exist, `--build` rebuild the image, `--native`/`--local` force a local run even when Docker is available. -- Tuning via env: `SG_RETRIES`, `SG_TF_PARALLELISM`, `SG_NATIVE=1`. +- Output is concise by default (terraform's plan/init noise is hidden; shown on error). Add `-v`/`--verbose` for full output. Known API errors come with a hint naming the `terraform.tfvars` field to fix. +- Flags: `-y` skip the import prompt (CI; also makes `init` non-interactive), `--concurrency N` parallel jobs, `--org NAME`, `--no-create-groups` require groups to pre-exist, `--skip-preflight`, `--build` rebuild the image, `--native`/`--local` force a local run even when Docker is available. +- Tuning via env: `SG_RETRIES`, `SG_TF_PARALLELISM`, `SG_NATIVE=1`, `SG_UI_URL` (base URL for the checklist's links, default `https://app.stackguardian.io`). - Tab completion for the current shell session: `source <(./sg-migrate.sh completion zsh)` (or `bash`). `init` prints this line for your shell. -- TFC auth: set `TFE_TOKEN` (recommended — a long-lived token avoids re-running `terraform login`); otherwise the `terraform login` credentials file is mounted read-only into the container. SG/TFC tokens are passed as env vars. +- TFC auth: set `TFE_TOKEN` (recommended — a long-lived token avoids re-running `terraform login`); otherwise the `terraform login` credentials file is mounted read-only into the container. Tokens are only ever read from the environment; `init` never writes them to disk. The manual, step-by-step flow below remains supported for fine-grained control and is what each phase runs under the hood (the helper scripts live in `scripts/`). @@ -162,7 +164,9 @@ To update workflows with different details, re-run the sg-cli command with the m - **Workflow groups.** Each TFC project imports into an SG workflow group `tfc-<project>`, created via the API if missing (disable with `--no-create-groups`). Override the target group per project in `.sg/workflow-groups.json`; override groups must already exist. - **Variable Sets are migrated** (the `enrich` phase) — global, project-, and workspace-scoped sets are resolved per workspace with TFC precedence (priority sets override workspace vars; otherwise workspace vars win). **Sensitive** set variables can't be read from the API, so they're skipped and reported — recreate them as StackGuardian secrets. -- **Sensitive variables are skipped.** TFC never returns sensitive values via the API, so they are omitted from the payload and listed in `migration-summary.md`. Recreate them as StackGuardian secrets. +- **Sensitive variables become placeholder secrets.** TFC never returns sensitive values via the API. The export omits them (listed in `migration-summary.md`); after import the orchestrator creates an SG secret `tfc-<workflow>-<VAR>` with the value `CHANGE_ME` for each, references it from the workflow (`${secret::<name>}`, as an environment variable or IaC input) and lists it in `export/post-import-checklist.md`. Set the real values in the SG UI. `--no-secret-stubs` leaves SG secrets untouched. - **Terraform version fallback (FOSS ceiling).** Workspaces set to `latest` or a version constraint use `SGDefaultTerraformVersion` at export time. Pinned versions are carried over as-is and tried first at import, so a custom runtime image or private runner that ships that binary keeps working. StackGuardian's _managed_ runtimes only go up to **1.5.7**, the last MPL-licensed (FOSS) Terraform release; newer versions are BSL-licensed and are not bundled. When the API rejects a workflow for that reason, the importer **automatically re-imports it with `SGDefaultTerraformVersion`**, patches the payload file to match, prints a notice, and records each case in `export/terraform-version-fallbacks.log`. Those workflows run a different Terraform than they did in TFC, so check compatibility before the first run. To keep a newer version, set `workspaceOverrides[<name>].terraformVersion` to a binary path mounted from a private runner (or use a custom runtime container template) and re-import. - **State export is idempotent.** Re-running `terraform apply` only pulls state for workspaces not yet exported. Set `forceStateRefresh = true` to re-pull everything. Workspaces not using `remote` execution may export incomplete state — see the summary. -- **Workflow naming.** `ResourceName` currently mirrors the TFC workspace name. Confirm it satisfies StackGuardian's naming rules before import. +- **Workflow naming.** `ResourceName` mirrors the TFC workspace name, sanitized to StackGuardian's rules (1-100 chars, `[-a-zA-Z0-9_]`); any rename is listed in the summary and the checklist. +- **Preflight.** `apply`, `import` and `all` first verify the TFC and SG tokens, the TFC org and workspace selection, and that every connector, secret and runner group referenced in `terraform.tfvars` exists. Fix what it reports (or re-run `init`); `--skip-preflight` bypasses it. +- **Post-import checklist.** `export/post-import-checklist.md` (also printed) collects the secrets to fill in, failed imports, Terraform version fallbacks, failed VCS triggers, missing state exports and renames, with links into the SG UI. diff --git a/scripts/migrate.sh b/scripts/migrate.sh index 2ad0bf1..48ad44d 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -77,11 +77,12 @@ Commands: triggers Register VCS triggers for already-imported workflows (second pass) checklist Create placeholder secrets for skipped sensitive variables and write export/post-import-checklist.md (runs automatically after import) - all apply -> enrich -> convert -> validate -> import + all preflight -> apply -> enrich -> convert -> validate -> import -> checklist + (resumes where a previous run stopped; --fresh to redo everything) clean Remove local working artifacts for a fresh start (export/, TF state, tool cache). Add --all to also remove config (terraform.tfvars, mapping). completion Print a shell completion script for the current session: - \`source <($0 completion zsh)\` (or bash) + \`source <($PROG completion zsh)\` (or bash) Each TFC project maps to an SG workflow group named tfc-<project>, created via the API if missing. Override a project's target group in .sg/workflow-groups.json From 5e25afe355749b82809668e2c5563427f7e2fb15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 7 Sep 2026 09:38:14 +0200 Subject: [PATCH 21/30] fix: read connector kind from Settings.kind in the init wizard The integrations list has no ResourceType; the kind (GITHUB_COM, AWS_RBAC, ...) is Settings.kind. Map it from there, make the type filters null-safe, and only treat msg/data as the item list when it is actually an array. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01YKBeQrpJgR9DVFtR83myqX --- scripts/lib/sg_api.sh | 9 +++++---- scripts/lib/wizard.sh | 4 ++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/scripts/lib/sg_api.sh b/scripts/lib/sg_api.sh index 9937a4b..28f24d4 100644 --- a/scripts/lib/sg_api.sh +++ b/scripts/lib/sg_api.sh @@ -81,7 +81,7 @@ sg_workflow_exists() { [ "$(sg_http_code GET "$(wf_url "$1" "$2")")" = "200" ]; sg_list_workflows() { local body if body="$(sg_api_get "$(sg_org_url)/wfgrps/$1/wfs/listall/" 2>/dev/null)"; then - printf '%s' "$body" | "$(sg_resolve jq sg_ensure_jq)" -c '[(.msg // .data // [])[] | .ResourceName] | map(select(. != null))' + printf '%s' "$body" | "$(sg_resolve jq sg_ensure_jq)" -c '[(if (.msg | type) == "array" then .msg elif (.data | type) == "array" then .data elif type == "array" then . else [] end)[] | .ResourceName] | map(select(. != null))' else echo '[]' fi @@ -93,10 +93,11 @@ sg_patch_workflow() { sg_api_patch "$(wf_url "$1" "$2")" "$3"; } # --- integrations (connectors) -------------------------------------------- # sg_list_integrations — [{name, type}, ...] for the org (fails on error). +# The connector kind (GITHUB_COM, AWS_RBAC, ...) is Settings.kind in the API. sg_list_integrations() { local body body="$(sg_api_get "$(sg_org_url)/integrations/listall/")" || return $? - printf '%s' "$body" | "$(sg_resolve jq sg_ensure_jq)" -c '[(.msg // .data // [])[] | {name: .ResourceName, type: .ResourceType}] | map(select(.name != null))' + printf '%s' "$body" | "$(sg_resolve jq sg_ensure_jq)" -c '[(if (.msg | type) == "array" then .msg elif (.data | type) == "array" then .data elif type == "array" then . else [] end)[] | {name: (.ResourceName // .Id // ""), type: (.Settings.kind // .kind // .ResourceType // "")}] | map(select(.name != ""))' } # sg_integration_exists <name-or-/integrations/name> — exit 0 when it exists. @@ -115,7 +116,7 @@ sg_runnergroup_exists() { [ "$(sg_http_code GET "$(sg_org_url)/runnergroups/$1/" sg_list_runnergroups() { local body body="$(sg_api_get "$(sg_org_url)/runnergroups/listall/" 2>/dev/null)" || return 1 - printf '%s' "$body" | "$(sg_resolve jq sg_ensure_jq)" -c '[(.msg // .data // [])[] | .ResourceName] | map(select(. != null))' + printf '%s' "$body" | "$(sg_resolve jq sg_ensure_jq)" -c '[(if (.msg | type) == "array" then .msg elif (.data | type) == "array" then .data elif type == "array" then . else [] end)[] | .ResourceName] | map(select(. != null))' } # --- secrets --------------------------------------------------------------- @@ -124,7 +125,7 @@ sg_list_runnergroups() { sg_secret_exists() { local body body="$(sg_api_get "$(sg_org_url)/secrets/listall/" 2>/dev/null)" || return 1 - printf '%s' "$body" | "$(sg_resolve jq sg_ensure_jq)" -e --arg n "$1" '[(.msg // .data // [])[] | .ResourceName] | index($n) != null' >/dev/null + printf '%s' "$body" | "$(sg_resolve jq sg_ensure_jq)" -e --arg n "$1" '[(if (.msg | type) == "array" then .msg elif (.data | type) == "array" then .data elif type == "array" then . else [] end)[] | .ResourceName] | index($n) != null' >/dev/null } # sg_create_secret <name> <value> [description] diff --git a/scripts/lib/wizard.sh b/scripts/lib/wizard.sh index 5096092..f388ac4 100644 --- a/scripts/lib/wizard.sh +++ b/scripts/lib/wizard.sh @@ -122,7 +122,7 @@ wizard_sg() { # VCS connector -> integration id, source kind, repo prefix. vcs="" - [ "$W_SG_DISCOVERY" -eq 1 ] && vcs="$(printf '%s' "$ints" | "$jqb" -r '.[] | select(.type | test("^(GITHUB_COM|GITHUB_APP_CUSTOM|GITLAB_COM|BITBUCKET_ORG|AZURE_DEVOPS|GIT_OTHER)$")) | "\(.name)|\(.type)"')" + [ "$W_SG_DISCOVERY" -eq 1 ] && vcs="$(printf '%s' "$ints" | "$jqb" -r '.[] | select((.type // "") | test("^(GITHUB_COM|GITHUB_APP_CUSTOM|GITLAB_COM|BITBUCKET_ORG|AZURE_DEVOPS|GIT_OTHER)$")) | "\(.name)|\(.type)"')" if [ -n "$vcs" ]; then # shellcheck disable=SC2046 pick="$(SG_SELECT_OTHER=1 sg_select "Which VCS connector should clone the repositories?" $(printf '%s\n' "$vcs" | tr '\n' ' '))" || return 1 @@ -142,7 +142,7 @@ wizard_sg() { # Cloud connector -> DeploymentPlatformConfig. cloud="" - [ "$W_SG_DISCOVERY" -eq 1 ] && cloud="$(printf '%s' "$ints" | "$jqb" -r '.[] | select(.type | test("^(AWS|AZURE|GCP)_")) | "\(.name)|\(.type)"')" + [ "$W_SG_DISCOVERY" -eq 1 ] && cloud="$(printf '%s' "$ints" | "$jqb" -r '.[] | select((.type // "") | test("^(AWS|AZURE|GCP)_")) | "\(.name)|\(.type)"')" if [ -n "$cloud" ]; then # shellcheck disable=SC2046 pick="$(SG_SELECT_OTHER=1 sg_select "Which cloud connector should the workflows deploy with?" $(printf '%s\n' "$cloud" | tr '\n' ' ') "skip|decide later (leaves a placeholder to edit)")" || return 1 From 0be0082b24ceb8277041595d69336365fbf7d520 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 7 Sep 2026 09:40:45 +0200 Subject: [PATCH 22/30] fix: init wrote invalid tfvars for an empty approver list _w_csv_json emitted "[]" twice when the list was empty (grep -v on empty input trips the fallback under pipefail), so the generated file was not valid HCL and preflight reported every field as missing. Validate the file right after writing it, report an unparsable tfvars as such in preflight, remember the SG org / API host chosen in the wizard so a new shell does not need SG_ORG, pretty-print objects in the generated file and separate menu values from their descriptions. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01YKBeQrpJgR9DVFtR83myqX --- scripts/lib/preflight.sh | 5 +++++ scripts/lib/prompt.sh | 2 +- scripts/lib/tfvars.sh | 15 +++++++++++++-- scripts/lib/wizard.sh | 13 +++++++++++-- scripts/migrate.sh | 9 +++++++++ 5 files changed, 39 insertions(+), 5 deletions(-) diff --git a/scripts/lib/preflight.sh b/scripts/lib/preflight.sh index c0bb2ad..4e8725f 100644 --- a/scripts/lib/preflight.sh +++ b/scripts/lib/preflight.sh @@ -190,6 +190,11 @@ preflight_run() { PF_FAIL=0 PF_WARN=0 PF_TFC_WORKSPACES="" + local parse_err + if ! parse_err="$(tfvars_valid)"; then + pf_fail "$(sg_rel "$TFVARS") is not valid HCL: ${parse_err:-parse error}" + die "fix the file (or re-run '$PROG init') and try again." + fi case "$ctx" in apply) preflight_tfc diff --git a/scripts/lib/prompt.sh b/scripts/lib/prompt.sh index 3ca5540..7e754e5 100644 --- a/scripts/lib/prompt.sh +++ b/scripts/lib/prompt.sh @@ -95,7 +95,7 @@ sg_select() { printf '%s? %s%s\n' "$C_BOLD" "$q" "$C_RESET" >&2 for ((i = 0; i < n; i++)); do if [ -n "${descs[i]}" ]; then - printf ' %s%2d)%s %s %s%s%s\n' "$C_CYAN" "$((i + 1))" "$C_RESET" "${vals[i]}" "$C_DIM" "${descs[i]}" "$C_RESET" >&2 + printf ' %s%2d)%s %-10s %s%s%s\n' "$C_CYAN" "$((i + 1))" "$C_RESET" "${vals[i]}" "$C_DIM" "— ${descs[i]}" "$C_RESET" >&2 else printf ' %s%2d)%s %s\n' "$C_CYAN" "$((i + 1))" "$C_RESET" "${vals[i]}" >&2 fi diff --git a/scripts/lib/tfvars.sh b/scripts/lib/tfvars.sh index 8bfc766..f37f8b6 100644 --- a/scripts/lib/tfvars.sh +++ b/scripts/lib/tfvars.sh @@ -35,6 +35,17 @@ tfvars_get_json() { # tfvars_invalidate — forget the cached conversion (after writing the file). tfvars_invalidate() { _TFVARS_JSON=""; } +# tfvars_valid — exit 0 when the file exists and hcl2json can parse it; +# the parser's message is printed on stdout otherwise. +tfvars_valid() { + [ -f "$TFVARS" ] || return 1 + # shellcheck disable=SC2069 # stderr is the result; stdout (the JSON) is discarded + "$(sg_resolve hcl2json sg_ensure_hcl2json)" "$TFVARS" 2>&1 >/dev/null +} + +# _tfvars_hcl <json> — pretty-print a JSON value so it reads like HCL in the file. +_tfvars_hcl() { printf '%s' "$1" | "$(sg_resolve jq sg_ensure_jq)" --indent 2 '.' 2>/dev/null || printf '%s' "$1"; } + # tfvars_write <dest> — render terraform.tfvars from W_* variables set by the # wizard (lists/objects are passed as compact JSON, which HCL accepts). Keeps # the same order and comments as terraform.tfvars.example so the file stays @@ -80,11 +91,11 @@ SGDefaultIACVCSRepoPrefix = "$W_REPO_PREFIX" SGDefaultVCSAuthIntegrationID = "$W_VCS_INTEGRATION" # Cloud connector the workflows deploy with -SGDefaultDeploymentPlatformConfig = $W_DPC_JSON +SGDefaultDeploymentPlatformConfig = $(_tfvars_hcl "$W_DPC_JSON") # Runners for every workflow: { type = "shared" } for SG-hosted runners, or # { type = "private", names = ["<runner-group>"] } for a private runner group. -SGDefaultRunnerConstraints = $W_RUNNER_JSON +SGDefaultRunnerConstraints = $(_tfvars_hcl "$W_RUNNER_JSON") # Choose from: GITHUB_COM, BITBUCKET_ORG, GITLAB_COM, AZURE_DEVOPS, GIT_OTHER SGDefaultSourceConfigDestKind = "$W_DEST_KIND" diff --git a/scripts/lib/wizard.sh b/scripts/lib/wizard.sh index f388ac4..edbbffe 100644 --- a/scripts/lib/wizard.sh +++ b/scripts/lib/wizard.sh @@ -10,9 +10,11 @@ _w_default() { local v; v="$(tfvars_get "$1")"; printf '%s' "${v:-$2}"; } # _w_csv_json <csv> — "a, b" -> ["a","b"]; empty -> []. _w_csv_json() { - local jqb + local jqb out jqb="$(sg_resolve jq sg_ensure_jq)" - printf '%s' "$1" | tr ',' '\n' | sed 's/^ *//; s/ *$//' | grep -v '^$' | "$jqb" -R . | "$jqb" -sc . 2>/dev/null || echo '[]' + # (grep -v exits 1 on empty input; never let that trigger the fallback twice) + out="$(printf '%s' "$1" | tr ',' '\n' | sed 's/^ *//; s/ *$//' | { grep -v '^$' || true; } | "$jqb" -R . | "$jqb" -sc . 2>/dev/null)" + printf '%s' "${out:-[]}" } # _w_repo_prefix_for <sourceConfigDestKind> — proposed repo URL prefix. @@ -226,5 +228,12 @@ wizard_run() { sg_log "previous file kept as $(sg_rel "$TFVARS.bak")" fi tfvars_write "$TFVARS" + if ! tfvars_valid; then + sg_err "the generated $(sg_rel "$TFVARS") is not valid HCL — this is a bug in the wizard; the file was kept for inspection" + return 1 + fi + # Remember the SG org (and API host) for later phases, so users don't have to + # export SG_ORG again in a new shell. Tokens are never stored. + state_update '.config = ((.config // {}) + {sg_org: $o, sg_base_url: $u})' --arg o "$ORG" --arg u "$SG_BASE_URL" sg_success "wrote $(sg_rel "$TFVARS")" } diff --git a/scripts/migrate.sh b/scripts/migrate.sh index 48ad44d..a7df492 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -30,6 +30,7 @@ source "$SCRIPT_DIR/lib/errors.sh" # shellcheck source=lib/checklist.sh source "$SCRIPT_DIR/lib/checklist.sh" + # SCRIPT_DIR holds the sibling scripts; SG_REPO_ROOT (from tools.sh) is the repo # root used for all repo-relative paths. TRANSFORMER_DIR="$SG_REPO_ROOT/transformer/terraform-cloud" @@ -38,6 +39,7 @@ PROG="${SG_PROG:-$0}" EXPORT_DIR="${SG_EXPORT_DIR:-$SG_REPO_ROOT/export}" MAPPING="${SG_WFGROUP_MAP:-$SG_REPO_ROOT/.sg/workflow-groups.json}" ORG="${SG_ORG:-}" +SG_BASE_URL_SET="${SG_BASE_URL:-}" SG_BASE_URL="${SG_BASE_URL:-https://api.app.stackguardian.io}" ASSUME_YES=0 PURGE=0 @@ -59,6 +61,13 @@ TF_PARALLELISM="${SG_TF_PARALLELISM:-20}" RETRIES="${SG_RETRIES:-4}" RETRY_BASE="${SG_RETRY_BASE:-2}" PF=() +# The init wizard remembers the SG org / API host in .sg/state.json so a new +# shell without SG_ORG still works; flags and env always win. +if [ -z "$ORG" ] && [ -f "$STATE_FILE" ]; then + ORG="$(state_read | "$(sg_resolve jq sg_ensure_jq)" -r '.config.sg_org // empty' 2>/dev/null || true)" + [ -n "$ORG" ] && [ -z "${SG_BASE_URL_SET:-}" ] && SG_BASE_URL="$(state_read | "$(sg_resolve jq sg_ensure_jq)" -r --arg d "$SG_BASE_URL" '.config.sg_base_url // $d' 2>/dev/null || echo "$SG_BASE_URL")" +fi + usage() { cat >&2 <<EOF From 1753eb771a870b58b4c92a35c9bb55c94add769b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 7 Sep 2026 09:45:41 +0200 Subject: [PATCH 23/30] fix: completion follows the shell you are actually running The hint used $SHELL, which is the login shell and was wrong for a bash login shell running zsh, so the bash script got sourced into zsh and broke on the first Tab. sg-migrate.sh now detects the parent shell and 'completion' with no argument prints the matching script; each script also hands off to the other shell's version when sourced by mistake. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01YKBeQrpJgR9DVFtR83myqX --- README.md | 2 +- scripts/migrate.sh | 29 ++++++++++++++++++++++------- sg-migrate.sh | 7 ++++++- 3 files changed, 29 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index d823216..2cd8050 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ That's it — no IDs to look up and no workflow-group mapping to fill in. `init` - Output is concise by default (terraform's plan/init noise is hidden; shown on error). Add `-v`/`--verbose` for full output. Known API errors come with a hint naming the `terraform.tfvars` field to fix. - Flags: `-y` skip the import prompt (CI; also makes `init` non-interactive), `--concurrency N` parallel jobs, `--org NAME`, `--no-create-groups` require groups to pre-exist, `--skip-preflight`, `--build` rebuild the image, `--native`/`--local` force a local run even when Docker is available. - Tuning via env: `SG_RETRIES`, `SG_TF_PARALLELISM`, `SG_NATIVE=1`, `SG_UI_URL` (base URL for the checklist's links, default `https://app.stackguardian.io`). -- Tab completion for the current shell session: `source <(./sg-migrate.sh completion zsh)` (or `bash`). `init` prints this line for your shell. +- Tab completion for the current shell session: `source <(./sg-migrate.sh completion)` (bash/zsh detected; or pass `bash`/`zsh`). `init` prints this line. - TFC auth: set `TFE_TOKEN` (recommended — a long-lived token avoids re-running `terraform login`); otherwise the `terraform login` credentials file is mounted read-only into the container. Tokens are only ever read from the environment; `init` never writes them to disk. The manual, step-by-step flow below remains supported for fine-grained control and is what each phase runs under the hood (the helper scripts live in `scripts/`). diff --git a/scripts/migrate.sh b/scripts/migrate.sh index a7df492..24ad72a 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -90,8 +90,8 @@ Commands: (resumes where a previous run stopped; --fresh to redo everything) clean Remove local working artifacts for a fresh start (export/, TF state, tool cache). Add --all to also remove config (terraform.tfvars, mapping). - completion Print a shell completion script for the current session: - \`source <($PROG completion zsh)\` (or bash) + completion Print a completion script for your shell (bash/zsh auto-detected): + \`source <($PROG completion)\` Each TFC project maps to an SG workflow group named tfc-<project>, created via the API if missing. Override a project's target group in .sg/workflow-groups.json @@ -250,9 +250,14 @@ cmd_init() { # A child process cannot register completions in the parent shell, so this only # prints the one-liner (nothing is written to the user's rc files). completion_hint() { - local sh - case "$(basename "${SHELL:-}")" in bash) sh=bash ;; *) sh=zsh ;; esac - sg_log "tab completion for this shell session: source <(./sg-migrate.sh completion $sh)" + sg_log "tab completion for this shell session: source <(./sg-migrate.sh completion)" +} + +# current_shell — bash|zsh: the shell the user is typing in (detected by +# sg-migrate.sh from its parent process), else the login shell. +current_shell() { + case "${SG_SHELL:-}" in bash | zsh) printf '%s' "$SG_SHELL"; return ;; esac + case "$(basename "${SHELL:-}")" in bash) printf 'bash' ;; *) printf 'zsh' ;; esac } # --- workflow-group mapping (API helpers live in lib/sg_api.sh) ------------- @@ -703,11 +708,16 @@ SG_OPTIONS="--org --export-dir --mapping --concurrency --no-create-groups --no-v # migrate.sh to stdout. Both shells fall back to the basename when the command # is invoked by path, so ./sg-migrate.sh completes too. cmd_completion() { - local shell="${1:-}" + local shell="${1:-$(current_shell)}" case "$shell" in bash) cat <<BASH # bash completion for sg-migrate.sh — generated by: sg-migrate.sh completion bash +# Sourced from zsh by mistake? Load the zsh version instead. +if [ -n "\${ZSH_VERSION:-}" ]; then + source <("$SG_REPO_ROOT/sg-migrate.sh" completion zsh) + return 0 +fi _sg_migrate() { local cur prev cmds opts w has_cmd=0 cur="\${COMP_WORDS[COMP_CWORD]}"; prev="\${COMP_WORDS[COMP_CWORD-1]}" @@ -735,6 +745,11 @@ BASH cat <<ZSH #compdef sg-migrate.sh migrate.sh # zsh completion for sg-migrate.sh — generated by: sg-migrate.sh completion zsh +# Sourced from bash by mistake? Load the bash version instead. +if [ -n "\${BASH_VERSION:-}" ]; then + source <("$SG_REPO_ROOT/sg-migrate.sh" completion bash) + return 0 +fi _sg_migrate() { local -a cmds cmds=( @@ -781,7 +796,7 @@ _sg_migrate() { compdef _sg_migrate sg-migrate.sh migrate.sh ZSH ;; - *) die "usage: $PROG completion <bash|zsh>" ;; + *) die "usage: $PROG completion [bash|zsh] (default: the shell you are running)" ;; esac } diff --git a/sg-migrate.sh b/sg-migrate.sh index 0d96281..8a6f500 100755 --- a/sg-migrate.sh +++ b/sg-migrate.sh @@ -40,6 +40,11 @@ done # Let migrate.sh print the name the user actually invoked in its help/hints. export SG_PROG="${0##*/}" case "$0" in */*) SG_PROG="./${0##*/}" ;; esac +# The shell the user is typing in (for the completion hint): the parent process, +# not $SHELL, which is only the login shell and is often wrong (bash vs zsh). +SG_SHELL="$(ps -p "$PPID" -o comm= 2>/dev/null | sed 's/^-//; s#.*/##')" +case "$SG_SHELL" in bash | zsh) ;; *) SG_SHELL="$(basename "${SHELL:-zsh}")" ;; esac +export SG_SHELL if [ "$NATIVE" = "1" ] || ! command -v docker >/dev/null 2>&1; then [ "$NATIVE" = "1" ] || sg_warn "docker not found; running natively" @@ -54,7 +59,7 @@ fi DOCKER_ARGS=(--rm -i -v "$SCRIPT_DIR:/app" -w /app -e SG_API_TOKEN -e SG_ORG -e SG_BASE_URL -e SG_CONCURRENCY -e SG_RETRIES -e SG_TF_PARALLELISM - -e TFE_TOKEN -e SG_PROG -e SG_NONINTERACTIVE -e SG_UI_URL) + -e TFE_TOKEN -e SG_PROG -e SG_SHELL -e SG_NONINTERACTIVE -e SG_UI_URL) # Interactive TTY only when attached to one (so the confirmation prompt works, # but CI/non-tty invocations still run — use -y there). From e2832336a9dc46fdb78873ffe105e5c36a90b7ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 7 Sep 2026 09:45:42 +0200 Subject: [PATCH 24/30] feat: SG_TFVARS override for the tfvars path Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01YKBeQrpJgR9DVFtR83myqX --- scripts/migrate.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/migrate.sh b/scripts/migrate.sh index 24ad72a..472f21b 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -34,7 +34,7 @@ source "$SCRIPT_DIR/lib/checklist.sh" # SCRIPT_DIR holds the sibling scripts; SG_REPO_ROOT (from tools.sh) is the repo # root used for all repo-relative paths. TRANSFORMER_DIR="$SG_REPO_ROOT/transformer/terraform-cloud" -TFVARS="$TRANSFORMER_DIR/terraform.tfvars" +TFVARS="${SG_TFVARS:-$TRANSFORMER_DIR/terraform.tfvars}" PROG="${SG_PROG:-$0}" EXPORT_DIR="${SG_EXPORT_DIR:-$SG_REPO_ROOT/export}" MAPPING="${SG_WFGROUP_MAP:-$SG_REPO_ROOT/.sg/workflow-groups.json}" From b148418e86ca26b1cddc4793ebf6e248b9fc8ea0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 7 Sep 2026 09:49:35 +0200 Subject: [PATCH 25/30] fix: import plan survives an unexpected listall shape; shorter init MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The per-workflow plan passed whatever sg_list_workflows returned to jq --argjson and crashed when the response was not a plain array. List helpers now always yield an array (also unwrapping data.Workflows) and the plan guards the value. The wizard no longer asks for the profile name (not required), the repo URL prefix, approvers or the fallback Terraform version — they take sensible defaults and are edited in terraform.tfvars. sg-migrate.sh says when it runs in Docker. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01YKBeQrpJgR9DVFtR83myqX --- scripts/lib/report.sh | 1 + scripts/lib/sg_api.sh | 7 +++---- scripts/lib/wizard.sh | 27 +++++++++++---------------- sg-migrate.sh | 1 + 4 files changed, 16 insertions(+), 20 deletions(-) diff --git a/scripts/lib/report.sh b/scripts/lib/report.sh index a670516..10b81db 100644 --- a/scripts/lib/report.sh +++ b/scripts/lib/report.sh @@ -61,6 +61,7 @@ show_import_plan() { seg="$(seg_of "$f")" grp="$(group_for "$seg")" existing="$(sg_list_workflows "$grp")" + printf '%s' "$existing" | "$JQ_BIN" -e 'type == "array"' >/dev/null 2>&1 || existing='[]' rows="$("$JQ_BIN" -r --argjson ex "$existing" --arg def "$SG_DEFAULT_TF_VERSION" --argjson ws "$(ws_filter_json)" \ --slurpfile sum "${summary:-/dev/null}" ' ($sum[0] // {}) as $S diff --git a/scripts/lib/sg_api.sh b/scripts/lib/sg_api.sh index 28f24d4..f165f6b 100644 --- a/scripts/lib/sg_api.sh +++ b/scripts/lib/sg_api.sh @@ -79,12 +79,11 @@ sg_workflow_exists() { [ "$(sg_http_code GET "$(wf_url "$1" "$2")")" = "200" ]; # sg_list_workflows <group> — ["wf-name", ...] in the group ([] on 404). sg_list_workflows() { - local body + local body out if body="$(sg_api_get "$(sg_org_url)/wfgrps/$1/wfs/listall/" 2>/dev/null)"; then - printf '%s' "$body" | "$(sg_resolve jq sg_ensure_jq)" -c '[(if (.msg | type) == "array" then .msg elif (.data | type) == "array" then .data elif type == "array" then . else [] end)[] | .ResourceName] | map(select(. != null))' - else - echo '[]' + out="$(printf '%s' "$body" | "$(sg_resolve jq sg_ensure_jq)" -c '[(if (.msg | type) == "array" then .msg elif (.data | type) == "array" then .data elif (.data.Workflows? | type) == "array" then .data.Workflows elif type == "array" then . else [] end)[] | (.ResourceName // .Id // empty)]' 2>/dev/null)" fi + printf '%s' "${out:-[]}" } # sg_patch_workflow <group> <wf> <json> — PATCH a workflow. diff --git a/scripts/lib/wizard.sh b/scripts/lib/wizard.sh index edbbffe..c6c20b2 100644 --- a/scripts/lib/wizard.sh +++ b/scripts/lib/wizard.sh @@ -140,7 +140,8 @@ wizard_sg() { GITHUB_COM | GITLAB_COM | BITBUCKET_ORG | AZURE_DEVOPS | GIT_OTHER) W_DEST_KIND="$kind" ;; *) W_DEST_KIND="$(sg_select "VCS provider kind" GITHUB_COM GITLAB_COM BITBUCKET_ORG AZURE_DEVOPS GIT_OTHER)" || return 1 ;; esac - W_REPO_PREFIX="$(sg_ask "Repository URL prefix" "$(_w_default .SGDefaultIACVCSRepoPrefix "$(_w_repo_prefix_for "$W_DEST_KIND")")")" || return 1 + # Repo URL prefix follows the connector kind; editable in terraform.tfvars. + W_REPO_PREFIX="$(_w_default .SGDefaultIACVCSRepoPrefix "$(_w_repo_prefix_for "$W_DEST_KIND")")" # Cloud connector -> DeploymentPlatformConfig. cloud="" @@ -154,12 +155,11 @@ wizard_sg() { kind="" fi if [ -z "$pick" ] || [ "$pick" = "skip" ]; then - W_DPC_JSON='[{"kind":"AWS_RBAC","config":{"integrationId":"/integrations/CHANGE_ME","profileName":"default"}}]' + W_DPC_JSON='[{"kind":"AWS_RBAC","config":{"integrationId":"/integrations/CHANGE_ME"}}]' W_DPC_PLACEHOLDER=1 else [ -n "$kind" ] || kind="$(sg_select "Connector kind" AWS_RBAC AWS_STATIC AWS_OIDC AZURE_STATIC AZURE_OIDC AZURE_MANAGED_ID_OIDC GCP_STATIC GCP_OIDC)" || return 1 - name="$(sg_ask "Profile name" "$(_w_default .SGDefaultDeploymentPlatformConfig[0].config.profileName default)")" || return 1 - W_DPC_JSON="$("$jqb" -nc --arg k "$kind" --arg i "/integrations/${pick#/integrations/}" --arg p "$name" '[{kind:$k, config:{integrationId:$i, profileName:$p}}]')" + W_DPC_JSON="$("$jqb" -nc --arg k "$kind" --arg i "/integrations/${pick#/integrations/}" '[{kind:$k, config:{integrationId:$i}}]')" W_DPC_PLACEHOLDER=0 fi @@ -187,19 +187,14 @@ wizard_sg() { # --- step 3: policy ------------------------------------------------------------ wizard_policy() { - local approvers sg_step "3/4 Workflow defaults" - approvers="$(sg_ask "Approver emails for plans (comma-separated, empty for none)" "$(tfvars_get_json .SGDefaultWfApprovers | "$(sg_resolve jq sg_ensure_jq)" -r 'if . == null then "" else join(", ") end')")" || return 1 - W_APPROVERS_JSON="$(_w_csv_json "$approvers")" + # Approvers, repo prefix and the fallback Terraform version are plain values + # with sensible defaults — edit them in terraform.tfvars if needed. + W_APPROVERS_JSON="$(tfvars_get_json .SGDefaultWfApprovers)" + [ "$W_APPROVERS_JSON" = "null" ] && W_APPROVERS_JSON='[]' + W_TF_VERSION="$(_w_default .SGDefaultTerraformVersion TERRAFORM-1.5.7)" if sg_confirm "Export Terraform state for each workspace?" "$([ "$(_w_default .exportStateFiles true)" = "false" ] && echo N || echo Y)"; then W_EXPORT_STATE=true; else W_EXPORT_STATE=false; fi if sg_confirm "Pre-configure VCS triggers (push / pull-request runs) from the TFC settings?" "$([ "$(_w_default .SGDefaultEnableVCSTriggers true)" = "false" ] && echo N || echo Y)"; then W_TRIGGERS=true; else W_TRIGGERS=false; fi - sg_dim "StackGuardian bundles managed Terraform only up to 1.5.7 (last MPL/FOSS release);" - sg_dim "workspaces pinned above it are imported with the fallback version below." - while :; do - W_TF_VERSION="$(sg_ask "Fallback Terraform version" "$(_w_default .SGDefaultTerraformVersion TERRAFORM-1.5.7)")" || return 1 - [[ "$W_TF_VERSION" =~ ^TERRAFORM-[0-9]+\.[0-9]+\.[0-9]+$ ]] && break - sg_warn "use the SG format, e.g. TERRAFORM-1.5.7" - done } # --- step 4: review + write ---------------------------------------------------- @@ -212,10 +207,10 @@ wizard_review() { row "VCS connector" "$W_VCS_INTEGRATION ($W_DEST_KIND, $W_REPO_PREFIX)" row "Cloud connector" "$W_DPC_JSON" row "Runners" "$W_RUNNER_JSON" - row "Approvers" "$W_APPROVERS_JSON" row "State export / triggers" "$W_EXPORT_STATE / $W_TRIGGERS" - row "Fallback Terraform" "$W_TF_VERSION${W_WS_ABOVE_CEILING:+ ($W_WS_ABOVE_CEILING workspace(s) above 1.5.7 will use it)}" + row "Fallback Terraform" "$W_TF_VERSION${W_WS_ABOVE_CEILING:+ ($W_WS_ABOVE_CEILING workspace(s) pinned above 1.5.7 (the last FOSS runtime SG bundles) will use it)}" [ "${W_DPC_PLACEHOLDER:-0}" -eq 1 ] && sg_warn "cloud connector left as a placeholder — edit SGDefaultDeploymentPlatformConfig in $(sg_rel "$TFVARS") before 'apply'" + sg_dim "approvers, repo URL prefix and the fallback version can be edited in $(sg_rel "$TFVARS")" sg_confirm "Write $(sg_rel "$TFVARS")?" Y } diff --git a/sg-migrate.sh b/sg-migrate.sh index 8a6f500..9374ceb 100755 --- a/sg-migrate.sh +++ b/sg-migrate.sh @@ -50,6 +50,7 @@ if [ "$NATIVE" = "1" ] || ! command -v docker >/dev/null 2>&1; then [ "$NATIVE" = "1" ] || sg_warn "docker not found; running natively" exec "$SCRIPT_DIR/scripts/migrate.sh" ${ARGS[@]+"${ARGS[@]}"} fi +sg_dim "running in Docker ($IMAGE); pass --native to run on this machine instead" if [ "$BUILD" = "1" ] || ! docker image inspect "$IMAGE" >/dev/null 2>&1; then sg_log "building image $IMAGE ..." From 4f3b86d7b56d25ad0b77cdb57e752bcae068ce6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 7 Sep 2026 09:54:07 +0200 Subject: [PATCH 26/30] fix: 'all' continues after the init wizard instead of stopping When terraform.tfvars is missing, all runs the wizard and then asks whether to continue right away or edit the file first; only the non-interactive template copy still stops, since it holds placeholders. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01YKBeQrpJgR9DVFtR83myqX --- scripts/migrate.sh | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/scripts/migrate.sh b/scripts/migrate.sh index 472f21b..9ceb4bf 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -883,8 +883,16 @@ preflight) checklist) cmd_checklist ;; all) if [ ! -f "$TFVARS" ]; then - cmd_init - die "Edit $(sg_rel "$TFVARS"), then re-run '$PROG all'." + cmd_init || exit 1 + if ! sg_interactive; then + # Template copy: it still contains placeholders. + die "Edit $(sg_rel "$TFVARS"), then re-run '$PROG all'." + fi + echo >&2 + if ! sg_confirm "Continue with the migration now? (No = edit $(sg_rel "$TFVARS") first, e.g. workspaceOverrides, then re-run '$PROG all')" Y; then + sg_log "edit $(sg_rel "$TFVARS"), then re-run '$PROG all'" + exit 0 + fi fi # Fail fast on everything the whole pipeline needs, before the (long) apply. export SG_API_TOKEN SG_BASE_URL From 329dc9b644f0ce3065c92f065d7a3bf0a0b73778 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 7 Sep 2026 09:56:12 +0200 Subject: [PATCH 27/30] fix: parse migrate.sh fully before running it Wrap argument parsing and dispatch in main so bash reads the whole script up front. The repo is bind-mounted into the container, and a file edited during a long run was read half-old, half-new, ending in "unexpected EOF while looking for matching quote" after the import had already succeeded. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01YKBeQrpJgR9DVFtR83myqX --- scripts/migrate.sh | 208 +++++++++++++++++++++++---------------------- 1 file changed, 108 insertions(+), 100 deletions(-) diff --git a/scripts/migrate.sh b/scripts/migrate.sh index 9ceb4bf..6fad061 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -800,109 +800,117 @@ ZSH esac } -CMD="" -while [ $# -gt 0 ]; do - case "$1" in - -y | --yes) ASSUME_YES=1 ;; - --org) - ORG="$2" - shift - ;; - --org=*) ORG="${1#*=}" ;; - --export-dir) - EXPORT_DIR="$2" - shift - ;; - --export-dir=*) EXPORT_DIR="${1#*=}" ;; - --mapping) - MAPPING="$2" - shift - ;; - --mapping=*) MAPPING="${1#*=}" ;; - --concurrency) - CONC="$2" - shift - ;; - --concurrency=*) CONC="${1#*=}" ;; - --no-create-groups) CREATE_GROUPS=0 ;; - --no-variable-sets) ENRICH_VARSETS=0 ;; - --no-vcs-triggers) VCS_TRIGGERS=0 ;; - --skip-preflight) export SKIP_PREFLIGHT=1 ;; - --fresh) FRESH=1 ;; - --dry-run) DRY_RUN=1 ;; - --no-secret-stubs) SECRET_STUBS=0 ;; - --project) - PROJECT_FILTER+=("$2") - shift - ;; - --project=*) PROJECT_FILTER+=("${1#*=}") ;; - --workspace) - WS_FILTER+=("$2") +# main — argument parsing and dispatch. Kept in a function so bash parses the +# whole file before running anything: the repo is bind-mounted into the +# container, and a script edited while it runs would otherwise be read +# half-old, half-new. +main() { + local CMD="" + while [ $# -gt 0 ]; do + case "$1" in + -y | --yes) ASSUME_YES=1 ;; + --org) + ORG="$2" + shift + ;; + --org=*) ORG="${1#*=}" ;; + --export-dir) + EXPORT_DIR="$2" + shift + ;; + --export-dir=*) EXPORT_DIR="${1#*=}" ;; + --mapping) + MAPPING="$2" + shift + ;; + --mapping=*) MAPPING="${1#*=}" ;; + --concurrency) + CONC="$2" + shift + ;; + --concurrency=*) CONC="${1#*=}" ;; + --no-create-groups) CREATE_GROUPS=0 ;; + --no-variable-sets) ENRICH_VARSETS=0 ;; + --no-vcs-triggers) VCS_TRIGGERS=0 ;; + --skip-preflight) export SKIP_PREFLIGHT=1 ;; + --fresh) FRESH=1 ;; + --dry-run) DRY_RUN=1 ;; + --no-secret-stubs) SECRET_STUBS=0 ;; + --project) + PROJECT_FILTER+=("$2") + shift + ;; + --project=*) PROJECT_FILTER+=("${1#*=}") ;; + --workspace) + WS_FILTER+=("$2") + shift + ;; + --workspace=*) WS_FILTER+=("${1#*=}") ;; + -v | --verbose) VERBOSE=1 ;; + --all) PURGE=1 ;; + -h | --help) + usage + exit 0 + ;; + init | apply | enrich | convert | validate | import | triggers | all | clean | preflight | checklist) CMD="$1" ;; + completion) + cmd_completion "${2:-}" + exit 0 + ;; + *) + echo "Unknown argument: $1" >&2 + usage + exit 1 + ;; + esac shift - ;; - --workspace=*) WS_FILTER+=("${1#*=}") ;; - -v | --verbose) VERBOSE=1 ;; - --all) PURGE=1 ;; - -h | --help) + done + # No command: show the help menu instead of running the whole pipeline. + if [ -z "$CMD" ]; then usage exit 0 + fi + export SG_VERBOSE="$VERBOSE" + + case "$CMD" in + init) cmd_init ;; + clean) cmd_clean ;; + apply) cmd_apply ;; + enrich) cmd_enrich ;; + convert) cmd_convert ;; + validate) cmd_validate ;; + import) cmd_import ;; + triggers) cmd_triggers ;; + preflight) + export SG_API_TOKEN SG_BASE_URL + cmd_preflight ;; - init | apply | enrich | convert | validate | import | triggers | all | clean | preflight | checklist) CMD="$1" ;; - completion) - cmd_completion "${2:-}" - exit 0 - ;; - *) - echo "Unknown argument: $1" >&2 - usage - exit 1 + checklist) cmd_checklist ;; + all) + if [ ! -f "$TFVARS" ]; then + cmd_init || exit 1 + if ! sg_interactive; then + # Template copy: it still contains placeholders. + die "Edit $(sg_rel "$TFVARS"), then re-run '$PROG all'." + fi + echo >&2 + if ! sg_confirm "Continue with the migration now? (No = edit $(sg_rel "$TFVARS") first, e.g. workspaceOverrides, then re-run '$PROG all')" Y; then + sg_log "edit $(sg_rel "$TFVARS"), then re-run '$PROG all'" + exit 0 + fi + fi + # Fail fast on everything the whole pipeline needs, before the (long) apply. + export SG_API_TOKEN SG_BASE_URL + preflight_run all + [ "$FRESH" -eq 1 ] && { state_reset; sg_log "--fresh: previous run state discarded"; } + JQ_BIN="$(sg_resolve jq sg_ensure_jq)" + run_phase apply "$(sg_sha "$(sg_sha_files "$TFVARS")|$(ws_filter_json)")" cmd_apply + if [ "$ENRICH_VARSETS" -eq 1 ]; then run_phase enrich "$(payload_sha)" cmd_enrich; fi + run_phase convert "$(payload_sha)" cmd_convert + run_phase validate "$(payload_sha)" cmd_validate + cmd_import ;; esac - shift -done -# No command: show the help menu instead of running the whole pipeline. -if [ -z "$CMD" ]; then - usage - exit 0 -fi -export SG_VERBOSE="$VERBOSE" - -case "$CMD" in -init) cmd_init ;; -clean) cmd_clean ;; -apply) cmd_apply ;; -enrich) cmd_enrich ;; -convert) cmd_convert ;; -validate) cmd_validate ;; -import) cmd_import ;; -triggers) cmd_triggers ;; -preflight) - export SG_API_TOKEN SG_BASE_URL - cmd_preflight - ;; -checklist) cmd_checklist ;; -all) - if [ ! -f "$TFVARS" ]; then - cmd_init || exit 1 - if ! sg_interactive; then - # Template copy: it still contains placeholders. - die "Edit $(sg_rel "$TFVARS"), then re-run '$PROG all'." - fi - echo >&2 - if ! sg_confirm "Continue with the migration now? (No = edit $(sg_rel "$TFVARS") first, e.g. workspaceOverrides, then re-run '$PROG all')" Y; then - sg_log "edit $(sg_rel "$TFVARS"), then re-run '$PROG all'" - exit 0 - fi - fi - # Fail fast on everything the whole pipeline needs, before the (long) apply. - export SG_API_TOKEN SG_BASE_URL - preflight_run all - [ "$FRESH" -eq 1 ] && { state_reset; sg_log "--fresh: previous run state discarded"; } - JQ_BIN="$(sg_resolve jq sg_ensure_jq)" - run_phase apply "$(sg_sha "$(sg_sha_files "$TFVARS")|$(ws_filter_json)")" cmd_apply - if [ "$ENRICH_VARSETS" -eq 1 ]; then run_phase enrich "$(payload_sha)" cmd_enrich; fi - run_phase convert "$(payload_sha)" cmd_convert - run_phase validate "$(payload_sha)" cmd_validate - cmd_import - ;; -esac +} + +main "$@" From 52e52373a69b7fddb6f1b6c6c56bf7d4a512b027 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 7 Sep 2026 10:07:55 +0200 Subject: [PATCH 28/30] feat: strip TFC-specific variables (ignoreVarPatterns) TFC_* / TFE_* variables (TFC_WORKSPACE_NAME, the TFC_AWS_* dynamic credential settings, ...) only mean something inside Terraform Cloud. A new ignoreVarPatterns input (default ["^TFC_", "^TFE_"]) drops matching terraform and env variables from the payloads, from workspaces and from variable sets, lists them in the migration summary and keeps them out of the sensitive-variable list so no secret stubs are created for them. The init wizard asks whether to strip them. Also fix the secrets link in the checklist (orgs/<org>?tab=secrets). Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01YKBeQrpJgR9DVFtR83myqX --- README.md | 1 + scripts/enrich_variable_sets.sh | 8 ++++++-- scripts/lib/checklist.sh | 2 +- scripts/lib/report.sh | 2 ++ scripts/lib/tfvars.sh | 6 +++++- scripts/lib/wizard.sh | 7 +++++++ transformer/terraform-cloud/locals.tf | 20 +++++++++++++++---- transformer/terraform-cloud/summary.tmpl | 10 ++++++++++ .../terraform-cloud/terraform.tfvars.example | 4 ++++ transformer/terraform-cloud/variables.tf | 6 ++++++ 10 files changed, 58 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 2cd8050..c762143 100644 --- a/README.md +++ b/README.md @@ -164,6 +164,7 @@ To update workflows with different details, re-run the sg-cli command with the m - **Workflow groups.** Each TFC project imports into an SG workflow group `tfc-<project>`, created via the API if missing (disable with `--no-create-groups`). Override the target group per project in `.sg/workflow-groups.json`; override groups must already exist. - **Variable Sets are migrated** (the `enrich` phase) — global, project-, and workspace-scoped sets are resolved per workspace with TFC precedence (priority sets override workspace vars; otherwise workspace vars win). **Sensitive** set variables can't be read from the API, so they're skipped and reported — recreate them as StackGuardian secrets. +- **TFC-specific variables are stripped.** Variables whose name matches `ignoreVarPatterns` (default `^TFC_`, `^TFE_`, e.g. `TFC_WORKSPACE_NAME` or the `TFC_AWS_*` dynamic-credential settings) only mean something inside Terraform Cloud and are not migrated, from workspaces or variable sets. They are listed in the migration summary; set `ignoreVarPatterns = []` to keep them. - **Sensitive variables become placeholder secrets.** TFC never returns sensitive values via the API. The export omits them (listed in `migration-summary.md`); after import the orchestrator creates an SG secret `tfc-<workflow>-<VAR>` with the value `CHANGE_ME` for each, references it from the workflow (`${secret::<name>}`, as an environment variable or IaC input) and lists it in `export/post-import-checklist.md`. Set the real values in the SG UI. `--no-secret-stubs` leaves SG secrets untouched. - **Terraform version fallback (FOSS ceiling).** Workspaces set to `latest` or a version constraint use `SGDefaultTerraformVersion` at export time. Pinned versions are carried over as-is and tried first at import, so a custom runtime image or private runner that ships that binary keeps working. StackGuardian's _managed_ runtimes only go up to **1.5.7**, the last MPL-licensed (FOSS) Terraform release; newer versions are BSL-licensed and are not bundled. When the API rejects a workflow for that reason, the importer **automatically re-imports it with `SGDefaultTerraformVersion`**, patches the payload file to match, prints a notice, and records each case in `export/terraform-version-fallbacks.log`. Those workflows run a different Terraform than they did in TFC, so check compatibility before the first run. To keep a newer version, set `workspaceOverrides[<name>].terraformVersion` to a binary path mounted from a private runner (or use a custom runtime container template) and re-import. - **State export is idempotent.** Re-running `terraform apply` only pulls state for workspaces not yet exported. Set `forceStateRefresh = true` to re-pull everything. Workspaces not using `remote` execution may export incomplete state — see the summary. diff --git a/scripts/enrich_variable_sets.sh b/scripts/enrich_variable_sets.sh index 167b04a..88fd77d 100755 --- a/scripts/enrich_variable_sets.sh +++ b/scripts/enrich_variable_sets.sh @@ -106,15 +106,19 @@ sg_log "resolving $set_count variable set(s) across workspaces..." ) ' >"$WORK/effective.json" +# TFC-specific variables are stripped here too (same patterns as the transformer). +IGNORE_JSON="$(tfvars_get_json .ignoreVarPatterns)" +[ "$IGNORE_JSON" = "null" ] && IGNORE_JSON='["^TFC_","^TFE_"]' + # Merge the effective set vars into each payload, then report counts. for f in "$@"; do before_tf="$("$JQ_BIN" '[.[].VCSConfig.iacInputData.data | length] | add // 0' "$f")" out="$WORK/merged.json" - "$JQ_BIN" --slurpfile eff "$WORK/effective.json" ' + "$JQ_BIN" --slurpfile eff "$WORK/effective.json" --argjson ignore "$IGNORE_JSON" ' ($eff[0]) as $E | map( ((.CLIConfiguration.TfStateFilePath // "") | sub(".*/"; "") | sub("\\.tfstate$"; "")) as $wsName - | ($E[$wsName] // []) as $all + | ($E[$wsName] // [] | map(select(.key as $k | [$ignore[] | . as $p | select($k | test($p))] | length == 0))) as $all | ($all | map(select(.sensitive != true and .category == "terraform"))) as $tf | ($all | map(select(.sensitive != true and .category == "env"))) as $env | .VCSConfig.iacInputData.data = ( diff --git a/scripts/lib/checklist.sh b/scripts/lib/checklist.sh index 748683c..0bb3c61 100644 --- a/scripts/lib/checklist.sh +++ b/scripts/lib/checklist.sh @@ -7,7 +7,7 @@ SG_UI_URL="${SG_UI_URL:-https://app.stackguardian.io}" wf_ui_url() { printf '%s/orchestrator/orgs/%s/wfgrps/%s/wfs/%s' "$SG_UI_URL" "$ORG" "$1" "$2"; } -secrets_ui_url() { printf '%s/orchestrator/orgs/%s/settings?tab=secrets' "$SG_UI_URL" "$ORG"; } +secrets_ui_url() { printf '%s/orchestrator/orgs/%s?tab=secrets' "$SG_UI_URL" "$ORG"; } # secret_name_for <workflow> <var> — SG secret name for a stubbed variable. secret_name_for() { printf 'tfc-%s-%s' "$1" "$2" | tr -c 'A-Za-z0-9_-\n' '-' | cut -c1-100; } diff --git a/scripts/lib/report.sh b/scripts/lib/report.sh index 10b81db..36d39f5 100644 --- a/scripts/lib/report.sh +++ b/scripts/lib/report.sh @@ -16,6 +16,8 @@ show_migration_summary() { _summary_section "$f" '.skippedSensitiveVars' "Sensitive variables skipped (TFC never exposes them)" \ 'to_entries[] | "\(.key): \(.value | join(", "))"' "recreated as SG secrets after import" + _summary_section "$f" '.strippedVars' "TFC-specific variables stripped (ignoreVarPatterns)" \ + 'to_entries[] | "\(.key): \(.value | join(", "))"' "" _summary_section "$f" '.terraformVersionFallbacks' "Terraform version not pinned (SGDefaultTerraformVersion used)" \ 'to_entries[] | "\(.key): \"\(.value)\""' "" _summary_section "$f" '.nonRemoteExecutionModes' "Non-remote execution mode (state may not be in TFC)" \ diff --git a/scripts/lib/tfvars.sh b/scripts/lib/tfvars.sh index f37f8b6..6ec7412 100644 --- a/scripts/lib/tfvars.sh +++ b/scripts/lib/tfvars.sh @@ -52,7 +52,7 @@ _tfvars_hcl() { printf '%s' "$1" | "$(sg_resolve jq sg_ensure_jq)" --indent 2 '. # hand-editable afterwards. # W_TFORG W_TFHOST W_WSNAMES_JSON W_TAGS_JSON W_IGNORE_TAGS_JSON W_EXPORT_STATE # W_APPROVERS_JSON W_REPO_PREFIX W_VCS_INTEGRATION W_DPC_JSON W_RUNNER_JSON -# W_DEST_KIND W_TF_VERSION W_TRIGGERS +# W_DEST_KIND W_TF_VERSION W_TRIGGERS W_IGNORE_PATTERNS_JSON tfvars_write() { local dest="$1" host_line="" if [ "${W_TFHOST:-app.terraform.io}" != "app.terraform.io" ]; then @@ -80,6 +80,10 @@ tfWorkspaceIgnoreTags = $W_IGNORE_TAGS_JSON # Directory to export Terraform files to exportPath = "export" +# TFC/TFE-specific variables that are not migrated (regexes on the variable +# name), e.g. TFC_WORKSPACE_NAME or TFC_AWS_RUN_ROLE_ARN. [] keeps everything. +ignoreVarPatterns = $W_IGNORE_PATTERNS_JSON + # Emails of the users who must approve plans (approvalPreApply is set for # workspaces without auto-apply) SGDefaultWfApprovers = $W_APPROVERS_JSON diff --git a/scripts/lib/wizard.sh b/scripts/lib/wizard.sh index c6c20b2..ba4a7a2 100644 --- a/scripts/lib/wizard.sh +++ b/scripts/lib/wizard.sh @@ -195,6 +195,12 @@ wizard_policy() { W_TF_VERSION="$(_w_default .SGDefaultTerraformVersion TERRAFORM-1.5.7)" if sg_confirm "Export Terraform state for each workspace?" "$([ "$(_w_default .exportStateFiles true)" = "false" ] && echo N || echo Y)"; then W_EXPORT_STATE=true; else W_EXPORT_STATE=false; fi if sg_confirm "Pre-configure VCS triggers (push / pull-request runs) from the TFC settings?" "$([ "$(_w_default .SGDefaultEnableVCSTriggers true)" = "false" ] && echo N || echo Y)"; then W_TRIGGERS=true; else W_TRIGGERS=false; fi + sg_dim "TFC_* / TFE_* variables (e.g. TFC_WORKSPACE_NAME, TFC_AWS_RUN_ROLE_ARN) only mean something inside Terraform Cloud." + if sg_confirm "Strip TFC-specific variables (TFC_*, TFE_*) from the migrated workflows?" "$([ "$(tfvars_get_json .ignoreVarPatterns)" = "[]" ] && echo N || echo Y)"; then + W_IGNORE_PATTERNS_JSON='["^TFC_","^TFE_"]' + else + W_IGNORE_PATTERNS_JSON='[]' + fi } # --- step 4: review + write ---------------------------------------------------- @@ -208,6 +214,7 @@ wizard_review() { row "Cloud connector" "$W_DPC_JSON" row "Runners" "$W_RUNNER_JSON" row "State export / triggers" "$W_EXPORT_STATE / $W_TRIGGERS" + row "Strip variables matching" "$W_IGNORE_PATTERNS_JSON" row "Fallback Terraform" "$W_TF_VERSION${W_WS_ABOVE_CEILING:+ ($W_WS_ABOVE_CEILING workspace(s) pinned above 1.5.7 (the last FOSS runtime SG bundles) will use it)}" [ "${W_DPC_PLACEHOLDER:-0}" -eq 1 ] && sg_warn "cloud connector left as a placeholder — edit SGDefaultDeploymentPlatformConfig in $(sg_rel "$TFVARS") before 'apply'" sg_dim "approvers, repo URL prefix and the fallback version can be edited in $(sg_rel "$TFVARS")" diff --git a/transformer/terraform-cloud/locals.tf b/transformer/terraform-cloud/locals.tf index d2ba8f0..5692f95 100644 --- a/transformer/terraform-cloud/locals.tf +++ b/transformer/terraform-cloud/locals.tf @@ -24,11 +24,21 @@ locals { name => length(local.sanitizedGroups[san]) > 1 ? "${length(san) > 93 ? substr(san, 0, 93) : san}-${substr(md5(name), 0, 6)}" : san } + # TFC-specific variables (TFC_*, TFE_* by default) are meaningless in SG and + # are stripped; recorded per workspace so the summary can list them. + strippedVars = { + for name, id in data.tfe_workspace_ids.data.ids : + name => [for v in data.tfe_variables.data[id].variables : "${v.category}:${v.name}" + if anytrue([for p in var.ignoreVarPatterns : can(regex(p, v.name))])] + } + # TFC never returns values for sensitive variables, so they cannot be - # migrated. Record them per workspace so the summary can flag them. + # migrated. Record them per workspace so the summary can flag them + # (stripped variables excluded — nobody needs a secret stub for those). sensitiveVars = { for name, id in data.tfe_workspace_ids.data.ids : - name => [for v in data.tfe_variables.data[id].variables : "${v.category}:${v.name}" if v.sensitive] + name => [for v in data.tfe_variables.data[id].variables : "${v.category}:${v.name}" + if v.sensitive && !anytrue([for p in var.ignoreVarPatterns : can(regex(p, v.name))])] } # Workspaces whose terraform_version is not a pinned semver (e.g. "latest" or @@ -72,7 +82,7 @@ locals { EnvironmentVariables = concat( [for v in data.tfe_variables.data[wsId].variables : { "config" : { "textValue" : v.value, "varName" : v.name }, "kind" : "PLAIN_TEXT" } - if v.category == "env" && v.sensitive == false], + if v.category == "env" && v.sensitive == false && !anytrue([for p in var.ignoreVarPatterns : can(regex(p, v.name))])], try(var.workspaceOverrides[wsName].extraEnvironmentVariables, []) ) @@ -96,7 +106,7 @@ locals { }, "iacInputData" : { "schemaType" : "RAW_JSON", - "data" : { for v in data.tfe_variables.data[wsId].variables : v.name => try(jsondecode(v.value), v.value) if v.category == "terraform" && v.sensitive == false } + "data" : { for v in data.tfe_variables.data[wsId].variables : v.name => try(jsondecode(v.value), v.value) if v.category == "terraform" && v.sensitive == false && !anytrue([for p in var.ignoreVarPatterns : can(regex(p, v.name))]) } } } @@ -192,6 +202,8 @@ locals { workspaceCount = length(local.workflowNames) projectWorkspaceCounts = { for pid in local.projectsUsed : try(local.projectNames[pid], pid) => length(local.payloadByProject[pid]) } skippedSensitiveVars = { for name, vars in local.sensitiveVars : name => vars if length(vars) > 0 } + strippedVars = { for name, vars in local.strippedVars : name => vars if length(vars) > 0 } + ignoreVarPatterns = var.ignoreVarPatterns terraformVersionFallbacks = local.versionFallbacks nonRemoteExecutionModes = local.nonRemoteModes renamedWorkspaces = { for name in local.workflowNames : name => local.resourceNames[name] if local.resourceNames[name] != name } diff --git a/transformer/terraform-cloud/summary.tmpl b/transformer/terraform-cloud/summary.tmpl index 8dc1ba4..7e7f811 100644 --- a/transformer/terraform-cloud/summary.tmpl +++ b/transformer/terraform-cloud/summary.tmpl @@ -18,6 +18,16 @@ These are not migrated (TFC never returns sensitive values). Recreate them as SG %{ endfor ~} %{ endif ~} +## Stripped TFC-specific variables +Not migrated because they only mean something inside Terraform Cloud (patterns: ${join(", ", summary.ignoreVarPatterns)}; set ignoreVarPatterns = [] to keep them). +%{ if length(summary.strippedVars) == 0 ~} +- None. +%{ else ~} +%{ for ws, vars in summary.strippedVars ~} +- ${ws}: ${join(", ", vars)} +%{ endfor ~} +%{ endif ~} + ## Terraform version fallbacks Workspaces whose version was not a pinned semver; the configured SGDefaultTerraformVersion was used. %{ if length(summary.terraformVersionFallbacks) == 0 ~} diff --git a/transformer/terraform-cloud/terraform.tfvars.example b/transformer/terraform-cloud/terraform.tfvars.example index 5b7143f..f588f18 100644 --- a/transformer/terraform-cloud/terraform.tfvars.example +++ b/transformer/terraform-cloud/terraform.tfvars.example @@ -16,6 +16,10 @@ tfWorkspaceIgnoreTags = null # Directory to export Terraform files to exportPath = "export" +# TFC/TFE-specific variables that are not migrated (regexes on the variable +# name), e.g. TFC_WORKSPACE_NAME or TFC_AWS_RUN_ROLE_ARN. [] keeps everything. +ignoreVarPatterns = ["^TFC_", "^TFE_"] + # Add emails of the users who should approve the terraform plan, since approvalPreApply is set to true SGDefaultWfApprovers = [] diff --git a/transformer/terraform-cloud/variables.tf b/transformer/terraform-cloud/variables.tf index 26d8299..a8a103d 100644 --- a/transformer/terraform-cloud/variables.tf +++ b/transformer/terraform-cloud/variables.tf @@ -39,6 +39,12 @@ variable "exportPath" { type = string } +variable "ignoreVarPatterns" { + default = ["^TFC_", "^TFE_"] + description = "Regexes (matched against the variable name) for TFC/TFE-specific variables that have no meaning outside Terraform Cloud and are not migrated, e.g. TFC_WORKSPACE_NAME or the TFC_AWS_* dynamic-credential settings. Applies to terraform and env variables, including variable-set variables. Set to [] to keep everything; stripped variables are listed in migration-summary.md." + type = list(string) +} + variable "SGDefaultWfApprovers" { default = [] description = "Add emails of the users who should approve the terraform plan, since approvalPreApply is set to true" From fb20f80522beed85c1ec083e214a4b90242fd478 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 7 Sep 2026 12:18:34 +0200 Subject: [PATCH 29/30] fix: cloud connector picker offered VCS connectors The "^(AWS|AZURE|GCP)_" filter also matched AZURE_DEVOPS, so an Azure DevOps VCS connector could be chosen as the cloud connector and only schema validation caught it. Both pickers now use the exact kind lists (VCS also accepts AZURE_DEVOPS_SP and GITLAB_OAUTH_SSH and maps them to the source kind), are sorted by name, and preflight rejects a DeploymentPlatformConfig kind outside the schema enum. The validator no longer prints each failure twice. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01YKBeQrpJgR9DVFtR83myqX --- scripts/lib/preflight.sh | 7 +++++++ scripts/lib/wizard.sh | 9 ++++++--- scripts/validate_payload.sh | 2 +- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/scripts/lib/preflight.sh b/scripts/lib/preflight.sh index 4e8725f..bed47c6 100644 --- a/scripts/lib/preflight.sh +++ b/scripts/lib/preflight.sh @@ -137,6 +137,13 @@ preflight_config() { GITHUB_COM | GITHUB_APP_CUSTOM | GIT_OTHER | INLINE | BITBUCKET_ORG | GITLAB_COM | AZURE_DEVOPS) pf_ok "SGDefaultSourceConfigDestKind = $v" ;; *) pf_fail "SGDefaultSourceConfigDestKind '$v' is not one of GITHUB_COM, GITLAB_COM, BITBUCKET_ORG, AZURE_DEVOPS, GIT_OTHER" ;; esac + while IFS= read -r v; do + [ -n "$v" ] || continue + case "$v" in + AWS_STATIC | AWS_RBAC | AWS_OIDC | AZURE_STATIC | AZURE_OIDC | AZURE_MANAGED_ID_OIDC | GCP_STATIC | GCP_OIDC) pf_ok "DeploymentPlatformConfig kind $v" ;; + *) pf_fail "DeploymentPlatformConfig kind '$v' is not a cloud connector kind (AWS_STATIC, AWS_RBAC, AWS_OIDC, AZURE_STATIC, AZURE_OIDC, AZURE_MANAGED_ID_OIDC, GCP_STATIC, GCP_OIDC) — a VCS connector was picked as the cloud connector?" ;; + esac + done < <(tfvars_json | "$jqb" -r '[ (.SGDefaultDeploymentPlatformConfig // [])[]?.kind, ((.workspaceOverrides // {}) | to_entries[]? | .value.DeploymentPlatformConfig // [] | .[]?.kind) ] | map(select(. != null)) | unique | .[]') v="$(tfvars_get .SGDefaultTerraformVersion TERRAFORM-1.5.7)" if [[ "$v" =~ ^TERRAFORM-([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then if [ "$(printf '%03d%03d%03d' "${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}" "${BASH_REMATCH[3]}")" -gt "001005007" ]; then diff --git a/scripts/lib/wizard.sh b/scripts/lib/wizard.sh index ba4a7a2..80bb2d8 100644 --- a/scripts/lib/wizard.sh +++ b/scripts/lib/wizard.sh @@ -124,7 +124,7 @@ wizard_sg() { # VCS connector -> integration id, source kind, repo prefix. vcs="" - [ "$W_SG_DISCOVERY" -eq 1 ] && vcs="$(printf '%s' "$ints" | "$jqb" -r '.[] | select((.type // "") | test("^(GITHUB_COM|GITHUB_APP_CUSTOM|GITLAB_COM|BITBUCKET_ORG|AZURE_DEVOPS|GIT_OTHER)$")) | "\(.name)|\(.type)"')" + [ "$W_SG_DISCOVERY" -eq 1 ] && vcs="$(printf '%s' "$ints" | "$jqb" -r '[.[] | select((.type // "") | IN("GITHUB_COM","GITHUB_APP_CUSTOM","GITLAB_COM","GITLAB_OAUTH_SSH","BITBUCKET_ORG","AZURE_DEVOPS","AZURE_DEVOPS_SP","GIT_OTHER"))] | sort_by(.name) | .[] | "\(.name)|\(.type)"')" if [ -n "$vcs" ]; then # shellcheck disable=SC2046 pick="$(SG_SELECT_OTHER=1 sg_select "Which VCS connector should clone the repositories?" $(printf '%s\n' "$vcs" | tr '\n' ' '))" || return 1 @@ -137,6 +137,8 @@ wizard_sg() { W_VCS_INTEGRATION="/integrations/${pick#/integrations/}" case "$kind" in GITHUB_APP_CUSTOM) W_DEST_KIND=GITHUB_COM ;; + GITLAB_OAUTH_SSH) W_DEST_KIND=GITLAB_COM ;; + AZURE_DEVOPS_SP) W_DEST_KIND=AZURE_DEVOPS ;; GITHUB_COM | GITLAB_COM | BITBUCKET_ORG | AZURE_DEVOPS | GIT_OTHER) W_DEST_KIND="$kind" ;; *) W_DEST_KIND="$(sg_select "VCS provider kind" GITHUB_COM GITLAB_COM BITBUCKET_ORG AZURE_DEVOPS GIT_OTHER)" || return 1 ;; esac @@ -145,7 +147,8 @@ wizard_sg() { # Cloud connector -> DeploymentPlatformConfig. cloud="" - [ "$W_SG_DISCOVERY" -eq 1 ] && cloud="$(printf '%s' "$ints" | "$jqb" -r '.[] | select((.type // "") | test("^(AWS|AZURE|GCP)_")) | "\(.name)|\(.type)"')" + # Only kinds DeploymentPlatformConfig accepts (AZURE_DEVOPS* are VCS connectors). + [ "$W_SG_DISCOVERY" -eq 1 ] && cloud="$(printf '%s' "$ints" | "$jqb" -r '[.[] | select((.type // "") | IN("AWS_STATIC","AWS_RBAC","AWS_OIDC","AZURE_STATIC","AZURE_OIDC","AZURE_MANAGED_ID_OIDC","GCP_STATIC","GCP_OIDC"))] | sort_by(.name) | .[] | "\(.name)|\(.type)"')" if [ -n "$cloud" ]; then # shellcheck disable=SC2046 pick="$(SG_SELECT_OTHER=1 sg_select "Which cloud connector should the workflows deploy with?" $(printf '%s\n' "$cloud" | tr '\n' ' ') "skip|decide later (leaves a placeholder to edit)")" || return 1 @@ -169,7 +172,7 @@ wizard_sg() { "private|a private runner group in your own network")" || return 1 if [ "$runner" = "private" ]; then groups="" - [ "$W_SG_DISCOVERY" -eq 1 ] && groups="$(sg_list_runnergroups 2>/dev/null | "$jqb" -r '.[]' 2>/dev/null || true)" + [ "$W_SG_DISCOVERY" -eq 1 ] && groups="$(sg_list_runnergroups 2>/dev/null | "$jqb" -r 'sort | .[]' 2>/dev/null || true)" if [ -n "$groups" ]; then # shellcheck disable=SC2046 name="$(SG_SELECT_OTHER=1 sg_select "Which runner group?" $(printf '%s\n' "$groups" | tr '\n' ' '))" || return 1 diff --git a/scripts/validate_payload.sh b/scripts/validate_payload.sh index 234638e..c5fbd4a 100755 --- a/scripts/validate_payload.sh +++ b/scripts/validate_payload.sh @@ -25,5 +25,5 @@ sg_log "validating $# file(s) against schema/sg-payload.schema.json" # Strip the repo-root prefix from its output for readable, relative paths # (pipefail off so the pipeline's status is sed's; yajsv's status via PIPESTATUS). set +o pipefail -"$YAJSV_BIN" -s "$SCHEMA" "$@" 2>&1 | sed "s#${SG_REPO_ROOT}/##g" +"$YAJSV_BIN" -s "$SCHEMA" "$@" 2>&1 | sed "s#${SG_REPO_ROOT}/##g" | awk '!seen[$0]++' exit "${PIPESTATUS[0]}" From cbd03dac347b4a05355e4b742040007a91e54a02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 7 Sep 2026 14:37:02 +0200 Subject: [PATCH 30/30] feat: execution preset support; cleaner logs and flow Presets: SGTerraformVersionSource (carry|preset) plus nullable SGDefaultTerraformVersion and SGDefaultRunnerConstraints. Keys left out of the payload are filled from the org's execution preset by the SG API at import. init offers the preset with the current value shown, preflight reports it, the plan marks 'preset (...)' cells, the checklist explains what changed, and a pin rejected above the 1.5.7 ceiling falls back to the preset when the fallback is null. Flow: numbered phases with timings, live progress lines for terraform and the parallel phases, plain-language preflight with connector/kind/prefix consistency checks, readable wizard review, per-file convert/validate lines, a condensed post-import checklist with group links, and a neutral message when the import is declined. Fixes: init kept the previous repo URL prefix when the connector switched provider (the prefix now comes from the TFC repositories); enrich re-ran on every 'all' because convert changed its recorded hash; quotes in connector names broke the generated tfvars; a RunnerConstraints override with names failed HCL type unification. --- .gitignore | 2 + CLAUDE.md | 12 +- README.md | 7 +- scripts/convert_hcl_to_json.sh | 13 +- scripts/enrich_variable_sets.sh | 12 +- scripts/lib/checklist.sh | 140 +++++--- scripts/lib/preflight.sh | 191 ++++++++--- scripts/lib/prompt.sh | 7 +- scripts/lib/report.sh | 132 ++++++-- scripts/lib/sg_api.sh | 78 +++++ scripts/lib/tfc_api.sh | 61 +++- scripts/lib/tfvars.sh | 63 +++- scripts/lib/wizard.sh | 298 +++++++++++++++--- scripts/migrate.sh | 288 ++++++++++++----- scripts/tools.sh | 59 ++++ scripts/validate_payload.sh | 21 +- .../terraform-cloud/example_payload.jsonc | 4 +- transformer/terraform-cloud/locals.tf | 66 +++- transformer/terraform-cloud/summary.tmpl | 18 +- .../terraform-cloud/terraform.tfvars.example | 23 +- transformer/terraform-cloud/variables.tf | 22 +- 21 files changed, 1192 insertions(+), 325 deletions(-) diff --git a/.gitignore b/.gitignore index 4e5a8c2..7819528 100644 --- a/.gitignore +++ b/.gitignore @@ -148,6 +148,8 @@ crash.log # control as they are data points which are potentially sensitive and subject # to change depending on the environment. *.tfvars +# The init wizard keeps the previous file as terraform.tfvars.bak +*.tfvars.bak # Ignore override files as they are usually used to override resources locally and so # are not checked in diff --git a/CLAUDE.md b/CLAUDE.md index ed6fc39..1a3d3f1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -15,8 +15,8 @@ The migration is a user-driven pipeline, not a single program: 1. **Extract + transform** — `transformer/terraform-cloud/` is a Terraform root module. `terraform apply` reads TFC/TFE workspaces and emits one `<exportPath>/sg-payload.<project>.json` per TFC project, a `migration-summary.md`/`.json` report, and per-workspace `.tfstate` files when `exportStateFiles` is true. 2. **Enrich (variable sets)** — `scripts/enrich_variable_sets.sh` merges TFC Variable Set variables into the payloads via the TFC API (the provider can't enumerate sets). Resolves global/project/workspace scope + TFC precedence per workspace; non-sensitive only. 3. **Tuning** — adjust per-workspace differences (integration IDs, VCS auth, runners, approvers, version) via the `workspaceOverrides` variable and re-apply; or hand-edit the payload files. `example_payload.jsonc` is the annotated field reference. -4. **HCL→JSON conversion** — `scripts/convert_hcl_to_json.sh` rewrites HCL-string variable values in each payload to real JSON. -5. **Validation** — `scripts/validate_payload.sh` checks each payload against `schema/sg-payload.schema.json` (downloads `yajsv`). +4. **HCL→JSON conversion** — `scripts/convert_hcl_to_json.sh` rewrites HCL-string variable values in each payload to real JSON (one result line per file; details with `-v`). +5. **Validation** — `scripts/validate_payload.sh` checks each payload against `schema/sg-payload.schema.json` (downloads `yajsv`; yajsv's lines are re-rendered as ✓/✗ per file, raw with `-v`). 6. **Import** — `sg-cli workflow create --bulk`, run once per project file, each into its own workflow group. ### Orchestration & tooling @@ -24,8 +24,8 @@ The migration is a user-driven pipeline, not a single program: The five phases are wrapped by an orchestrator so users don't run them by hand: - `sg-migrate.sh` (host entrypoint, repo root) — runs `scripts/migrate.sh` **inside the Docker image** (`Dockerfile`), bind-mounting the repo at `/app` and forwarding `SG_API_TOKEN`/`SG_ORG`/`SG_BASE_URL`/`TFE_TOKEN`/`TF_TOKEN_*`. For TFC auth it prefers a long-lived `TFE_TOKEN` (env); otherwise it mounts `~/.terraform.d/credentials.tfrc.json` read-only. Runs natively instead when `--native`/`--local` is passed, `SG_NATIVE=1` is set, the command is `clean`/`completion`/help (or no command is given), or Docker is absent. Exports `SG_PROG` so `migrate.sh` shows `./sg-migrate.sh` in its usage/hints. -- `scripts/migrate.sh` — the actual orchestrator (sources `scripts/lib/*.sh`, see below). Subcommands `init|preflight|apply|enrich|convert|validate|import|triggers|checklist|all|clean|completion` (no command prints the help menu; `enrich` runs in `all` unless `--no-variable-sets`). Flags beyond the basics: `--dry-run` (import: plan only), `--fresh` (ignore run state), `--project SEG` / `--workspace NAME` (repeatable filters; apply maps `--workspace` to `-var workspacenames=[...]`, import works on a jq-filtered temp copy), `--skip-preflight`, `--no-secret-stubs`. Resolves each tool via `sg_resolve` (PATH first — the image installs them — else `sg_ensure_*` cache). `apply`/`all` first run `require_tfc_auth`, which resolves the token the tfe provider will use (`TFE_TOKEN`, `TF_TOKEN_<host>`, or the `terraform login` file for `tfHostname`) and verifies it with `GET /api/v2/account/details`, failing fast on a missing or rejected (expired) credential. Runs `convert` and `import` in parallel (`--concurrency`, default 4), raises `terraform apply -parallelism`, and retries `sg-cli` imports with backoff (`sg_retry`, `SG_RETRIES`). Import resolves each project's workflow group, checks existence via the SG API, shows a plan (`exists`/`create`), prompts (skip with `-y`), **creates the missing `tfc-<project>` groups via the API**, then imports. sg-cli exits 0 even when individual workflows fail, so `do_import` parses its output: a workflow rejected as above SG's managed Terraform ceiling (1.5.7, last MPL/FOSS release) is re-imported with `SGDefaultTerraformVersion` (read from `terraform.tfvars`), the payload patched in place, and the case logged to `export/terraform-version-fallbacks.log` plus a printed notice; any other per-workflow failure fails the run. The trigger pass skips workflows that do not exist in SG, and `sg_api_post` returns 22 on 4xx so `sg_retry` (via `SG_NO_RETRY_RC`) does not retry definitive errors. `clean` removes local artifacts (`export/`, TF state, tool cache); `clean --all` also removes config. `completion bash|zsh` prints a completion script (`cmd_completion`; commands/options come from `SG_COMMANDS`/`SG_OPTIONS`, keep them in sync with the parser and the host flags); like `clean` it always runs natively. Output is concise by default — `apply` captures terraform's init/plan output and shows only progress + the final summary (or the full log on failure, `TF_IN_AUTOMATION=1`/`-no-color`); `-v`/`--verbose` (`SG_VERBOSE`) streams everything and un-gates the convert detail lines. Colored logging via `sg_step`/`sg_log`/`sg_success`/`sg_warn`/`sg_err`; paths shown relative via `sg_rel` (all auto-off when not a TTY / `NO_COLOR`). -- `scripts/lib/` — sourced libraries (a `lib/` gitignore rule is negated for this dir): `prompt.sh` (`sg_ask`/`sg_confirm`/`sg_select`; tty or `SG_ANSWERS_FILE` for tests; `SG_NONINTERACTIVE=1`/`-y`/no TTY → defaults), `tfvars.sh` (`tfvars_get` cached hcl2json reads, `tfvars_write` renders the wizard's `W_*` vars), `tfc_api.sh` (`tfc_http`/`tfc_get_all` paginated, `tfc_list_{orgs,projects,workspaces}`, `tfc_token` with provider precedence, `require_tfc_auth`), `sg_api.sh` (`_sg_api` 0/22/1 contract + `SG_HTTP_CODE`, `sg_list_integrations`, `sg_*_exists`, `sg_list_workflows`, `sg_create_secret`, `sg_patch_workflow`, `wfgroup_*`), `wizard.sh` (`wizard_run`: 4 steps TFC → SG → defaults → review), `preflight.sh` (`preflight_run <apply|import|all>`, once per process), `state.sh` (`.sg/state.json`: `phases.<name>.input_sha`, `import.<seg>.{imported,failed,tf_fallback}`, `triggers.<seg>`, `secrets.<name>`; `run_phase` in migrate.sh skips unchanged phases), `report.sh` (`show_migration_summary` after apply, `show_import_plan` per-workflow table), `errors.sh` (`explain_api_error` regex→hint table, `SG_API_HINTS`), `checklist.sh` (`create_secret_stubs` → SG secret `tfc-<wf>-<VAR>` = `CHANGE_ME`, referenced as `${secret::<name>}` via PATCH + payload patch; `write_checklist` → `export/post-import-checklist.md`). Parallel jobs (`run_parallel`) run in subshells, so `do_import`/`do_set_triggers` write `.import-result.<seg>.json` / `.triggers-result.<seg>.json` into the export dir and the caller merges them into state. +- `scripts/migrate.sh` — the actual orchestrator (sources `scripts/lib/*.sh`, see below). Subcommands `init|preflight|apply|enrich|convert|validate|import|triggers|checklist|all|clean|completion` (no command prints the help menu; `enrich` runs in `all` unless `--no-variable-sets`). Flags beyond the basics: `--dry-run` (import: plan only), `--fresh` (ignore run state), `--project SEG` / `--workspace NAME` (repeatable filters; apply maps `--workspace` to `-var workspacenames=[...]`, import works on a jq-filtered temp copy), `--skip-preflight`, `--no-secret-stubs`. Resolves each tool via `sg_resolve` (PATH first — the image installs them — else `sg_ensure_*` cache). `apply`/`all` first run `require_tfc_auth`, which resolves the token the tfe provider will use (`TFE_TOKEN`, `TF_TOKEN_<host>`, or the `terraform login` file for `tfHostname`) and verifies it with `GET /api/v2/account/details`, failing fast on a missing or rejected (expired) credential. Runs `convert` and `import` in parallel (`--concurrency`, default 4), raises `terraform apply -parallelism`, and retries `sg-cli` imports with backoff (`sg_retry`, `SG_RETRIES`). Import resolves each project's workflow group, checks existence via the SG API, shows a plan (`exists`/`create`), prompts (skip with `-y`), **creates the missing `tfc-<project>` groups via the API**, then imports. sg-cli exits 0 even when individual workflows fail, so `do_import` parses its output: a workflow rejected as above SG's managed Terraform ceiling (1.5.7, last MPL/FOSS release) is re-imported with `SGDefaultTerraformVersion` (read from `terraform.tfvars`; an explicit `null` there means the `terraformVersion` key is deleted instead, so the org's execution preset decides — `tfvars_is_null` tells an explicit null from a missing key), the payload patched in place, and the case logged to `export/terraform-version-fallbacks.log` plus a printed notice; any other per-workflow failure fails the run. `cmd_import` reads the org's execution preset (`sg_execution_preset`, `GET /orgs/{org}/` → `Settings.workflowDefaults`) and `preset_labels` (report.sh) turns it into the `preset (1.5.7)` / `preset (private:rg)` cells of the plan and the `SG_TF_FALLBACK_LABEL` used in messages. The trigger pass skips workflows that do not exist in SG, and `sg_api_post` returns 22 on 4xx so `sg_retry` (via `SG_NO_RETRY_RC`) does not retry definitive errors. `clean` removes local artifacts (`export/`, TF state, tool cache); `clean --all` also removes config. `completion bash|zsh` prints a completion script (`cmd_completion`; commands/options come from `SG_COMMANDS`/`SG_OPTIONS`, keep them in sync with the parser and the host flags); like `clean` it always runs natively. Output is concise by default — `apply` captures terraform's init/plan output and shows only progress + the final summary (or the full log on failure, `TF_IN_AUTOMATION=1`/`-no-color`); `-v`/`--verbose` (`SG_VERBOSE`) streams everything and un-gates the convert detail lines. Colored logging via `sg_step`/`sg_log`/`sg_success`/`sg_warn`/`sg_err`/`sg_row`; paths shown relative via `sg_rel` (all auto-off when not a TTY / `NO_COLOR`). Long steps show a live progress line (`sg_run_quiet` for terraform init/apply, `run_parallel <fn> <max> <label> <items>` for convert/import/triggers — animated only when `SG_ANIMATE=1`, i.e. colors are on; plain log lines otherwise) and report their duration (`sg_fmt_secs`). Phases start with `phase_begin` — inside `all` the header is numbered (`Phase 2/5: ...`, `PHASE_TOTAL` set in `main`) and `phase_took` gives the elapsed time; `run_parallel` prints a job's output as-is when it is a single line and adds the `── <file> ──` header only for longer or failed output (always with `-v`). The import confirmation goes through `sg_confirm` (so `SG_ANSWERS_FILE` works; a declined prompt is a neutral "cancelled" line, not an error) and `import`/`all` end with `finish_line` (outcome, total time, open checklist items from `CHECKLIST_OPEN`). A Ctrl-C is trapped to clear the progress line and print a resume hint. +- `scripts/lib/` — sourced libraries (a `lib/` gitignore rule is negated for this dir): `prompt.sh` (`sg_ask`/`sg_confirm`/`sg_select`; tty or `SG_ANSWERS_FILE` for tests; `SG_NONINTERACTIVE=1`/`-y`/no TTY → defaults), `tfvars.sh` (`tfvars_get` cached hcl2json reads, `tfvars_has`/`tfvars_is_null` to tell an explicit `null` from a missing key, `tfvars_write` renders the wizard's `W_*` vars — `W_TF_SOURCE`, and `W_TF_VERSION`/`W_RUNNER_JSON` may be the literal `null`; plain strings go through `_tfvars_str`, so quotes/backslashes in names stay valid HCL), `tfc_api.sh` (`tfc_http`/`tfc_get_all` paginated, `tfc_list_{orgs,projects,workspaces}` — workspaces include `vcs_provider`/`vcs_url`/`vcs_identifier` —, `tfc_select_workspaces` (the module's name-glob / include-tags(all) / exclude-tags filter, shared by wizard and preflight), `tfc_vcs_summary`/`tfc_vcs_kind_for`/`tfc_vcs_label_for`, `tfc_token` with provider precedence, `require_tfc_auth`), `sg_api.sh` (`_sg_api` 0/22/1 contract + `SG_HTTP_CODE`, `sg_list_integrations`, `sg_integration_type`, `sg_vcs_kind_of` (connector type → `sourceConfigDestKind`), `sg_execution_preset` + `sg_preset_{desc,runner_desc,version_desc,runner_provided}` (the org's execution preset and how to describe it), `sg_*_exists`, `sg_list_workflows`, `sg_create_secret`, `sg_patch_workflow`, `wfgroup_*`), `wizard.sh` (`wizard_run`: 4 steps TFC → SG → defaults → review; the TFC step keeps the workspace list and derives, via `tfc_vcs_summary`, which VCS provider the selected workspaces use and the repositories' URL prefix — the SG step lists matching connectors first and takes the repo URL prefix from TFC (never from a previous tfvars written for another provider kind), warns on a kind mismatch, reads the org's execution preset and offers it as the runner choice; the defaults step asks whether to carry TFC versions or use the preset (`SGTerraformVersionSource`) and, for carry, which fallback (`SGDefaultTerraformVersion` or the preset = null); the review shows human-readable rows plus the `tfc-<project>` groups; `_w_select_lines` feeds menus line-by-line so names with spaces/globs survive), `preflight.sh` (`preflight_run <apply|import|all>`, once per process; ✓ lines are plain language and name the connector's role/kind, ✗/! lines name the tfvars field to fix; besides existence it checks consistency — VCS kind vs. the connector's type (fail), repo URL prefix vs. where the TFC repositories live or the kind's well-known host (warn); it also reports the org's execution preset next to the version/runner policy and warns when `carry` meets a preset that mounts a runner-provided binary), `state.sh` (`.sg/state.json`: `phases.<name>.input_sha`, `import.<seg>.{imported,failed,tf_fallback}`, `triggers.<seg>`, `secrets.<name>`; `run_phase` in migrate.sh skips unchanged phases), `report.sh` (`show_migration_summary` after apply — incl. a state-files row —, `show_import_plan` per-workflow table with columns sized to their content), `errors.sh` (`explain_api_error` regex→hint table, `SG_API_HINTS`), `checklist.sh` (`create_secret_stubs` → SG secret `tfc-<wf>-<VAR>` = `CHANGE_ME`, referenced as `${secret::<name>}` via PATCH + payload patch; `write_checklist` → `export/post-import-checklist.md`, printed to the terminal as one ✓/! status line per section plus the imported groups with UI links — the file has the details; sets `CHECKLIST_OPEN`). Parallel jobs (`run_parallel`) run in subshells, so `do_import`/`do_set_triggers` write `.import-result.<seg>.json` / `.triggers-result.<seg>.json` into the export dir and the caller merges them into state. - `scripts/tools.sh` — sourced by the other scripts (sets `SG_REPO_ROOT` to its parent dir). Provides `sg_resolve` (PATH-or-cache), `sg_ensure_{jq,hcl2json,yajsv,sgcli}` (pinned downloads into `.sg/cached/`; `sg-cli` is the Go binary), `sg_retry`, the color vars + log helpers. The Docker image bundles these tools at build time (`yajsv` built from source so arm64 works), so in-container runs never download. - **Variable sets** — `scripts/enrich_variable_sets.sh <tfc-org> <payload...>` (run by `cmd_enrich`, which reads `tfOrg`/`tfHostname` from `terraform.tfvars` via `hcl2json` — variable sets live in the **TFC** org, not `SG_ORG`). Lists all sets + vars via the TFC API (Bearer token from the `terraform login` creds file or `TFE_TOKEN`), computes per-workspace effective vars (scope: global/project/workspace; precedence: priority sets > workspace vars > non-priority sets), and merges them into each workflow (matched by workspace name parsed from `TfStateFilePath`). Sensitive set vars and key conflicts are reported. Skipped by `--no-variable-sets`. - **Workflow groups** — each TFC project maps to an SG workflow group `tfc-<project-segment>`, created via `POST /api/v1/orgs/{org}/wfgrps/` (auth `Authorization: apikey <token>`) if missing. (Both the API calls and sg-cli honor the undocumented `SG_BASE_URL` for non-prod targets.) Groups are addressed by name, so no generated ID is tracked. `.sg/workflow-groups.json` (gitignored, optional) overrides the target group per segment (`{"<segment>": "<existing-group>"}`); override groups are not auto-created. `--no-create-groups` requires all groups to pre-exist. @@ -39,14 +39,14 @@ The whole transformation lives in `locals.tf` — there are no `outputs.tf`/`mai - TFC `terraform` (non-sensitive) variables → `VCSConfig.iacInputData.data` (kept as strings here; `try(jsondecode(...), v.value)` only decodes values that are already valid JSON). - TFC `env` (non-sensitive) variables → `EnvironmentVariables` as `PLAIN_TEXT`. - `auto_apply` → inverted into `approvalPreApply` / gated `Approvers`. - - `terraform_version` → `TERRAFORM-<version>` only for a pinned semver; otherwise `SGDefaultTerraformVersion`. + - `terraform_version` → with `SGTerraformVersionSource = "carry"` (default) `TERRAFORM-<version>` for a pinned semver, otherwise `SGDefaultTerraformVersion`; with `"preset"`, or when that default is `null`, the `terraformVersion` key is **left out** of `TerraformConfig` so the SG API fills it from the org's **execution preset** (`Settings.workflowDefaults`, UI: Settings → Runner groups → Execution presets; platform default: managed Terraform 1.5.7). The API only fills keys that are absent, never `null`/`""`, so the transformer filters nulls out (`local.tfVersion`). Likewise `RunnerConstraints` is omitted when `SGDefaultRunnerConstraints = null` (`local.runnerConstraints`, picked via a tuple because an override with `names` and a default without it are different object types). The summary records `terraformVersionSource`, `terraformVersionDefault`, `runnerConstraintsSource` and `tfcTerraformVersions` so later phases can explain what each workflow runs. - `project_id` → `CLIConfiguration.WorkflowGroup.name` = `tfc-<project-segment>` (matches the per-project filename and the group the importer creates/targets). - Sensitive variables (terraform + env) are skipped and recorded in the summary. - Per-workspace `var.workspaceOverrides[<name>]` fields take precedence over the `SGDefault*` values (resolved via `try(var.workspaceOverrides[name].<field>, null) != null ? ... : <default>`). - `local.resourceNames` sanitizes workspace names to a valid SG `ResourceName` (≤100 chars, `^[-a-zA-Z0-9_]+$`, collision-disambiguated). For normal TFC names this is a no-op; any actual rename is reported in the summary. This is the single place to adjust naming rules. - `resources.tf` — writes one `sg-payload.<project>.json` per project directly via `for_each` (no `mv`), plus `migration-summary.{md,json}`. When `exportStateFiles=true`, `null_resource.exportState` pulls each workspace's state **directly from the TFC/TFE API** (`GET /api/v2/workspaces/{id}/current-state-version` → `hosted-state-download-url`) via a `local-exec` `curl`/`jq` script — no `terraform init` or providers per workspace (avoids the plugin-cache concurrency bug and per-workspace provider downloads). The token is read at runtime from `~/.terraform.d/credentials.tfrc.json` (the `terraform login` file) or `TFE_TOKEN`, so it never enters TF state. Idempotent (keyed by workspace name/id; `forceStateRefresh` re-pulls), with per-workspace failures (no token / no state / download error) recorded in `state-export-failures.log` instead of aborting. `cmd_apply` ensures `jq`/`curl` are on PATH for the apply. - `summary.tmpl` — renders `local.summary` to `migration-summary.md`. -- `variables.tf` / `terraform.tfvars.example` — inputs. `SGDefault*` variables supply global defaults baked into every workflow (deployment platform, VCS auth, repo prefix, source kind, approvers, Terraform version, runner constraints — `SGDefaultRunnerConstraints`, validated to `shared` or `private`+`names`); `workspaceOverrides` overrides them per workspace; `forceStateRefresh` controls state re-pull. Requires Terraform `>= 1.3` (for `optional()` object attributes). +- `variables.tf` / `terraform.tfvars.example` — inputs. `SGDefault*` variables supply global defaults baked into every workflow (deployment platform, VCS auth, repo prefix, source kind, approvers, Terraform version, runner constraints — `SGDefaultRunnerConstraints`, validated to `shared` or `private`+`names`, or `null` to defer to the execution preset); `SGTerraformVersionSource` (`carry`|`preset`) and a nullable `SGDefaultTerraformVersion` choose between carrying TFC pins and leaving the version to the execution preset; `workspaceOverrides` overrides them per workspace; `forceStateRefresh` controls state re-pull. Requires Terraform `>= 1.3` (for `optional()` object attributes). ### `scripts/convert_hcl_to_json.sh` diff --git a/README.md b/README.md index c762143..8ada2c7 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ export SG_ORG=<your SG org> ./sg-migrate.sh all # preflight -> apply -> enrich -> convert -> validate -> import (shows a plan, asks before importing) ``` -That's it — no IDs to look up and no workflow-group mapping to fill in. `init` lists what the tokens can see (TFC organisations and workspaces, SG VCS/cloud connectors and runner groups) and writes `terraform.tfvars` from your picks; `all` verifies every reference **before** running terraform (preflight), prints a migration summary after the export, shows a per-workflow import plan (create/update, Terraform version, runner, triggers, secrets), and ends with a **post-import checklist** of what still needs a human. Each TFC project is imported into an SG workflow group named `tfc-<project>`, **created automatically via the API** if it doesn't exist. +That's it — no IDs to look up and no workflow-group mapping to fill in. `init` lists what the tokens can see (TFC organisations and workspaces, SG VCS/cloud connectors and runner groups, the org's execution preset) and writes `terraform.tfvars` from your picks — it also reads which VCS provider your TFC workspaces are connected to, lists the matching connectors first and takes the repository URL prefix from TFC. `all` verifies every reference **before** running terraform (preflight — including that the VCS connector, the VCS kind and the repo URL prefix agree with each other and with the TFC repositories), prints a migration summary after the export, shows a per-workflow import plan (create/update, Terraform version, runner, triggers, secrets), and ends with a **post-import checklist** of what still needs a human. Phases are numbered, long steps show a live progress line, and every phase reports how long it took. Each TFC project is imported into an SG workflow group named `tfc-<project>`, **created automatically via the API** if it doesn't exist. - Single phase: `./sg-migrate.sh preflight|apply|enrich|convert|validate|import|triggers|checklist`. Running `./sg-migrate.sh` with no command prints the help menu. - **Resume.** `all` remembers what it completed (`.sg/state.json`) and skips phases whose inputs have not changed, so after a failure you just re-run it; files already imported in full are skipped and files with failures are retried. `--fresh` redoes everything. @@ -166,8 +166,9 @@ To update workflows with different details, re-run the sg-cli command with the m - **Variable Sets are migrated** (the `enrich` phase) — global, project-, and workspace-scoped sets are resolved per workspace with TFC precedence (priority sets override workspace vars; otherwise workspace vars win). **Sensitive** set variables can't be read from the API, so they're skipped and reported — recreate them as StackGuardian secrets. - **TFC-specific variables are stripped.** Variables whose name matches `ignoreVarPatterns` (default `^TFC_`, `^TFE_`, e.g. `TFC_WORKSPACE_NAME` or the `TFC_AWS_*` dynamic-credential settings) only mean something inside Terraform Cloud and are not migrated, from workspaces or variable sets. They are listed in the migration summary; set `ignoreVarPatterns = []` to keep them. - **Sensitive variables become placeholder secrets.** TFC never returns sensitive values via the API. The export omits them (listed in `migration-summary.md`); after import the orchestrator creates an SG secret `tfc-<workflow>-<VAR>` with the value `CHANGE_ME` for each, references it from the workflow (`${secret::<name>}`, as an environment variable or IaC input) and lists it in `export/post-import-checklist.md`. Set the real values in the SG UI. `--no-secret-stubs` leaves SG secrets untouched. -- **Terraform version fallback (FOSS ceiling).** Workspaces set to `latest` or a version constraint use `SGDefaultTerraformVersion` at export time. Pinned versions are carried over as-is and tried first at import, so a custom runtime image or private runner that ships that binary keeps working. StackGuardian's _managed_ runtimes only go up to **1.5.7**, the last MPL-licensed (FOSS) Terraform release; newer versions are BSL-licensed and are not bundled. When the API rejects a workflow for that reason, the importer **automatically re-imports it with `SGDefaultTerraformVersion`**, patches the payload file to match, prints a notice, and records each case in `export/terraform-version-fallbacks.log`. Those workflows run a different Terraform than they did in TFC, so check compatibility before the first run. To keep a newer version, set `workspaceOverrides[<name>].terraformVersion` to a binary path mounted from a private runner (or use a custom runtime container template) and re-import. +- **Terraform version fallback (FOSS ceiling).** With `SGTerraformVersionSource = "carry"` (the default) pinned versions are carried over as-is and tried first at import, so a custom runtime image or private runner that ships that binary keeps working; workspaces set to `latest` or a version constraint use `SGDefaultTerraformVersion` at export time. StackGuardian's _managed_ runtimes only go up to **1.5.7**, the last MPL-licensed (FOSS) Terraform release; newer versions are BSL-licensed and are not bundled. When the API rejects a workflow for that reason, the importer **automatically re-imports it with `SGDefaultTerraformVersion`** (or, when that is `null`, without any version so the execution preset decides — see the next bullet), patches the payload file to match, prints a notice, and records each case in `export/terraform-version-fallbacks.log`. Those workflows run a different Terraform than they did in TFC, so check compatibility before the first run. To keep a newer version, set `workspaceOverrides[<name>].terraformVersion` to a binary path mounted from a private runner (or use a custom runtime container template) and re-import. +- **Execution presets.** StackGuardian org admins can define an execution preset (Settings → Runner groups → Execution presets): default runner constraints plus a Terraform and an OpenTofu configuration that the API applies to any new workflow whose payload does not carry those fields. To lean on it, set `SGTerraformVersionSource = "preset"` (no version is sent at all) and/or `SGDefaultRunnerConstraints = null` (no runner constraints are sent); `SGDefaultTerraformVersion = null` keeps carrying TFC pins but hands the unpinned and rejected ones to the preset. `init` offers these choices with the org's current preset shown inline, preflight prints what the preset would supply, and the import plan marks such cells as `preset (…)`. The preset's custom runtime image or runner-provided binary is inherited in every mode, since the migrator never sets those keys. - **State export is idempotent.** Re-running `terraform apply` only pulls state for workspaces not yet exported. Set `forceStateRefresh = true` to re-pull everything. Workspaces not using `remote` execution may export incomplete state — see the summary. - **Workflow naming.** `ResourceName` mirrors the TFC workspace name, sanitized to StackGuardian's rules (1-100 chars, `[-a-zA-Z0-9_]`); any rename is listed in the summary and the checklist. - **Preflight.** `apply`, `import` and `all` first verify the TFC and SG tokens, the TFC org and workspace selection, and that every connector, secret and runner group referenced in `terraform.tfvars` exists. Fix what it reports (or re-run `init`); `--skip-preflight` bypasses it. -- **Post-import checklist.** `export/post-import-checklist.md` (also printed) collects the secrets to fill in, failed imports, Terraform version fallbacks, failed VCS triggers, missing state exports and renames, with links into the SG UI. +- **Post-import checklist.** `export/post-import-checklist.md` collects the secrets to fill in, failed imports, Terraform version fallbacks, failed VCS triggers, missing state exports and renames, with links into the SG UI. The terminal shows one status line per section (plus links to the imported workflow groups); the file has the details. diff --git a/scripts/convert_hcl_to_json.sh b/scripts/convert_hcl_to_json.sh index cb778f5..db51ae4 100755 --- a/scripts/convert_hcl_to_json.sh +++ b/scripts/convert_hcl_to_json.sh @@ -41,8 +41,11 @@ tmpfile="$WORKDIR/updated.ndjson" : >"$tmpfile" JSON_PATH=".VCSConfig.iacInputData.data" +converted=0 +touched=0 for ((i = 0; i < length; i++)); do + wf_converted=0 # Extract ith object obj=$($JQ_BIN ".[$i]" <<<"$json_data") @@ -86,6 +89,8 @@ for ((i = 0; i < length; i++)); do if [[ -n "$parsed" && "$parsed" != "null" ]]; then log " workflow $((i + 1)): converted '$key' from HCL to JSON" new_val=$($JQ_BIN --arg k "$key" --argjson v "$parsed" '. + {($k): $v}' <<<"$new_val") + converted=$((converted + 1)) + wf_converted=1 else sg_warn "$(sg_rel "$INPUT_FILE_JSON") workflow $((i + 1)): could not parse '$key' as HCL; keeping original value" fi @@ -93,6 +98,7 @@ for ((i = 0; i < length; i++)); do # Assign the converted data back at JSON_PATH updated_obj=$($JQ_BIN --argjson nv "$new_val" "$JSON_PATH = \$nv" <<<"$obj") + touched=$((touched + wf_converted)) echo "$updated_obj" >>"$tmpfile" done @@ -102,4 +108,9 @@ done outfile="$WORKDIR/output.json" $JQ_BIN -s '.' "$tmpfile" >"$outfile" mv "$outfile" "$INPUT_FILE_JSON" -log "Done. Updated $(sg_rel "$INPUT_FILE_JSON") in place." +# One result line per file (the orchestrator shows it as-is). +if [ "$converted" -gt 0 ]; then + sg_log "$(basename "$INPUT_FILE_JSON"): $converted HCL value(s) converted to JSON in $touched of $length workflow(s)" +else + sg_log "$(basename "$INPUT_FILE_JSON"): nothing to convert ($length workflow(s), values already JSON)" +fi diff --git a/scripts/enrich_variable_sets.sh b/scripts/enrich_variable_sets.sh index 88fd77d..8a653a2 100755 --- a/scripts/enrich_variable_sets.sh +++ b/scripts/enrich_variable_sets.sh @@ -78,7 +78,9 @@ if [ "$set_count" -eq 0 ]; then sg_log "no variable sets found; nothing to enrich" exit 0 fi -sg_log "resolving $set_count variable set(s) across workspaces..." +sg_log "resolving $set_count variable set(s) across workspaces:" +# One line per set: name, scope, size — so it is clear where merged vars come from. +"$JQ_BIN" -r '.[] | " - \(.name): \(if .global then "global" else ([(if (.projids | length) > 0 then "\(.projids | length) project(s)" else empty end), (if (.wsids | length) > 0 then "\(.wsids | length) workspace(s)" else empty end)] | if length == 0 then "unassigned" else join(", ") end) end), \(.vars | length) var(s)\(if .priority then ", priority" else "" end)"' "$WORK/sets.json" >&2 # Per workspace -> list of winning vars (set-vs-set precedence resolved; tagged # with priority + sensitive + conflict). rank: non-priority global/proj/ws = 1/2/3, @@ -113,6 +115,7 @@ IGNORE_JSON="$(tfvars_get_json .ignoreVarPatterns)" # Merge the effective set vars into each payload, then report counts. for f in "$@"; do before_tf="$("$JQ_BIN" '[.[].VCSConfig.iacInputData.data | length] | add // 0' "$f")" + before_env="$("$JQ_BIN" '[.[].EnvironmentVariables | length] | add // 0' "$f")" out="$WORK/merged.json" "$JQ_BIN" --slurpfile eff "$WORK/effective.json" --argjson ignore "$IGNORE_JSON" ' ($eff[0]) as $E @@ -136,7 +139,12 @@ for f in "$@"; do ) ' "$f" >"$out" && mv "$out" "$f" after_tf="$("$JQ_BIN" '[.[].VCSConfig.iacInputData.data | length] | add // 0' "$f")" - sg_log "$(basename "$f"): +$((after_tf - before_tf)) terraform var(s) from variable sets" + after_env="$("$JQ_BIN" '[.[].EnvironmentVariables | length] | add // 0' "$f")" + if [ "$((after_tf - before_tf + after_env - before_env))" -eq 0 ]; then + sg_log "$(basename "$f"): no new variables (sets only override or add nothing here)" + else + sg_log "$(basename "$f"): +$((after_tf - before_tf)) terraform, +$((after_env - before_env)) env var(s) from variable sets" + fi done # Report sensitive set vars (cannot be migrated) and key conflicts. diff --git a/scripts/lib/checklist.sh b/scripts/lib/checklist.sh index 0bb3c61..2bfa9e1 100644 --- a/scripts/lib/checklist.sh +++ b/scripts/lib/checklist.sh @@ -7,7 +7,12 @@ SG_UI_URL="${SG_UI_URL:-https://app.stackguardian.io}" wf_ui_url() { printf '%s/orchestrator/orgs/%s/wfgrps/%s/wfs/%s' "$SG_UI_URL" "$ORG" "$1" "$2"; } +wfgrp_ui_url() { printf '%s/orchestrator/orgs/%s/wfgrps/%s' "$SG_UI_URL" "$ORG" "$1"; } secrets_ui_url() { printf '%s/orchestrator/orgs/%s?tab=secrets' "$SG_UI_URL" "$ORG"; } +# Set by write_checklist: how many checklist items still need a human (read by +# finish_line in migrate.sh). +# shellcheck disable=SC2034 +CHECKLIST_OPEN=0 # secret_name_for <workflow> <var> — SG secret name for a stubbed variable. secret_name_for() { printf 'tfc-%s-%s' "$1" "$2" | tr -c 'A-Za-z0-9_-\n' '-' | cut -c1-100; } @@ -84,73 +89,116 @@ create_secret_stubs() { return 0 } -# write_checklist — render export/post-import-checklist.md and print it. +# _cl_count <text> — number of non-empty lines in a block of items. +_cl_count() { + if [ -z "$1" ]; then printf '0'; else printf '%s\n' "$1" | grep -c . || true; fi +} + +# _cl_status <count> <ok-text> <attention-text> — one terminal line per section. +_cl_status() { + if [ "${1:-0}" -gt 0 ]; then printf ' %s!%s %s\n' "$C_YELLOW" "$C_RESET" "$3" >&2 + else printf ' %s✓%s %s\n' "$C_GREEN" "$C_RESET" "$2" >&2; fi +} + +# _cl_block <items> — the markdown list for a section, or "- None." +_cl_block() { if [ -n "$1" ]; then printf '%s\n\n' "$1"; else printf -- '- None.\n\n'; fi; } + +# write_checklist — render export/post-import-checklist.md, then show what +# landed and which sections still need a human (the file has the details and +# links; the terminal gets one status line per section). Sets CHECKLIST_OPEN. write_checklist() { - local out="$EXPORT_DIR/post-import-checklist.md" summary="$EXPORT_DIR/migration-summary.json" st items + local out="$EXPORT_DIR/post-import-checklist.md" summary="$EXPORT_DIR/migration-summary.json" st + local i_secrets i_unstubbed="" i_failed i_fallback="" i_unpinned="" i_preset="" i_trig i_state="" i_nonremote="" i_renamed="" + local n_secrets n_unstubbed n_failed n_fallback n_unpinned n_preset n_trig n_state n_nonremote n_renamed + local f seg grp n total=0 gw preset_desc="" + local -a glines=() st="$(state_read)" + [ -n "${SG_PRESET_JSON:-}" ] && preset_desc="$(sg_preset_desc "$SG_PRESET_JSON")" + + # --- collect the items once (markdown lines) -------------------------------- + i_secrets="$(printf '%s' "$st" | "$JQ_BIN" -r --arg ui "$SG_UI_URL" --arg org "$ORG" '.secrets // {} | to_entries[] | "- [ ] `\(.key)` — \(.value.category) var `\(.value.var)` of [\(.value.group)/\(.value.workflow)](\($ui)/orchestrator/orgs/\($org)/wfgrps/\(.value.group)/wfs/\(.value.workflow))"')" + if [ -f "$summary" ]; then + i_unstubbed="$("$JQ_BIN" -r --argjson stubbed "$(printf '%s' "$st" | "$JQ_BIN" -c '[.secrets // {} | .[] | "\(.workspace)|\(.category):\(.var)"]')" \ + '.skippedSensitiveVars | to_entries[] | .key as $ws | .value[] | select(($ws + "|" + .) as $k | $stubbed | index($k) == null) | "- [ ] `\(.)` of workspace `\($ws)` — no stub created (workflow missing or --no-secret-stubs); create the secret and reference it by hand"' "$summary")" + i_unpinned="$("$JQ_BIN" -r --arg fb "${SG_TF_FALLBACK_LABEL:-the fallback version}" '.terraformVersionFallbacks | to_entries[] | "- [ ] `\(.key)` was not pinned in TFC (\"\(.value)\"); it runs \($fb) — confirm it is compatible"' "$summary")" + # Preset mode: no version was sent for anyone, so every workflow may run + # something other than its TFC version — list them all with what TFC had. + if [ "$("$JQ_BIN" -r '.terraformVersionSource // "carry"' "$summary")" = "preset" ]; then + i_preset="$("$JQ_BIN" -r --arg p "${preset_desc:-the execution preset of the org}" '.tfcTerraformVersions // {} | to_entries[] | "- [ ] `\(.key)` ran Terraform \(.value) in TFC; now \($p) applies — run a plan before relying on it"' "$summary")" + fi + i_nonremote="$("$JQ_BIN" -r '.nonRemoteExecutionModes | to_entries[] | "- [ ] `\(.key)` used `\(.value)` execution in TFC — its state may live outside TFC; verify the exported state is current"' "$summary")" + i_renamed="$("$JQ_BIN" -r '.renamedWorkspaces | to_entries[] | "- `\(.key)` → `\(.value)`"' "$summary")" + fi + i_failed="$(printf '%s' "$st" | "$JQ_BIN" -r '.import // {} | to_entries[] | .value.group as $g | .value.failed[]? | "- [ ] `\($g)/\(.)` — see the import output for the API error; fix terraform.tfvars (or workspaceOverrides) and re-run `./sg-migrate.sh import`"')" + [ -s "$EXPORT_DIR/terraform-version-fallbacks.log" ] && i_fallback="$(sed 's/^/- [ ] /' "$EXPORT_DIR/terraform-version-fallbacks.log")" + i_trig="$(printf '%s' "$st" | "$JQ_BIN" -r '.triggers // {} | to_entries[] | .value.group as $g | (.value.failed[]? | "- [ ] `\($g)/\(.)` — trigger registration failed; check the connector has admin/webhook rights on the repository, then re-run `./sg-migrate.sh triggers`"), (.value.missing[]? | "- [ ] `\($g)/\(.)` — workflow was not imported, so no trigger was registered")')" + [ -s "$EXPORT_DIR/state-export-failures.log" ] && i_state="$(sed 's/^/- [ ] /' "$EXPORT_DIR/state-export-failures.log")" + n_secrets="$(_cl_count "$i_secrets")"; n_unstubbed="$(_cl_count "$i_unstubbed")"; n_failed="$(_cl_count "$i_failed")" + n_fallback="$(_cl_count "$i_fallback")"; n_unpinned="$(_cl_count "$i_unpinned")"; n_preset="$(_cl_count "$i_preset")"; n_trig="$(_cl_count "$i_trig")" + n_state="$(_cl_count "$i_state")"; n_nonremote="$(_cl_count "$i_nonremote")"; n_renamed="$(_cl_count "$i_renamed")" + + # --- the file ----------------------------------------------------------------- { printf '# Post-import checklist — StackGuardian org %s\n\n' "$ORG" printf 'Generated %s by stackguardian-migrator. Tick items as you complete them.\n\n' "$(state_now)" - - # 1. secrets printf '## 1. Set the real values of the placeholder secrets\n\n' printf 'TFC never exposes sensitive variable values, so each one was recreated as an SG secret with the value `CHANGE_ME` and referenced from its workflow as `${secret::<name>}`. Set the real values under [Org settings → Secrets](%s).\n\n' "$(secrets_ui_url)" - items="$(printf '%s' "$st" | "$JQ_BIN" -r --arg ui "$SG_UI_URL" --arg org "$ORG" '.secrets // {} | to_entries[] | "- [ ] `\(.key)` — \(.value.category) var `\(.value.var)` of [\(.value.group)/\(.value.workflow)](\($ui)/orchestrator/orgs/\($org)/wfgrps/\(.value.group)/wfs/\(.value.workflow))"')" - if [ -n "$items" ]; then printf '%s\n\n' "$items"; else printf -- '- None.\n\n'; fi - if [ -f "$summary" ]; then - items="$("$JQ_BIN" -r --argjson stubbed "$(printf '%s' "$st" | "$JQ_BIN" -c '[.secrets // {} | .[] | "\(.workspace)|\(.category):\(.var)"]')" \ - '.skippedSensitiveVars | to_entries[] | .key as $ws | .value[] | select(($ws + "|" + .) as $k | $stubbed | index($k) == null) | "- [ ] `\(.)` of workspace `\($ws)` — no stub created (workflow missing or --no-secret-stubs); create the secret and reference it by hand"' "$summary")" - [ -n "$items" ] && printf '%s\n\n' "$items" - fi - - # 2. failed imports + _cl_block "$i_secrets" + [ -n "$i_unstubbed" ] && printf '%s\n\n' "$i_unstubbed" printf '## 2. Workflows that failed to import\n\n' - items="$(printf '%s' "$st" | "$JQ_BIN" -r '.import // {} | to_entries[] | .value.group as $g | .value.failed[]? | "- [ ] `\($g)/\(.)` — see the import output for the API error; fix terraform.tfvars (or workspaceOverrides) and re-run `./sg-migrate.sh import`"')" - if [ -n "$items" ]; then printf '%s\n\n' "$items"; else printf -- '- None.\n\n'; fi - - # 3. terraform version fallbacks + _cl_block "$i_failed" printf '## 3. Verify workflows moved to a different Terraform version\n\n' - printf 'StackGuardian bundles managed Terraform only up to 1.5.7 (the last MPL/FOSS release). These workflows were pinned higher in TFC and now run the fallback version; run a plan and check for incompatibilities. To keep the newer version, point `workspaceOverrides[<ws>].terraformVersion` at a binary on a private runner and re-import.\n\n' - items="" - [ -s "$EXPORT_DIR/terraform-version-fallbacks.log" ] && items="$(sed 's/^/- [ ] /' "$EXPORT_DIR/terraform-version-fallbacks.log")" - if [ -f "$summary" ]; then - items="$items$(printf '%s' "$items" | grep -q . && echo)$("$JQ_BIN" -r '.terraformVersionFallbacks | to_entries[] | "- [ ] `\(.key)` was not pinned in TFC (\"\(.value)\"); SGDefaultTerraformVersion was used — confirm it is compatible"' "$summary")" - fi - if [ -n "$items" ]; then printf '%s\n\n' "$items"; else printf -- '- None.\n\n'; fi - - # 4. VCS triggers + printf 'StackGuardian bundles managed Terraform only up to 1.5.7 (the last MPL/FOSS release). Workflows listed here run a different version than they did in TFC: the fallback `SGDefaultTerraformVersion`, or the version from the execution preset of the org (Settings → Runner groups → Execution presets) when no version was sent. Run a plan and check for incompatibilities. To keep a newer version, point `workspaceOverrides[<ws>].terraformVersion` at a binary on a private runner, or give the execution preset a custom runtime image that ships it, and re-import.\n\n' + _cl_block "$(printf '%s\n%s\n%s' "$i_fallback" "$i_unpinned" "$i_preset" | grep . || true)" printf '## 4. VCS triggers\n\n' - items="$(printf '%s' "$st" | "$JQ_BIN" -r '.triggers // {} | to_entries[] | .value.group as $g | (.value.failed[]? | "- [ ] `\($g)/\(.)` — trigger registration failed; check the connector has admin/webhook rights on the repository, then re-run `./sg-migrate.sh triggers`"), (.value.missing[]? | "- [ ] `\($g)/\(.)` — workflow was not imported, so no trigger was registered")')" - if [ -n "$items" ]; then printf '%s\n\n' "$items"; else printf -- '- None failed. Push to a tracked branch or open a pull request to confirm the webhooks fire.\n\n'; fi - - # 5. state + if [ -n "$i_trig" ]; then printf '%s\n\n' "$i_trig"; else printf -- '- None failed. Push to a tracked branch or open a pull request to confirm the webhooks fire.\n\n'; fi printf '## 5. Terraform state\n\n' - if [ -s "$EXPORT_DIR/state-export-failures.log" ]; then - printf 'State could not be pulled from TFC for these workspaces; upload it manually (Workflow → Settings → State) or run an import in the new workflow.\n\n' - sed 's/^/- [ ] /' "$EXPORT_DIR/state-export-failures.log"; echo + if [ -n "$i_state" ]; then + printf 'State could not be pulled from TFC for these workspaces; upload it manually (Workflow → Settings → State) or run an import in the new workflow.\n\n%s\n\n' "$i_state" else printf -- '- All selected workspaces had their state exported.\n\n' fi - if [ -f "$summary" ]; then - items="$("$JQ_BIN" -r '.nonRemoteExecutionModes | to_entries[] | "- [ ] `\(.key)` used `\(.value)` execution in TFC — its state may live outside TFC; verify the exported state is current"' "$summary")" - [ -n "$items" ] && printf '%s\n\n' "$items" - fi - - # 6. renames - if [ -f "$summary" ] && [ "$("$JQ_BIN" '.renamedWorkspaces | length' "$summary")" -gt 0 ]; then - printf '## 6. Renamed workflows\n\nThese TFC workspace names were not valid StackGuardian workflow names and were adjusted:\n\n' - "$JQ_BIN" -r '.renamedWorkspaces | to_entries[] | "- `\(.key)` → `\(.value)`"' "$summary"; echo + [ -n "$i_nonremote" ] && printf '%s\n\n' "$i_nonremote" + if [ -n "$i_renamed" ]; then + printf '## 6. Renamed workflows\n\nThese TFC workspace names were not valid StackGuardian workflow names and were adjusted:\n\n%s\n\n' "$i_renamed" fi - printf '## Finally\n\n- [ ] Run a plan on one workflow per project and compare with the last TFC run.\n- [ ] Disable auto-apply / triggers on the TFC workspaces once StackGuardian owns the deployments.\n' } >"$out" + + # --- the terminal view -------------------------------------------------------- sg_step "Post-import checklist" - sed 's/^/ /' "$out" >&2 - sg_dim "saved to $(sg_rel "$out")" + for f in "${PF[@]}"; do + seg="$(seg_of "$f")" + grp="$(group_for "$seg")" + n="$(printf '%s' "$st" | "$JQ_BIN" -r --arg s "$seg" '.import[$s].imported // [] | length')" + total=$((total + n)) + glines+=("$grp ($n)|$(wfgrp_ui_url "$grp")") + done + if [ "$total" -gt 0 ]; then + printf ' %s✓%s %s\n' "$C_GREEN" "$C_RESET" "$total workflow(s) are in StackGuardian org $ORG:" >&2 + gw="$(sg_maxlen 10 "${glines[@]%%|*}")" + for n in "${glines[@]}"; do printf " %-${gw}s %s%s%s\n" "${n%%|*}" "$C_DIM" "${n#*|}" "$C_RESET" >&2; done + fi + _cl_status "$((n_secrets + n_unstubbed))" "secrets: none to fill in" \ + "secrets: set the real value of $n_secrets placeholder secret(s) (value CHANGE_ME)${i_unstubbed:+; $n_unstubbed sensitive var(s) have no stub}" + _cl_status "$n_failed" "imports: none failed" "imports: $n_failed workflow(s) failed — fix and re-run '$PROG import'" + if [ "$n_preset" -gt 0 ]; then + _cl_status "$n_preset" "" "Terraform version: all $n_preset workflow(s) take the execution preset's version (${preset_desc:-see the SG org settings}) instead of their TFC version — run a plan before relying on them" + else + _cl_status "$((n_fallback + n_unpinned))" "Terraform version: every workflow keeps its TFC version" \ + "Terraform version: $((n_fallback + n_unpinned)) workflow(s) run ${SG_TF_FALLBACK_LABEL:-the fallback version} instead of what TFC used — run a plan before relying on them" + fi + _cl_status "$n_trig" "VCS triggers: registered for every workflow that had them" "VCS triggers: $n_trig workflow(s) without triggers — see the checklist, then '$PROG triggers'" + _cl_status "$((n_state + n_nonremote))" "state: exported for every selected workspace" \ + "state: $n_state workspace(s) without exported state${i_nonremote:+, $n_nonremote with non-remote execution} — upload by hand" + [ "$n_renamed" -gt 0 ] && _cl_status "$n_renamed" "" "names: $n_renamed workflow(s) were renamed to valid SG names" + # shellcheck disable=SC2034 + CHECKLIST_OPEN=$((n_secrets + n_unstubbed + n_failed + n_fallback + n_unpinned + n_preset + n_trig + n_state + n_nonremote)) + sg_dim "full checklist with links: $(sg_rel "$out")" } cmd_checklist() { - sg_step "Phase: checklist" + phase_begin "checklist" [ -n "${SG_API_TOKEN:-}" ] || die "SG_API_TOKEN is not set." [ -n "$ORG" ] || die "StackGuardian org not set (use --org or SG_ORG)." export SG_API_TOKEN SG_BASE_URL diff --git a/scripts/lib/preflight.sh b/scripts/lib/preflight.sh index bed47c6..ecf29b2 100644 --- a/scripts/lib/preflight.sh +++ b/scripts/lib/preflight.sh @@ -3,18 +3,21 @@ # # Verifies, before the long terraform apply or a bulk import, that the tokens # work and that everything terraform.tfvars refers to actually exists in TFC -# and StackGuardian. Prints one line per check (✓ ok, ! warning, ✗ failure) and -# fails the run when any check fails. Skipped with --skip-preflight. +# and StackGuardian — and that the pieces agree with each other (VCS connector +# kind vs. sourceConfigDestKind, repo URL prefix vs. where the TFC repositories +# live). Prints one line per check (✓ ok, ! warning, ✗ failure) and fails the +# run when any check fails. Skipped with --skip-preflight. +PF_OK=0 PF_FAIL=0 PF_WARN=0 -pf_ok() { printf ' %s✓%s %s\n' "$C_GREEN" "$C_RESET" "$*" >&2; } +pf_ok() { printf ' %s✓%s %s\n' "$C_GREEN" "$C_RESET" "$*" >&2; PF_OK=$((PF_OK + 1)); } pf_warn() { printf ' %s!%s %s\n' "$C_YELLOW" "$C_RESET" "$*" >&2; PF_WARN=$((PF_WARN + 1)); } pf_fail() { printf ' %s✗%s %s\n' "$C_RED$C_BOLD" "$C_RESET" "$*" >&2; PF_FAIL=$((PF_FAIL + 1)); } # --- Terraform Cloud ----------------------------------------------------------- preflight_tfc() { - local host token org body n names_json tags_json ignore_json jqb + local host token org body n sel jqb jqb="$(sg_resolve jq sg_ensure_jq)" host="$(tfc_hostname)" org="$(tfvars_get .tfOrg)" @@ -44,22 +47,15 @@ preflight_tfc() { fi # Workspace selection: mirror the module's filters (names glob, include/exclude tags). if body="$(tfc_list_workspaces "$org" 2>/dev/null)"; then - names_json="$(tfvars_get_json .workspacenames)" - tags_json="$(tfvars_get_json .tfWorkspaceTags)" - ignore_json="$(tfvars_get_json .tfWorkspaceIgnoreTags)" - n="$(printf '%s' "$body" | "$jqb" --argjson names "${names_json:-null}" --argjson tags "${tags_json:-null}" --argjson ignore "${ignore_json:-null}" ' - def glob($p): ("^" + ($p | gsub("\\*"; ".*")) + "$"); - [ .[] - | select(($names == null) or ($names == ["*"]) or ([.name] | inside([]) | not) and ([$names[] as $p | (.name | test(glob($p)))] | any)) - | select(($tags == null) or (($tags | length) == 0) or ([.tags[]?] | inside($tags) | not) or (([.tags[]?] | map(select(. as $t | $tags | index($t) != null)) | length) > 0)) - | select(($ignore == null) or (($ignore | length) == 0) or (([.tags[]?] | map(select(. as $t | $ignore | index($t) != null)) | length) == 0)) - ] | length')" + sel="$(tfc_select_workspaces "$body" "$(tfvars_get_json .workspacenames)" "$(tfvars_get_json .tfWorkspaceTags)" "$(tfvars_get_json .tfWorkspaceIgnoreTags)")" + n="$(printf '%s' "$sel" | "$jqb" 'length')" if [ "$n" -gt 0 ]; then pf_ok "$n workspace(s) match the selection (of $(printf '%s' "$body" | "$jqb" 'length') in the org)" else pf_warn "no workspace matches workspacenames/tfWorkspaceTags/tfWorkspaceIgnoreTags — apply would export nothing" fi PF_TFC_WORKSPACES="$body" + PF_TFC_SELECTED="$sel" else pf_warn "could not list workspaces for '$org' (HTTP $TFC_HTTP_CODE)" fi @@ -69,7 +65,7 @@ preflight_tfc() { # --- StackGuardian ------------------------------------------------------------ # preflight_sg <required:0|1> preflight_sg() { - local required="$1" ints names id n jqb rg + local required="$1" ints names label id kind n jqb rg jqb="$(sg_resolve jq sg_ensure_jq)" if [ -z "${SG_API_TOKEN:-}" ] || [ -z "$ORG" ]; then if [ "$required" -eq 1 ]; then @@ -90,30 +86,38 @@ preflight_sg() { return 0 fi pf_ok "StackGuardian credentials valid (org '$ORG', $SG_BASE_URL)" + PF_SG_INTS="$ints" + # The execution preset (org workflow defaults) decides whatever tfvars leaves + # to it; preflight_config reports it next to the version/runner settings. + if PF_PRESET="$(sg_execution_preset)"; then PF_PRESET_READ=1; else PF_PRESET_READ=0; fi - # Every integration id referenced anywhere in tfvars must exist. + # Every integration id referenced anywhere in tfvars must exist. Each line is + # "<role>\t<id>" so the report can say what the connector is for. names="$(printf '%s' "$ints" | "$jqb" -c '[.[].name]')" - while IFS= read -r id; do + while IFS=$'\t' read -r label id; do [ -n "$id" ] || continue case "$id" in - *CHANGE_ME* | *INTEGRATION_ID*) pf_fail "placeholder connector id '$id' in $(sg_rel "$TFVARS") — run '$PROG init' or edit the file" ;; + *CHANGE_ME* | *INTEGRATION_ID*) pf_fail "placeholder $label connector id '$id' in $(sg_rel "$TFVARS") — run '$PROG init' or edit the file" ;; /integrations/*) if printf '%s' "$names" | "$jqb" -e --arg n "${id#/integrations/}" 'index($n) != null' >/dev/null; then - pf_ok "connector $id exists" + kind="$(sg_integration_type "$ints" "$id")" + pf_ok "$label connector ${id#/integrations/} exists${kind:+ ($kind)}" else - pf_fail "connector $id not found in org '$ORG' (available: $(printf '%s' "$names" | "$jqb" -r 'join(", ")'))" + pf_fail "$label connector $id not found in org '$ORG' (available: $(printf '%s' "$names" | "$jqb" -r 'join(", ")'))" fi ;; /secrets/*) if sg_secret_exists "${id#/secrets/}"; then pf_ok "secret $id exists"; else pf_fail "secret $id not found in org '$ORG'"; fi ;; - *) pf_warn "unrecognised integration id '$id' (expected /integrations/<name> or /secrets/<name>)" ;; + *) pf_warn "unrecognised $label integration id '$id' (expected /integrations/<name> or /secrets/<name>)" ;; esac done < <(tfvars_json | "$jqb" -r ' - [ .SGDefaultVCSAuthIntegrationID, - (.SGDefaultDeploymentPlatformConfig // [])[]?.config.integrationId, - ((.workspaceOverrides // {}) | to_entries[]? | .value | (.vcsAuthIntegrationID, ((.DeploymentPlatformConfig // [])[]?.config.integrationId))) ] - | map(select(. != null and . != "")) | unique | .[]') + [ (.SGDefaultVCSAuthIntegrationID | select(. != null and . != "") | ["VCS", .]), + ((.SGDefaultDeploymentPlatformConfig // [])[]?.config.integrationId | select(. != null and . != "") | ["cloud", .]), + ((.workspaceOverrides // {}) | to_entries[]? | .value + | ((.vcsAuthIntegrationID | select(. != null and . != "") | ["override VCS", .]), + ((.DeploymentPlatformConfig // [])[]?.config.integrationId | select(. != null and . != "") | ["override cloud", .]))) ] + | unique_by(.[1]) | .[] | @tsv') # Every private runner group must exist. while IFS= read -r rg; do @@ -123,41 +127,137 @@ preflight_sg() { [ (.SGDefaultRunnerConstraints // {} | select(.type == "private") | .names[]?), ((.workspaceOverrides // {}) | to_entries[]? | .value.RunnerConstraints // {} | select(.type == "private") | .names[]?) ] | unique | .[]') - n="$(tfvars_json | "$jqb" -r '(.SGDefaultRunnerConstraints // {}).type // "shared"')" - if [ "$n" = "shared" ]; then pf_ok "runners: StackGuardian shared runners"; fi + if tfvars_is_null SGDefaultRunnerConstraints; then + if [ "${PF_PRESET_READ:-0}" -eq 1 ]; then + pf_ok "runners: from the org's execution preset — $(sg_preset_runner_desc "$PF_PRESET")" + else + pf_ok "runners: from the org's execution preset (platform default: shared runners)" + fi + else + n="$(tfvars_json | "$jqb" -r '(.SGDefaultRunnerConstraints // {}).type // "shared"')" + if [ "$n" = "shared" ]; then pf_ok "runners: StackGuardian shared runners"; fi + fi return 0 } -# --- static config -------------------------------------------------------------- +# --- static config + consistency --------------------------------------------------- +# _pf_host <url> — host part of a URL. +_pf_host() { local h="${1#*://}"; printf '%s' "${h%%/*}"; } + +# _pf_kind_of_host <host> — the provider kind a well-known VCS host implies. +_pf_kind_of_host() { + case "$1" in + github.com | www.github.com) printf 'GITHUB_COM' ;; + gitlab.com | www.gitlab.com) printf 'GITLAB_COM' ;; + bitbucket.org | www.bitbucket.org) printf 'BITBUCKET_ORG' ;; + dev.azure.com | *.visualstudio.com) printf 'AZURE_DEVOPS' ;; + *) printf '' ;; + esac +} + preflight_config() { - local v ceiling="1.5.7" export_path want jqb + local v kind ceiling="1.5.7" export_path want jqb prefix host conn conn_kind line prov tfc_prefix tfc_kind jqb="$(sg_resolve jq sg_ensure_jq)" + + # VCS kind, and does it agree with the connector / the repositories? v="$(tfvars_get .SGDefaultSourceConfigDestKind GIT_OTHER)" case "$v" in - GITHUB_COM | GITHUB_APP_CUSTOM | GIT_OTHER | INLINE | BITBUCKET_ORG | GITLAB_COM | AZURE_DEVOPS) pf_ok "SGDefaultSourceConfigDestKind = $v" ;; + GITHUB_COM | GITHUB_APP_CUSTOM | GIT_OTHER | INLINE | BITBUCKET_ORG | GITLAB_COM | AZURE_DEVOPS) ;; *) pf_fail "SGDefaultSourceConfigDestKind '$v' is not one of GITHUB_COM, GITLAB_COM, BITBUCKET_ORG, AZURE_DEVOPS, GIT_OTHER" ;; esac + conn="$(tfvars_get .SGDefaultVCSAuthIntegrationID)" + conn_kind="" + [ -n "${PF_SG_INTS:-}" ] && [ -n "$conn" ] && conn_kind="$(sg_vcs_kind_of "$(sg_integration_type "$PF_SG_INTS" "$conn")")" + if [ -n "$conn_kind" ] && [ "$conn_kind" != "$v" ]; then + pf_fail "VCS kind $v does not match connector ${conn#/integrations/}, which is a $conn_kind connector — set SGDefaultSourceConfigDestKind = \"$conn_kind\" (or pick another connector)" + elif [ -n "$conn_kind" ]; then + pf_ok "VCS kind $v matches connector ${conn#/integrations/}" + else + pf_ok "VCS kind $v" + fi + + # Repo URL prefix: compare with where the TFC workspaces' repositories live + # (the transformer emits <prefix>/<tfc repo identifier>); otherwise with the + # kind's well-known host. + prefix="$(tfvars_get .SGDefaultIACVCSRepoPrefix)" + host="$(_pf_host "$prefix")" + tfc_prefix="" + tfc_kind="" + if [ -n "${PF_TFC_SELECTED:-}" ]; then + while IFS=$'\t' read -r prov _ line; do + [ -n "$prov" ] || continue + tfc_kind="$(tfc_vcs_kind_for "$prov")" + [ "$line" != "-" ] && tfc_prefix="$line" + break # most common provider only + done <<<"$(tfc_vcs_summary "$PF_TFC_SELECTED")" + fi + if [ -z "$prefix" ] || [ "$prefix" = "https://VCS_PROVIDER_DOMAIN" ]; then + pf_fail "SGDefaultIACVCSRepoPrefix is not set — the repositories' base URL, e.g. https://github.com" + elif [ -n "$tfc_prefix" ]; then + case "$host" in + *"$(_pf_host "$tfc_prefix")") pf_ok "repo URL prefix $prefix matches the TFC repositories" ;; + *) pf_warn "repo URL prefix $prefix does not match where the TFC repositories live ($tfc_prefix) — every workflow would clone from the wrong place unless the repositories moved; check SGDefaultIACVCSRepoPrefix" ;; + esac + else + kind="$(_pf_kind_of_host "$host")" + if [ -n "$kind" ] && [ "$kind" != "$v" ] && [ "$v" != "GIT_OTHER" ]; then + pf_warn "repo URL prefix $prefix looks like $kind but the VCS kind is $v — check SGDefaultIACVCSRepoPrefix / SGDefaultSourceConfigDestKind" + else + pf_ok "repo URL prefix $prefix" + fi + fi + if [ -n "$tfc_kind" ] && [ "$tfc_kind" != "$v" ] && [ "$v" != "GIT_OTHER" ]; then + pf_warn "the TFC workspaces are connected to $(tfc_vcs_label_for "$prov") but the VCS kind is $v" + fi + while IFS= read -r v; do [ -n "$v" ] || continue case "$v" in - AWS_STATIC | AWS_RBAC | AWS_OIDC | AZURE_STATIC | AZURE_OIDC | AZURE_MANAGED_ID_OIDC | GCP_STATIC | GCP_OIDC) pf_ok "DeploymentPlatformConfig kind $v" ;; - *) pf_fail "DeploymentPlatformConfig kind '$v' is not a cloud connector kind (AWS_STATIC, AWS_RBAC, AWS_OIDC, AZURE_STATIC, AZURE_OIDC, AZURE_MANAGED_ID_OIDC, GCP_STATIC, GCP_OIDC) — a VCS connector was picked as the cloud connector?" ;; + AWS_STATIC | AWS_RBAC | AWS_OIDC | AZURE_STATIC | AZURE_OIDC | AZURE_MANAGED_ID_OIDC | GCP_STATIC | GCP_OIDC) pf_ok "cloud connector kind $v" ;; + *) pf_fail "cloud connector kind '$v' (DeploymentPlatformConfig) is not one of AWS_STATIC, AWS_RBAC, AWS_OIDC, AZURE_STATIC, AZURE_OIDC, AZURE_MANAGED_ID_OIDC, GCP_STATIC, GCP_OIDC — a VCS connector was picked as the cloud connector?" ;; esac done < <(tfvars_json | "$jqb" -r '[ (.SGDefaultDeploymentPlatformConfig // [])[]?.kind, ((.workspaceOverrides // {}) | to_entries[]? | .value.DeploymentPlatformConfig // [] | .[]?.kind) ] | map(select(. != null)) | unique | .[]') - v="$(tfvars_get .SGDefaultTerraformVersion TERRAFORM-1.5.7)" - if [[ "$v" =~ ^TERRAFORM-([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then - if [ "$(printf '%03d%03d%03d' "${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}" "${BASH_REMATCH[3]}")" -gt "001005007" ]; then - pf_warn "SGDefaultTerraformVersion $v is above SG's managed ceiling ($ceiling); fallbacks would fail too unless a private runner ships that binary" + + # Terraform version policy, and what the execution preset would supply where + # tfvars leaves the decision to it. + local src preset_note="" + src="$(tfvars_get .SGTerraformVersionSource carry)" + case "$src" in + carry | preset) ;; + *) + pf_fail "SGTerraformVersionSource '$src' must be \"carry\" or \"preset\"" + src="carry" + ;; + esac + [ "${PF_PRESET_READ:-0}" -eq 1 ] && preset_note="$(sg_preset_desc "$PF_PRESET")" + if tfvars_is_null SGDefaultTerraformVersion; then v=""; else v="$(tfvars_get .SGDefaultTerraformVersion TERRAFORM-1.5.7)"; fi + if [ "$src" = "preset" ]; then + if [ -n "$preset_note" ]; then + pf_ok "Terraform version: from the org's execution preset — $preset_note" else - pf_ok "SGDefaultTerraformVersion = $v" + pf_warn "Terraform version: from the org's execution preset, which could not be read here (platform default if none is configured: managed Terraform 1.5.7 on shared runners)" fi else - pf_fail "SGDefaultTerraformVersion '$v' must look like TERRAFORM-1.5.7" + if [ -z "$v" ]; then + pf_ok "Terraform version: pinned TFC versions are carried over; unpinned or rejected pins go to the execution preset${preset_note:+ — $preset_note}" + elif [[ "$v" =~ ^TERRAFORM-([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then + if [ "$(printf '%03d%03d%03d' "${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}" "${BASH_REMATCH[3]}")" -gt "001005007" ]; then + pf_warn "fallback Terraform ${v#TERRAFORM-} (SGDefaultTerraformVersion) is above SG's managed ceiling ($ceiling); fallbacks would fail too unless a private runner ships that binary" + else + pf_ok "Terraform version: pinned TFC versions are carried over; fallback ${v#TERRAFORM-} is within SG's managed ceiling ($ceiling)" + fi + else + pf_fail "SGDefaultTerraformVersion '$v' must look like TERRAFORM-1.5.7 (or be null to defer to the execution preset)" + fi + if [ "${PF_PRESET_READ:-0}" -eq 1 ] && sg_preset_runner_provided "$PF_PRESET"; then + pf_warn "the execution preset mounts a runner-provided Terraform binary; with SGTerraformVersionSource = \"carry\" the carried pins and that binary would be sent together — consider \"preset\"" + fi fi + export_path="$(tfvars_get .exportPath export)" case "$export_path" in /*) want="$export_path" ;; *) want="$SG_REPO_ROOT/$export_path" ;; esac if [ "$(cd "$(dirname "$want")" 2>/dev/null && pwd)/$(basename "$want")" = "$(cd "$(dirname "$EXPORT_DIR")" 2>/dev/null && pwd)/$(basename "$EXPORT_DIR")" ]; then - pf_ok "export directory: $(sg_rel "$EXPORT_DIR")" + pf_ok "export directory $(sg_rel "$EXPORT_DIR")/" else pf_warn "exportPath in tfvars ($export_path) differs from the orchestrator's export dir ($(sg_rel "$EXPORT_DIR")) — payloads would be written where later phases don't look" fi @@ -178,7 +278,7 @@ preflight_config() { preflight_import_inputs() { payload_files if [ "${#PF[@]}" -gt 0 ]; then - pf_ok "${#PF[@]} payload file(s) in $(sg_rel "$EXPORT_DIR")" + pf_ok "${#PF[@]} payload file(s) in $(sg_rel "$EXPORT_DIR")/" else pf_fail "no payload files in $(sg_rel "$EXPORT_DIR") — run '$PROG apply' first" fi @@ -192,11 +292,16 @@ preflight_run() { local ctx="$1" [ "${SKIP_PREFLIGHT:-0}" -eq 1 ] && { sg_warn "preflight skipped (--skip-preflight)"; return 0; } [ "${PREFLIGHT_DONE:-0}" -eq 1 ] && return 0 - sg_step "Preflight ($ctx)" + sg_step "Preflight" [ -f "$TFVARS" ] || die "Missing $(sg_rel "$TFVARS"). Run: $PROG init" + PF_OK=0 PF_FAIL=0 PF_WARN=0 PF_TFC_WORKSPACES="" + PF_TFC_SELECTED="" + PF_SG_INTS="" + PF_PRESET="{}" + PF_PRESET_READ=0 local parse_err if ! parse_err="$(tfvars_valid)"; then pf_fail "$(sg_rel "$TFVARS") is not valid HCL: ${parse_err:-parse error}" @@ -225,9 +330,9 @@ preflight_run() { die "preflight found $PF_FAIL problem(s); fix them (or re-run '$PROG init') and try again. Use --skip-preflight to bypass." fi if [ "$PF_WARN" -gt 0 ]; then - sg_warn "preflight passed with $PF_WARN warning(s)" + sg_warn "preflight passed with $PF_WARN warning(s) — read them before continuing" else - sg_success "preflight passed" + sg_success "preflight passed ($PF_OK checks)" fi } diff --git a/scripts/lib/prompt.sh b/scripts/lib/prompt.sh index 7e754e5..fec08bc 100644 --- a/scripts/lib/prompt.sh +++ b/scripts/lib/prompt.sh @@ -93,9 +93,14 @@ sg_select() { return 0 fi printf '%s? %s%s\n' "$C_BOLD" "$q" "$C_RESET" >&2 + # Descriptions line up in one column sized to the longest value (capped so a + # single very long name cannot push everything off-screen). + local w + w="$(sg_maxlen 10 "${vals[@]}")" + [ "$w" -gt 40 ] && w=40 for ((i = 0; i < n; i++)); do if [ -n "${descs[i]}" ]; then - printf ' %s%2d)%s %-10s %s%s%s\n' "$C_CYAN" "$((i + 1))" "$C_RESET" "${vals[i]}" "$C_DIM" "— ${descs[i]}" "$C_RESET" >&2 + printf " %s%2d)%s %-${w}s %s%s%s\n" "$C_CYAN" "$((i + 1))" "$C_RESET" "${vals[i]}" "$C_DIM" "— ${descs[i]}" "$C_RESET" >&2 else printf ' %s%2d)%s %s\n' "$C_CYAN" "$((i + 1))" "$C_RESET" "${vals[i]}" >&2 fi diff --git a/scripts/lib/report.sh b/scripts/lib/report.sh index 36d39f5..edadbf5 100644 --- a/scripts/lib/report.sh +++ b/scripts/lib/report.sh @@ -4,42 +4,65 @@ # confirmation prompt (and by 'import --dry-run'). # show_migration_summary — condensed view of export/migration-summary.json plus -# state-export-failures.log, so nobody has to know to open the files. +# the state export result, so nobody has to know to open the files. show_migration_summary() { - local f="$EXPORT_DIR/migration-summary.json" jqb n + local f="$EXPORT_DIR/migration-summary.json" jqb n fails=0 states [ -f "$f" ] || return 0 jqb="$(sg_resolve jq sg_ensure_jq)" sg_step "Migration summary" - printf ' %s%-30s%s %s\n' "$C_BOLD" "TFC organisation" "$C_RESET" "$("$jqb" -r '.organization' "$f")" >&2 - printf ' %s%-30s%s %s\n' "$C_BOLD" "Workspaces exported" "$C_RESET" "$("$jqb" -r '.workspaceCount' "$f")" >&2 + sg_row "TFC organisation" "$("$jqb" -r '.organization' "$f")" + sg_row "Workspaces exported" "$("$jqb" -r '.workspaceCount' "$f")" "$jqb" -r '.projectWorkspaceCounts | to_entries[] | " tfc-\(.key | ascii_downcase | gsub("[^a-z0-9-]+"; "-")): \(.value) workflow(s)"' "$f" >&2 + if [ "$(tfvars_get .exportStateFiles true)" != "false" ]; then + states=0 + for n in "$EXPORT_DIR"/states/*.tfstate; do [ -f "$n" ] && states=$((states + 1)); done + [ -s "$EXPORT_DIR/state-export-failures.log" ] && fails="$(wc -l <"$EXPORT_DIR/state-export-failures.log" | tr -d ' ')" + if [ "$fails" -gt 0 ]; then + sg_row "State files" "$states exported, ${C_YELLOW}$fails failed${C_RESET} (see below)" + else + sg_row "State files" "$states exported to $(sg_rel "$EXPORT_DIR")/states/" + fi + fi + + # Version / runner policy (what the transformer sent, or left to the preset). + local src def + src="$("$jqb" -r '.terraformVersionSource // "carry"' "$f")" + def="$("$jqb" -r '.terraformVersionDefault // empty' "$f")" + if [ "$src" = "preset" ]; then + sg_row "Terraform version" "none sent; the org's execution preset applies at import" + elif [ -z "$def" ]; then + sg_row "Terraform version" "carried from TFC; unpinned or rejected pins go to the execution preset" + else + sg_row "Terraform version" "carried from TFC; fallback ${def#TERRAFORM-} for unpinned or rejected pins" + fi + [ "$("$jqb" -r '.runnerConstraintsSource // "config"' "$f")" = "preset" ] && sg_row "Runners" "none sent; the org's execution preset applies at import" - _summary_section "$f" '.skippedSensitiveVars' "Sensitive variables skipped (TFC never exposes them)" \ - 'to_entries[] | "\(.key): \(.value | join(", "))"' "recreated as SG secrets after import" - _summary_section "$f" '.strippedVars' "TFC-specific variables stripped (ignoreVarPatterns)" \ - 'to_entries[] | "\(.key): \(.value | join(", "))"' "" - _summary_section "$f" '.terraformVersionFallbacks' "Terraform version not pinned (SGDefaultTerraformVersion used)" \ - 'to_entries[] | "\(.key): \"\(.value)\""' "" - _summary_section "$f" '.nonRemoteExecutionModes' "Non-remote execution mode (state may not be in TFC)" \ - 'to_entries[] | "\(.key): \(.value)"' "" + _summary_section "$f" '.skippedSensitiveVars' "Sensitive variables skipped" \ + 'to_entries[] | "\(.key): \(.value | join(", "))"' "TFC never exposes their values; they become placeholder SG secrets after import" + _summary_section "$f" '.strippedVars' "TFC-specific variables stripped" \ + 'to_entries[] | "\(.key): \(.value | join(", "))"' "they only mean something inside Terraform Cloud (ignoreVarPatterns)" + _summary_section "$f" '.terraformVersionFallbacks' "Terraform version not pinned" \ + 'to_entries[] | "\(.key): \"\(.value)\""' "$([ -z "$def" ] && echo "left to the execution preset" || echo "the fallback ${def#TERRAFORM-} is used")" + _summary_section "$f" '.nonRemoteExecutionModes' "Non-remote execution mode" \ + 'to_entries[] | "\(.key): \(.value)"' "their state may not live in TFC" _summary_section "$f" '.renamedWorkspaces' "Renamed to a valid SG workflow name" \ 'to_entries[] | "\(.key) -> \(.value)"' "" - if [ -s "$EXPORT_DIR/state-export-failures.log" ]; then - n="$(wc -l <"$EXPORT_DIR/state-export-failures.log" | tr -d ' ')" - printf ' %s!%s %s\n' "$C_YELLOW" "$C_RESET" "state could not be exported for $n workspace(s):" >&2 + if [ "$fails" -gt 0 ]; then + printf ' %s!%s %s\n' "$C_YELLOW" "$C_RESET" "state could not be exported for $fails workspace(s):" >&2 sed 's/^/ /' "$EXPORT_DIR/state-export-failures.log" | head -10 >&2 - [ "$n" -gt 10 ] && sg_dim " ... see $(sg_rel "$EXPORT_DIR")/state-export-failures.log" + [ "$fails" -gt 10 ] && sg_dim " ... see $(sg_rel "$EXPORT_DIR")/state-export-failures.log" fi sg_dim "full report: $(sg_rel "$EXPORT_DIR")/migration-summary.md" } -# _summary_section <file> <jq-path> <title> <jq-line-filter> <hint> +# _summary_section <file> <jq-path> <title> <jq-line-filter> <hint> — prints +# "! <title> in <n> workspace(s) — <hint>" plus up to 10 detail lines. _summary_section() { local f="$1" path="$2" title="$3" lines="$4" hint="$5" jqb n jqb="$(sg_resolve jq sg_ensure_jq)" n="$("$jqb" -r "$path | length" "$f")" [ "$n" -gt 0 ] || return 0 - printf ' %s!%s %s (%s)%s\n' "$C_YELLOW" "$C_RESET" "$title" "$n" "${hint:+ — $hint}" >&2 + printf ' %s!%s %s in %s workspace(s)%s\n' "$C_YELLOW" "$C_RESET" "$title" "$n" "${hint:+ — $hint}" >&2 "$jqb" -r "$path | $lines" "$f" | head -10 | sed 's/^/ /' >&2 [ "$n" -gt 10 ] && sg_dim " ... $((n - 10)) more in migration-summary.md" return 0 @@ -51,42 +74,85 @@ tf_version_above_ceiling() { [ "$(printf '%03d%03d%03d' "${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}" "${BASH_REMATCH[3]}")" -gt "001005007" ] } +# preset_labels — short labels for the plan and the fallback messages, from +# SG_PRESET_JSON (the org's execution preset, "" when it could not be read) and +# SG_DEFAULT_TF_VERSION ("" = drop the version so the preset decides): +# SG_PRESET_TFV "1.5.7" / "bin:/opt/tf" / "platform default" +# SG_PRESET_RUNNER_SHORT "shared" / "private:rg" +# SG_TF_FALLBACK_SHORT "1.5.7" / "preset" +# SG_TF_FALLBACK_LABEL "TERRAFORM-1.5.7" / "no pinned version (the org's execution preset decides: ...)" +# shellcheck disable=SC2034 # consumed by migrate.sh (do_import, tf_fallback_notice) and checklist.sh +preset_labels() { + SG_PRESET_TFV="" + SG_PRESET_RUNNER_SHORT="" + if [ -n "${SG_PRESET_JSON:-}" ]; then + SG_PRESET_TFV="$(printf '%s' "$SG_PRESET_JSON" | "$JQ_BIN" -r ' + ((.terraformDefaults // {}).TerraformConfig // {}) as $c + | if (($c.terraformBinPath // []) | length) > 0 then "bin:\($c.terraformBinPath[0].source // "")" + elif ($c.terraformVersion // "") == "" then "platform default" + else ($c.terraformVersion | ltrimstr("TERRAFORM-")) end')" + SG_PRESET_RUNNER_SHORT="$(printf '%s' "$SG_PRESET_JSON" | "$JQ_BIN" -r '(.RunnerConstraints // {}) | if (.type // "shared") == "private" then "private:\((.names // []) | join(","))" else "shared" end')" + fi + if [ -n "${SG_DEFAULT_TF_VERSION:-}" ]; then + SG_TF_FALLBACK_SHORT="${SG_DEFAULT_TF_VERSION#TERRAFORM-}" + SG_TF_FALLBACK_LABEL="$SG_DEFAULT_TF_VERSION" + else + SG_TF_FALLBACK_SHORT="preset" + SG_TF_FALLBACK_LABEL="the version from the org's execution preset ($(sg_preset_desc "${SG_PRESET_JSON:-}"))" + fi +} + # show_import_plan <payload>... — per-workflow table: what will be created or # updated, with the Terraform version (and fallback), runner, triggers and the -# number of variables / skipped secrets. Needs JQ_BIN, ORG, SG_API_TOKEN. +# number of variables / skipped secrets. "preset" marks a value the payload +# leaves to the org's execution preset. Columns are sized to their content. +# Needs JQ_BIN, ORG, SG_API_TOKEN, preset_labels. show_import_plan() { - local f seg grp existing summary rows + local f seg grp existing summary rows all_rows="" name action tfv runner trig vars secrets wn gn rw + local -a names=() groups=() summary="$EXPORT_DIR/migration-summary.json" [ -f "$summary" ] || summary="" - printf '\n %s%-28s %-24s %-7s %-28s %-8s %-8s %-5s %s%s\n' "$C_BOLD" "WORKFLOW" "GROUP" "ACTION" "TERRAFORM" "RUNNER" "TRIGGERS" "VARS" "SECRETS" "$C_RESET" >&2 for f in "$@"; do seg="$(seg_of "$f")" grp="$(group_for "$seg")" existing="$(sg_list_workflows "$grp")" printf '%s' "$existing" | "$JQ_BIN" -e 'type == "array"' >/dev/null 2>&1 || existing='[]' - rows="$("$JQ_BIN" -r --argjson ex "$existing" --arg def "$SG_DEFAULT_TF_VERSION" --argjson ws "$(ws_filter_json)" \ + rows="$("$JQ_BIN" -r --argjson ex "$existing" --arg grp "$grp" --argjson ws "$(ws_filter_json)" \ --slurpfile sum "${summary:-/dev/null}" ' ($sum[0] // {}) as $S | .[] | select(($ws | length) == 0 or (.ResourceName as $n | $ws | index($n) != null)) | ((.CLIConfiguration.TfStateFilePath // "") | sub(".*/"; "") | sub("\\.tfstate$"; "")) as $wsName | .ResourceName as $n - | [ $n, + | [ $n, $grp, (if ($ex | index($n)) != null then "update" else "create" end), - (.TerraformConfig.terraformVersion // "-"), - ((.RunnerConstraints // {}) | if .type == "private" then "private" else "shared" end), + (.TerraformConfig.terraformVersion // "preset"), + ((.RunnerConstraints // null) | if . == null then "preset" elif .type == "private" then "private" else "shared" end), (if (.VCSTriggers // null) != null then "yes" else "no" end), ((.VCSConfig.iacInputData.data // {}) | length), (($S.skippedSensitiveVars // {})[$wsName] // [] | length) ] | @tsv' "$f")" - while IFS=$'\t' read -r name action tfv runner trig vars secrets; do - [ -n "$name" ] || continue - if tf_version_above_ceiling "$tfv"; then tfv="${tfv#TERRAFORM-} -> ${SG_DEFAULT_TF_VERSION#TERRAFORM-} (fallback)"; else tfv="${tfv#TERRAFORM-}"; fi - [ "$secrets" = "0" ] && secrets="-" - case "$action" in create) action="${C_GREEN}create ${C_RESET}" ;; update) action="${C_YELLOW}update ${C_RESET}" ;; esac - printf ' %-28s %-24s %s %-28s %-8s %-8s %-5s %s\n' "$name" "$grp" "$action" "$tfv" "$runner" "$trig" "$vars" "$secrets" >&2 - done <<<"$rows" + [ -n "$rows" ] && all_rows="$all_rows${all_rows:+$'\n'}$rows" + names+=("$grp") done + while IFS=$'\t' read -r name grp _; do [ -n "$name" ] && names+=("$name"); done <<<"$all_rows" + wn="$(sg_maxlen 8 ${names[@]+"${names[@]}"})" + while IFS=$'\t' read -r _ grp _; do [ -n "$grp" ] && groups+=("$grp"); done <<<"$all_rows" + gn="$(sg_maxlen 5 ${groups[@]+"${groups[@]}"})" + # The RUNNER column only grows when a row defers to the preset ("preset (private:rg)"). + rw=8 + case "$all_rows" in *$'\t'preset$'\t'*) rw="$(sg_maxlen 8 "preset (${SG_PRESET_RUNNER_SHORT:-})")" ;; esac + printf "\n %s%-${wn}s %-${gn}s %-7s %-28s %-${rw}s %-8s %-5s %s%s\n" "$C_BOLD" "WORKFLOW" "GROUP" "ACTION" "TERRAFORM" "RUNNER" "TRIGGERS" "VARS" "SECRETS" "$C_RESET" >&2 + while IFS=$'\t' read -r name grp action tfv runner trig vars secrets; do + [ -n "$name" ] || continue + if [ "$tfv" = "preset" ]; then tfv="preset${SG_PRESET_TFV:+ ($SG_PRESET_TFV)}" + elif tf_version_above_ceiling "$tfv"; then tfv="${tfv#TERRAFORM-} -> ${SG_TF_FALLBACK_SHORT:-${SG_DEFAULT_TF_VERSION#TERRAFORM-}} (fallback)" + else tfv="${tfv#TERRAFORM-}"; fi + [ "$runner" = "preset" ] && runner="preset${SG_PRESET_RUNNER_SHORT:+ ($SG_PRESET_RUNNER_SHORT)}" + [ "$secrets" = "0" ] && secrets="-" + case "$action" in create) action="${C_GREEN}create ${C_RESET}" ;; update) action="${C_YELLOW}update ${C_RESET}" ;; esac + printf " %-${wn}s %-${gn}s %s %-28s %-${rw}s %-8s %-5s %s\n" "$name" "$grp" "$action" "$tfv" "$runner" "$trig" "$vars" "$secrets" >&2 + done <<<"$all_rows" echo >&2 - sg_dim "TERRAFORM '-> fallback' = pinned above SG's managed ceiling (1.5.7, last FOSS release); SECRETS = sensitive vars to recreate" + sg_dim "TERRAFORM '-> fallback' = pinned above SG's managed ceiling (1.5.7, last FOSS release); 'preset' = left to the org's execution preset at import; SECRETS = sensitive vars recreated as placeholder secrets" } diff --git a/scripts/lib/sg_api.sh b/scripts/lib/sg_api.sh index f165f6b..e5856b6 100644 --- a/scripts/lib/sg_api.sh +++ b/scripts/lib/sg_api.sh @@ -89,6 +89,65 @@ sg_list_workflows() { # sg_patch_workflow <group> <wf> <json> — PATCH a workflow. sg_patch_workflow() { sg_api_patch "$(wf_url "$1" "$2")" "$3"; } +# --- execution preset (org workflow defaults) ------------------------------ +# Settings -> Runner groups -> Execution presets is stored as the org's +# Settings.workflowDefaults: RunnerConstraints plus a TerraformConfig each for +# TERRAFORM and OPENTOFU workflows (terraformVersion, optional terraformBinPath / +# wfStepTemplateRevisionId). At workflow creation the API fills those keys from +# it when the payload does not carry them. + +# sg_execution_preset — the preset as compact JSON; "{}" when the org has none. +# Exit 1 when the org could not be read (the caller decides how loud to be). +sg_execution_preset() { + local body out + body="$(sg_api_get "$(sg_org_url)/" 2>/dev/null)" || { printf '{}'; return 1; } + out="$(printf '%s' "$body" | "$(sg_resolve jq sg_ensure_jq)" -c '(.msg // .data // {}) | (.Settings // {}).workflowDefaults // {}' 2>/dev/null)" + [ -n "$out" ] || out='{}' + printf '%s' "$out" +} + +# sg_preset_runner_desc <preset-json> — "shared runners" / "private runner group X". +sg_preset_runner_desc() { + local p="${1:-}" + [ -n "$p" ] || p='{}' + printf '%s' "$p" | "$(sg_resolve jq sg_ensure_jq)" -r ' + (.RunnerConstraints // {}) as $r + | if ($r.type // "shared") == "private" then "private runner group \(($r.names // []) | join(", "))" else "shared runners" end' 2>/dev/null +} + +# sg_preset_version_desc <preset-json> [TERRAFORM|OPENTOFU] — e.g. "Terraform 1.5.7", +# "Terraform 1.5.7 with runtime image /org/img:3", "runner-provided binary /usr/bin/terraform", +# or "managed Terraform 1.5.7 (platform default)" when the preset has no version. +sg_preset_version_desc() { + local p="${1:-}" + [ -n "$p" ] || p='{}' + printf '%s' "$p" | "$(sg_resolve jq sg_ensure_jq)" -r --arg t "${2:-TERRAFORM}" ' + (if $t == "OPENTOFU" then "OpenTofu" else "Terraform" end) as $tool + | ((if $t == "OPENTOFU" then .openTofuDefaults else .terraformDefaults end // {}).TerraformConfig // {}) as $c + | (if (($c.terraformBinPath // []) | length) > 0 then "runner-provided binary \($c.terraformBinPath[0].source // "")" + elif ($c.terraformVersion // "") == "" then (if $t == "OPENTOFU" then "managed OpenTofu (platform default)" else "managed Terraform 1.5.7 (platform default)" end) + else "\($tool) \($c.terraformVersion | ltrimstr("TERRAFORM-") | ltrimstr("OPENTOFU-"))" end) + + (if ($c.wfStepTemplateRevisionId // "") != "" then " with runtime image \($c.wfStepTemplateRevisionId)" else "" end)' 2>/dev/null +} + +# sg_preset_desc <preset-json> — one line: "Terraform 1.5.7 on shared runners"; +# "none configured (platform defaults: managed Terraform 1.5.7 on shared runners)" for {}. +sg_preset_desc() { + if [ -z "$1" ] || [ "$1" = "{}" ] || [ "$1" = "null" ]; then + printf 'none configured (platform defaults: managed Terraform 1.5.7 on shared runners)' + else + printf '%s on %s' "$(sg_preset_version_desc "$1")" "$(sg_preset_runner_desc "$1")" + fi +} + +# sg_preset_runner_provided <preset-json> — exit 0 when the Terraform defaults +# mount a runner-provided binary (terraformBinPath). +sg_preset_runner_provided() { + local p="${1:-}" + [ -n "$p" ] || p='{}' + printf '%s' "$p" | "$(sg_resolve jq sg_ensure_jq)" -e '((.terraformDefaults // {}).TerraformConfig.terraformBinPath // []) | length > 0' >/dev/null 2>&1 +} + # --- integrations (connectors) -------------------------------------------- # sg_list_integrations — [{name, type}, ...] for the org (fails on error). @@ -99,6 +158,25 @@ sg_list_integrations() { printf '%s' "$body" | "$(sg_resolve jq sg_ensure_jq)" -c '[(if (.msg | type) == "array" then .msg elif (.data | type) == "array" then .data elif type == "array" then . else [] end)[] | {name: (.ResourceName // .Id // ""), type: (.Settings.kind // .kind // .ResourceType // "")}] | map(select(.name != ""))' } +# sg_vcs_kind_of <connector-type> — the sourceConfigDestKind a VCS connector +# type implies (GITHUB_APP_CUSTOM -> GITHUB_COM, AZURE_DEVOPS_SP -> AZURE_DEVOPS, +# ...); empty for cloud connectors and unknown types. +sg_vcs_kind_of() { + case "$1" in + GITHUB_COM | GITHUB_APP_CUSTOM) printf 'GITHUB_COM' ;; + GITLAB_COM | GITLAB_OAUTH_SSH) printf 'GITLAB_COM' ;; + BITBUCKET_ORG) printf 'BITBUCKET_ORG' ;; + AZURE_DEVOPS | AZURE_DEVOPS_SP) printf 'AZURE_DEVOPS' ;; + GIT_OTHER) printf 'GIT_OTHER' ;; + *) printf '' ;; + esac +} + +# sg_integration_type <integrations-json> <name> — the connector's kind, or empty. +sg_integration_type() { + printf '%s' "$1" | "$(sg_resolve jq sg_ensure_jq)" -r --arg n "${2#/integrations/}" '[.[] | select(.name == $n) | .type][0] // empty' +} + # sg_integration_exists <name-or-/integrations/name> — exit 0 when it exists. sg_integration_exists() { local n="${1#/integrations/}" diff --git a/scripts/lib/tfc_api.sh b/scripts/lib/tfc_api.sh index 5a9c74e..ba0bc3c 100644 --- a/scripts/lib/tfc_api.sh +++ b/scripts/lib/tfc_api.sh @@ -78,10 +78,67 @@ tfc_list_orgs() { tfc_get_all "organizations" | "$(sg_resolve jq sg_ensure_jq)" # tfc_list_projects <org> — [{id, name}, ...] tfc_list_projects() { tfc_get_all "organizations/$1/projects" | "$(sg_resolve jq sg_ensure_jq)" -c '[.[] | {id: .id, name: .attributes.name}]'; } -# tfc_list_workspaces <org> — [{name, id, project, tags, terraform_version, execution_mode}, ...] +# tfc_list_workspaces <org> — [{name, id, project, tags, terraform_version, +# execution_mode, vcs_provider, vcs_url, vcs_identifier}, ...]. The vcs_* fields +# are empty for CLI-driven workspaces (no VCS connection). tfc_list_workspaces() { tfc_get_all "organizations/$1/workspaces" | "$(sg_resolve jq sg_ensure_jq)" -c \ - '[.[] | {name: .attributes.name, id: .id, project: (.relationships.project.data.id // ""), tags: (.attributes."tag-names" // []), terraform_version: .attributes."terraform-version", execution_mode: .attributes."execution-mode"}]' + '[.[] | {name: .attributes.name, id: .id, project: (.relationships.project.data.id // ""), tags: (.attributes."tag-names" // []), terraform_version: .attributes."terraform-version", execution_mode: .attributes."execution-mode", + vcs_provider: (.attributes."vcs-repo"."service-provider" // ""), vcs_url: (.attributes."vcs-repo"."repository-http-url" // ""), vcs_identifier: (.attributes."vcs-repo".identifier // "")}]' +} + +# tfc_select_workspaces <workspaces-json> <names-json> <tags-json> <ignore-json> +# — the subset the transformer exports, mirroring tfe_workspace_ids: name globs +# (["*"] = all), include tags (a workspace must carry all of them), exclude +# tags (any of them drops the workspace). null/[] disables a filter. +tfc_select_workspaces() { + printf '%s' "$1" | "$(sg_resolve jq sg_ensure_jq)" -c --argjson names "${2:-null}" --argjson tags "${3:-null}" --argjson ignore "${4:-null}" ' + def glob($p): ("^" + ($p | gsub("\\*"; ".*")) + "$"); + [ .[] + | select(($names == null) or ($names == ["*"]) or ([$names[] as $p | (.name | test(glob($p)))] | any)) + | select(($tags == null) or (($tags | length) == 0) or ([$tags[] as $t | ([.tags[]?] | index($t) != null)] | all)) + | select(($ignore == null) or (($ignore | length) == 0) or (([.tags[]?] | map(select(. as $t | $ignore | index($t) != null)) | length) == 0)) + ]' +} + +# tfc_vcs_kind_for <tfc-service-provider> — the SG sourceConfigDestKind that +# matches a TFC VCS provider (github, github_app, gitlab_hosted, ado_services, +# ...); empty when unknown. +tfc_vcs_kind_for() { + case "$1" in + github | github_app | github_enterprise) printf 'GITHUB_COM' ;; + gitlab_hosted | gitlab_community_edition | gitlab_enterprise_edition) printf 'GITLAB_COM' ;; + bitbucket_hosted | bitbucket_server | bitbucket_data_center) printf 'BITBUCKET_ORG' ;; + ado_services | ado_server) printf 'AZURE_DEVOPS' ;; + *) printf '' ;; + esac +} + +# tfc_vcs_label_for <tfc-service-provider> — human name (GitHub, GitLab, ...). +tfc_vcs_label_for() { + case "$1" in + github*) printf 'GitHub' ;; + gitlab*) printf 'GitLab' ;; + bitbucket*) printf 'Bitbucket' ;; + ado*) printf 'Azure DevOps' ;; + *) printf '%s' "$1" ;; + esac +} + +# tfc_vcs_summary <workspaces-json> — one line per provider in use: +# "<provider>\t<count>\t<repo-url-prefix or ->", most common first. The prefix +# is repository-http-url with the repo identifier stripped (https://github.com, +# https://gitlab.example.com, https://dev.azure.com, ...); "-" when the +# workspaces of that provider disagree. +tfc_vcs_summary() { + printf '%s' "$1" | "$(sg_resolve jq sg_ensure_jq)" -r ' + [ .[] | select(.vcs_provider != "") | . as $o + | { provider: .vcs_provider, + prefix: (if ($o.vcs_url | length) > 0 and ($o.vcs_identifier | length) > 0 and ($o.vcs_url | endswith("/" + $o.vcs_identifier)) + then ($o.vcs_url | rtrimstr("/" + $o.vcs_identifier)) + else (($o.vcs_url | capture("^(?<h>https?://[^/]+)").h) // "") end) } ] + | group_by(.provider) | sort_by(-length) + | .[] | "\(.[0].provider)\t\(length)\t\(([.[].prefix] | unique | map(select(. != ""))) as $p | if ($p | length) == 1 then $p[0] else "-" end)"' } # require_tfc_auth — fail fast when the tfe provider would not be able to diff --git a/scripts/lib/tfvars.sh b/scripts/lib/tfvars.sh index 6ec7412..a0f52ea 100644 --- a/scripts/lib/tfvars.sh +++ b/scripts/lib/tfvars.sh @@ -32,6 +32,18 @@ tfvars_get_json() { tfvars_json | "$(sg_resolve jq sg_ensure_jq)" -c "$1 // null" 2>/dev/null || echo null } +# tfvars_has <key> — exit 0 when the top-level key is present in the file (an +# explicit `key = null` counts as present; a missing key means the variable's +# default applies). +tfvars_has() { + tfvars_json | "$(sg_resolve jq sg_ensure_jq)" -e --arg k "$1" 'has($k)' >/dev/null 2>&1 +} + +# tfvars_is_null <key> — exit 0 when the file sets the key to null explicitly. +tfvars_is_null() { + tfvars_has "$1" && [ "$(tfvars_get_json ".$1")" = "null" ] +} + # tfvars_invalidate — forget the cached conversion (after writing the file). tfvars_invalidate() { _TFVARS_JSON=""; } @@ -46,24 +58,38 @@ tfvars_valid() { # _tfvars_hcl <json> — pretty-print a JSON value so it reads like HCL in the file. _tfvars_hcl() { printf '%s' "$1" | "$(sg_resolve jq sg_ensure_jq)" --indent 2 '.' 2>/dev/null || printf '%s' "$1"; } +# _tfvars_str <text> — a quoted HCL string literal (backslashes, quotes and +# template sequences escaped, so any connector or org name round-trips). +_tfvars_str() { + local s="$1" + s="${s//\\/\\\\}" + s="${s//\"/\\\"}" + s="${s//\$\{/\$\$\{}" + s="${s//%\{/%%\{}" + printf '"%s"' "$s" +} + # tfvars_write <dest> — render terraform.tfvars from W_* variables set by the # wizard (lists/objects are passed as compact JSON, which HCL accepts). Keeps # the same order and comments as terraform.tfvars.example so the file stays # hand-editable afterwards. # W_TFORG W_TFHOST W_WSNAMES_JSON W_TAGS_JSON W_IGNORE_TAGS_JSON W_EXPORT_STATE # W_APPROVERS_JSON W_REPO_PREFIX W_VCS_INTEGRATION W_DPC_JSON W_RUNNER_JSON -# W_DEST_KIND W_TF_VERSION W_TRIGGERS W_IGNORE_PATTERNS_JSON +# W_DEST_KIND W_TF_SOURCE W_TF_VERSION W_TRIGGERS W_IGNORE_PATTERNS_JSON +# W_RUNNER_JSON and W_TF_VERSION may be the literal "null" (defer to the org's +# execution preset). tfvars_write() { - local dest="$1" host_line="" + local dest="$1" host_line="" tf_version_hcl + if [ "${W_TF_VERSION:-null}" = "null" ]; then tf_version_hcl="null"; else tf_version_hcl="$(_tfvars_str "$W_TF_VERSION")"; fi if [ "${W_TFHOST:-app.terraform.io}" != "app.terraform.io" ]; then - host_line="$(printf '\n# Terraform Enterprise hostname (omit for Terraform Cloud)\ntfHostname = "%s"\n' "$W_TFHOST")" + host_line="$(printf '\n# Terraform Enterprise hostname (omit for Terraform Cloud)\ntfHostname = %s\n' "$(_tfvars_str "$W_TFHOST")")" fi cat >"$dest" <<TFVARS # Generated by 'sg-migrate.sh init' on $(date -u +%Y-%m-%dT%H:%M:%SZ). Safe to edit by hand; # re-run 'sg-migrate.sh init' to go through the wizard again. # Terraform Cloud/Enterprise organization name -tfOrg = "$W_TFORG" +tfOrg = $(_tfvars_str "$W_TFORG") $host_line # List of workspace names to export. Use wildcards (e.g., ["*"]) for all workspaces workspacenames = $W_WSNAMES_JSON @@ -89,25 +115,32 @@ ignoreVarPatterns = $W_IGNORE_PATTERNS_JSON SGDefaultWfApprovers = $W_APPROVERS_JSON # Prefix for your repo URL -SGDefaultIACVCSRepoPrefix = "$W_REPO_PREFIX" +SGDefaultIACVCSRepoPrefix = $(_tfvars_str "$W_REPO_PREFIX") # VCS connector used to clone the repositories (/integrations/<name>) -SGDefaultVCSAuthIntegrationID = "$W_VCS_INTEGRATION" +SGDefaultVCSAuthIntegrationID = $(_tfvars_str "$W_VCS_INTEGRATION") # Cloud connector the workflows deploy with SGDefaultDeploymentPlatformConfig = $(_tfvars_hcl "$W_DPC_JSON") -# Runners for every workflow: { type = "shared" } for SG-hosted runners, or -# { type = "private", names = ["<runner-group>"] } for a private runner group. -SGDefaultRunnerConstraints = $(_tfvars_hcl "$W_RUNNER_JSON") +# Runners for every workflow: { type = "shared" } for SG-hosted runners, +# { type = "private", names = ["<runner-group>"] } for a private runner group, or +# null to let the org's execution preset decide (Settings -> Runner groups). +SGDefaultRunnerConstraints = $(_tfvars_hcl "${W_RUNNER_JSON:-null}") # Choose from: GITHUB_COM, BITBUCKET_ORG, GITLAB_COM, AZURE_DEVOPS, GIT_OTHER -SGDefaultSourceConfigDestKind = "$W_DEST_KIND" - -# SG Terraform version used when a workspace's version is not a pinned semver, or -# when the SG API rejects a pinned version as above the managed ceiling (1.5.7, -# the last MPL/FOSS release; newer versions are BSL and not bundled). -SGDefaultTerraformVersion = "$W_TF_VERSION" +SGDefaultSourceConfigDestKind = $(_tfvars_str "$W_DEST_KIND") + +# Where the workflows get their Terraform version from: "carry" keeps each +# workspace's pinned TFC version (the fallback below covers the rest); "preset" +# sends no version, so the org's execution preset applies. +SGTerraformVersionSource = $(_tfvars_str "${W_TF_SOURCE:-carry}") + +# Fallback for "carry": SG Terraform version used when a workspace's version is +# not a pinned semver, or when the SG API rejects a pinned version as above the +# managed ceiling (1.5.7, the last MPL/FOSS release; newer versions are BSL and +# not bundled). null = leave those workflows to the org's execution preset. +SGDefaultTerraformVersion = $tf_version_hcl # Pre-configure VCS triggers on each workflow from the workspace's TFC settings SGDefaultEnableVCSTriggers = $W_TRIGGERS diff --git a/scripts/lib/wizard.sh b/scripts/lib/wizard.sh index 80bb2d8..2398678 100644 --- a/scripts/lib/wizard.sh +++ b/scripts/lib/wizard.sh @@ -1,9 +1,16 @@ #!/bin/bash # Interactive 'init' wizard (sourced; needs tools.sh, prompt.sh, tfvars.sh, -# tfc_api.sh, sg_api.sh). Discovers TFC orgs/workspaces and SG integrations / -# runner groups with the tokens already in the environment and writes -# terraform.tfvars. Every step degrades to free-text entry when a token is +# tfc_api.sh, sg_api.sh, state.sh). Discovers TFC orgs/workspaces and SG +# integrations / runner groups with the tokens already in the environment and +# writes terraform.tfvars. Every step degrades to free-text entry when a token is # missing or an API call fails, so the wizard always completes. +# +# What TFC knows about the workspaces steers the StackGuardian step: the VCS +# provider each workspace is connected to ranks the matching connectors first, +# and the repositories' URL gives the repo URL prefix (so a rerun that switches +# connector never keeps a prefix from the previous provider). +# +# shellcheck disable=SC2034 # the W_* results are consumed by tfvars_write (lib/tfvars.sh) # _w_default <jq-expr> <fallback> — current tfvars value (re-run) or fallback. _w_default() { local v; v="$(tfvars_get "$1")"; printf '%s' "${v:-$2}"; } @@ -17,7 +24,11 @@ _w_csv_json() { printf '%s' "${out:-[]}" } -# _w_repo_prefix_for <sourceConfigDestKind> — proposed repo URL prefix. +# _w_csv <json-array> — ["a","b"] -> "a, b" (for the review). +_w_csv() { printf '%s' "$1" | "$(sg_resolve jq sg_ensure_jq)" -r 'if type == "array" then join(", ") else tostring end' 2>/dev/null; } + +# _w_repo_prefix_for <sourceConfigDestKind> — the provider's well-known host, +# used only when TFC does not tell us where the repositories live. _w_repo_prefix_for() { case "$1" in GITHUB_COM) printf 'https://github.com' ;; @@ -28,9 +39,40 @@ _w_repo_prefix_for() { esac } +# _w_host <url> — "https://www.github.com/x" -> "www.github.com". +_w_host() { local h="${1#*://}"; printf '%s' "${h%%/*}"; } + +# _w_select_lines <question> <lines> [extra-items...] — sg_select (with an +# "Other" entry) over newline-separated "value|desc" items; names may contain +# spaces or glob characters, so no word splitting. +_w_select_lines() { + local q="$1" lines="$2" line + shift 2 + local -a items=() + while IFS= read -r line; do [ -n "$line" ] && items+=("$line"); done <<<"$lines" + SG_SELECT_OTHER=1 sg_select "$q" "${items[@]}" "$@" +} + +# _w_project_counts <workspaces-json> <projects-json> <groups:0|1> — +# "Default Project (2), Team A (1)" or, as SG groups, "tfc-default-project (2), ...". +_w_project_counts() { + printf '%s' "$1" | "$(sg_resolve jq sg_ensure_jq)" -r --argjson pr "$2" --argjson g "$3" ' + ($pr | map({key: .id, value: .name}) | from_entries) as $names + | group_by(.project) | sort_by(-length) + | map((($names[.[0].project] // .[0].project) as $n + | if $g == 1 then "tfc-" + ($n | ascii_downcase | gsub("[^a-z0-9-]+"; "-")) else $n end) + " (\(length))") + | join(", ")' 2>/dev/null +} + +# _w_tfc_kind_matches <sg-kind> — exit 0 when the TFC workspaces use that provider. +_w_tfc_kind_matches() { + [ -n "$1" ] || return 1 + case " ${W_TFC_VCS_KINDS:-} " in *" $1 "*) return 0 ;; *) return 1 ;; esac +} + # --- step 1: Terraform Cloud ------------------------------------------------ wizard_tfc() { - local host token orgs n ws projects wsn prn scope tags alltags + local host token orgs n ws projects wsn prn scope tags alltags sel prov cnt prefix kind local jqb jqb="$(sg_resolve jq sg_ensure_jq)" sg_step "1/4 Terraform Cloud / Enterprise" @@ -54,8 +96,7 @@ wizard_tfc() { W_TFORG="$(printf '%s' "$orgs" | "$jqb" -r '.[0]')" sg_log "organisation: $W_TFORG (the only one this token can see)" else - # shellcheck disable=SC2046 - W_TFORG="$(SG_SELECT_OTHER=1 sg_select "Which TFC organisation do you want to migrate?" $(printf '%s' "$orgs" | "$jqb" -r '.[]'))" || return 1 + W_TFORG="$(_w_select_lines "Which TFC organisation do you want to migrate?" "$(printf '%s' "$orgs" | "$jqb" -r '.[]')")" || return 1 fi else W_TFORG="$(sg_ask "TFC organisation name" "$(_w_default .tfOrg '')")" || return 1 @@ -65,13 +106,23 @@ wizard_tfc() { W_WSNAMES_JSON='["*"]' W_TAGS_JSON=null W_IGNORE_TAGS_JSON=null + W_TFC_WS_JSON="" + W_WS_COUNT="" + W_WS_TOTAL="" + W_WS_ABOVE_CEILING="" + W_GROUPS="" + W_TFC_VCS="" + W_TFC_VCS_KINDS="" + W_TFC_REPO_PREFIX="" + W_TFC_VCS_OTHER=0 + projects='[]' if [ "$W_TFC_DISCOVERY" -eq 1 ] && ws="$(tfc_list_workspaces "$W_TFORG" 2>/dev/null)"; then + W_TFC_WS_JSON="$ws" wsn="$(printf '%s' "$ws" | "$jqb" 'length')" projects="$(tfc_list_projects "$W_TFORG" 2>/dev/null || echo '[]')" prn="$(printf '%s' "$projects" | "$jqb" 'length')" sg_log "found $wsn workspace(s) in $prn project(s); each project becomes SG workflow group tfc-<project>" - W_WS_COUNT="$wsn" - W_WS_ABOVE_CEILING="$(printf '%s' "$ws" | "$jqb" '[.[] | select((.terraform_version // "") | test("^[0-9]+\\.[0-9]+\\.[0-9]+$")) | select(((.terraform_version | split(".") | map(tonumber)) as $v | ($v[0] > 1) or ($v[0] == 1 and $v[1] > 5) or ($v[0] == 1 and $v[1] == 5 and $v[2] > 7)))] | length')" + [ "$prn" -le 8 ] && [ "$wsn" -gt 0 ] && sg_dim "$(_w_project_counts "$ws" "$projects" 0)" alltags="$(printf '%s' "$ws" | "$jqb" -r '[.[].tags[]?] | unique | join(", ")')" else alltags="" @@ -81,6 +132,7 @@ wizard_tfc() { "tags|only workspaces carrying certain tags" \ "exclude|all workspaces except those carrying certain tags" \ "names|specific workspace names")" || return 1 + W_SCOPE="$scope" case "$scope" in tags) [ -n "$alltags" ] && sg_dim "tags in use: $alltags" @@ -97,11 +149,41 @@ wizard_tfc() { W_WSNAMES_JSON="$(_w_csv_json "$tags")" ;; esac + + # What the selection looks like (drives the review and the SG step's hints). + if [ -n "$W_TFC_WS_JSON" ]; then + sel="$(tfc_select_workspaces "$W_TFC_WS_JSON" "$W_WSNAMES_JSON" "$W_TAGS_JSON" "$W_IGNORE_TAGS_JSON")" + W_WS_TOTAL="$wsn" + W_WS_COUNT="$(printf '%s' "$sel" | "$jqb" 'length')" + W_WS_ABOVE_CEILING="$(printf '%s' "$sel" | "$jqb" '[.[] | select((.terraform_version // "") | test("^[0-9]+\\.[0-9]+\\.[0-9]+$")) | select(((.terraform_version | split(".") | map(tonumber)) as $v | ($v[0] > 1) or ($v[0] == 1 and $v[1] > 5) or ($v[0] == 1 and $v[1] == 5 and $v[2] > 7)))] | length')" + W_GROUPS="$(_w_project_counts "$sel" "$projects" 1)" + if [ "$scope" != "all" ]; then + if [ "$W_WS_COUNT" -eq 0 ]; then + sg_warn "no workspace matches that selection — 'apply' would export nothing" + else + sg_log "$W_WS_COUNT of $wsn workspace(s) match" + fi + fi + # VCS providers in use, most common first; the first one's repo URL prefix + # becomes the default for every workflow (others need workspaceOverrides). + W_TFC_VCS="$(tfc_vcs_summary "$sel")" + while IFS=$'\t' read -r prov cnt prefix; do + [ -n "$prov" ] || continue + kind="$(tfc_vcs_kind_for "$prov")" + [ -n "$kind" ] && W_TFC_VCS_KINDS="$W_TFC_VCS_KINDS $kind" + if [ -z "$W_TFC_REPO_PREFIX" ]; then + [ "$prefix" != "-" ] && W_TFC_REPO_PREFIX="$prefix" + else + W_TFC_VCS_OTHER=$((W_TFC_VCS_OTHER + cnt)) + fi + done <<<"$W_TFC_VCS" + fi + return 0 } # --- step 2: StackGuardian --------------------------------------------------- wizard_sg() { - local ints vcs cloud pick kind name runner groups jqb + local ints vcs cloud pick kind name runner groups jqb hint prov cnt prefix prev_kind prev_prefix line k jqb="$(sg_resolve jq sg_ensure_jq)" sg_step "2/4 StackGuardian" if [ -z "$ORG" ]; then @@ -122,37 +204,71 @@ wizard_sg() { W_SG_DISCOVERY=1 fi - # VCS connector -> integration id, source kind, repo prefix. + # VCS connector -> integration id, source kind, repo prefix. Connectors of the + # provider the TFC workspaces are connected to come first (and are the default). + if [ -n "${W_TFC_VCS:-}" ]; then + hint="" + while IFS=$'\t' read -r prov cnt prefix; do + [ -n "$prov" ] || continue + hint="$hint${hint:+, }$(tfc_vcs_label_for "$prov") ($cnt)" + done <<<"$W_TFC_VCS" + sg_dim "the selected TFC workspaces are connected to: $hint — matching connectors are listed first" + elif [ -n "${W_TFC_WS_JSON:-}" ] && [ "${W_WS_COUNT:-0}" -gt 0 ]; then + sg_dim "none of the selected TFC workspaces is VCS-connected (CLI-driven); pick the connector for the repositories anyway" + fi vcs="" - [ "$W_SG_DISCOVERY" -eq 1 ] && vcs="$(printf '%s' "$ints" | "$jqb" -r '[.[] | select((.type // "") | IN("GITHUB_COM","GITHUB_APP_CUSTOM","GITLAB_COM","GITLAB_OAUTH_SSH","BITBUCKET_ORG","AZURE_DEVOPS","AZURE_DEVOPS_SP","GIT_OTHER"))] | sort_by(.name) | .[] | "\(.name)|\(.type)"')" + if [ "$W_SG_DISCOVERY" -eq 1 ]; then + local -a first=() rest=() + while IFS='|' read -r name kind; do + [ -n "$name" ] || continue + if _w_tfc_kind_matches "$(sg_vcs_kind_of "$kind")"; then first+=("$name|$kind"); else rest+=("$name|$kind"); fi + done <<<"$(printf '%s' "$ints" | "$jqb" -r '[.[] | select((.type // "") | IN("GITHUB_COM","GITHUB_APP_CUSTOM","GITLAB_COM","GITLAB_OAUTH_SSH","BITBUCKET_ORG","AZURE_DEVOPS","AZURE_DEVOPS_SP","GIT_OTHER"))] | sort_by(.name) | .[] | "\(.name)|\(.type)"')" + vcs="$(printf '%s\n' ${first[@]+"${first[@]}"} ${rest[@]+"${rest[@]}"})" + fi if [ -n "$vcs" ]; then - # shellcheck disable=SC2046 - pick="$(SG_SELECT_OTHER=1 sg_select "Which VCS connector should clone the repositories?" $(printf '%s\n' "$vcs" | tr '\n' ' '))" || return 1 - kind="$(printf '%s' "$ints" | "$jqb" -r --arg n "$pick" '.[] | select(.name == $n) | .type' | head -1)" + pick="$(_w_select_lines "Which VCS connector should clone the repositories?" "$vcs")" || return 1 + kind="$(sg_integration_type "$ints" "$pick")" else pick="$(sg_ask "VCS connector name (as in StackGuardian, e.g. github_com)" "$(_w_default .SGDefaultVCSAuthIntegrationID '' | sed 's#^/integrations/##')")" || return 1 [ -n "$pick" ] || pick="$(sg_ask_required "VCS connector name")" || return 1 kind="" fi W_VCS_INTEGRATION="/integrations/${pick#/integrations/}" - case "$kind" in - GITHUB_APP_CUSTOM) W_DEST_KIND=GITHUB_COM ;; - GITLAB_OAUTH_SSH) W_DEST_KIND=GITLAB_COM ;; - AZURE_DEVOPS_SP) W_DEST_KIND=AZURE_DEVOPS ;; - GITHUB_COM | GITLAB_COM | BITBUCKET_ORG | AZURE_DEVOPS | GIT_OTHER) W_DEST_KIND="$kind" ;; - *) W_DEST_KIND="$(sg_select "VCS provider kind" GITHUB_COM GITLAB_COM BITBUCKET_ORG AZURE_DEVOPS GIT_OTHER)" || return 1 ;; - esac - # Repo URL prefix follows the connector kind; editable in terraform.tfvars. - W_REPO_PREFIX="$(_w_default .SGDefaultIACVCSRepoPrefix "$(_w_repo_prefix_for "$W_DEST_KIND")")" + W_DEST_KIND="$(sg_vcs_kind_of "$kind")" + if [ -z "$W_DEST_KIND" ]; then + # Unknown connector type (or no discovery): ask, defaulting to what TFC uses. + local -a kinds=(GITHUB_COM GITLAB_COM BITBUCKET_ORG AZURE_DEVOPS GIT_OTHER) + for k in ${W_TFC_VCS_KINDS:-}; do kinds=("$k" "${kinds[@]}"); break; done + W_DEST_KIND="$(sg_select "VCS provider kind" "${kinds[@]}")" || return 1 + fi + if [ -n "${W_TFC_VCS_KINDS:-}" ] && ! _w_tfc_kind_matches "$W_DEST_KIND"; then + sg_warn "the TFC workspaces are connected to $(tfc_vcs_label_for "${W_TFC_VCS%% *}") but '$pick' is a $W_DEST_KIND connector — the workflows will not be able to clone unless the repositories moved" + fi + # Repo URL prefix: where TFC says the repositories live; otherwise the previous + # value (only if the provider kind did not change — this is what used to leave + # a stale prefix behind); otherwise the provider's well-known host. + prev_kind="$(tfvars_get .SGDefaultSourceConfigDestKind)" + prev_prefix="$(tfvars_get .SGDefaultIACVCSRepoPrefix)" + [ "$prev_kind" = "$W_DEST_KIND" ] || prev_prefix="" + if [ -n "${W_TFC_REPO_PREFIX:-}" ]; then + # Keep a hand-tuned variant of the same host (e.g. www.github.com). + case "$(_w_host "$prev_prefix")" in + *"$(_w_host "$W_TFC_REPO_PREFIX")") [ -n "$prev_prefix" ] && W_REPO_PREFIX="$prev_prefix" || W_REPO_PREFIX="$W_TFC_REPO_PREFIX" ;; + *) W_REPO_PREFIX="$W_TFC_REPO_PREFIX" ;; + esac + elif [ -n "$prev_prefix" ]; then + W_REPO_PREFIX="$prev_prefix" + else + W_REPO_PREFIX="$(_w_repo_prefix_for "$W_DEST_KIND")" + fi # Cloud connector -> DeploymentPlatformConfig. cloud="" # Only kinds DeploymentPlatformConfig accepts (AZURE_DEVOPS* are VCS connectors). - [ "$W_SG_DISCOVERY" -eq 1 ] && cloud="$(printf '%s' "$ints" | "$jqb" -r '[.[] | select((.type // "") | IN("AWS_STATIC","AWS_RBAC","AWS_OIDC","AZURE_STATIC","AZURE_OIDC","AZURE_MANAGED_ID_OIDC","GCP_STATIC","GCP_OIDC"))] | sort_by(.name) | .[] | "\(.name)|\(.type)"')" + [ "$W_SG_DISCOVERY" -eq 1 ] && cloud="$(printf '%s' "$ints" | "$jqb" -r '[.[] | select((.type // "") | IN("AWS_STATIC","AWS_RBAC","AWS_OIDC","AZURE_STATIC","AZURE_OIDC","AZURE_MANAGED_ID_OIDC","GCP_STATIC","GCP_OIDC"))] | sort_by(.type, .name) | .[] | "\(.name)|\(.type)"')" if [ -n "$cloud" ]; then - # shellcheck disable=SC2046 - pick="$(SG_SELECT_OTHER=1 sg_select "Which cloud connector should the workflows deploy with?" $(printf '%s\n' "$cloud" | tr '\n' ' ') "skip|decide later (leaves a placeholder to edit)")" || return 1 - kind="$(printf '%s' "$ints" | "$jqb" -r --arg n "$pick" '.[] | select(.name == $n) | .type' | head -1)" + pick="$(_w_select_lines "Which cloud connector should the workflows deploy with?" "$cloud" "skip|decide later (leaves a placeholder to edit)")" || return 1 + kind="$(sg_integration_type "$ints" "$pick")" else pick="$(sg_ask "Cloud connector name (as in StackGuardian; empty to decide later)" "$(_w_default .SGDefaultDeploymentPlatformConfig[0].config.integrationId '' | sed 's#^/integrations/##')")" || return 1 kind="" @@ -160,22 +276,53 @@ wizard_sg() { if [ -z "$pick" ] || [ "$pick" = "skip" ]; then W_DPC_JSON='[{"kind":"AWS_RBAC","config":{"integrationId":"/integrations/CHANGE_ME"}}]' W_DPC_PLACEHOLDER=1 + W_CLOUD_DESC="none yet — a placeholder is written; edit it before 'apply'" else [ -n "$kind" ] || kind="$(sg_select "Connector kind" AWS_RBAC AWS_STATIC AWS_OIDC AZURE_STATIC AZURE_OIDC AZURE_MANAGED_ID_OIDC GCP_STATIC GCP_OIDC)" || return 1 W_DPC_JSON="$("$jqb" -nc --arg k "$kind" --arg i "/integrations/${pick#/integrations/}" '[{kind:$k, config:{integrationId:$i}}]')" W_DPC_PLACEHOLDER=0 + W_CLOUD_DESC="${pick#/integrations/} ($kind)" + fi + + # The org's execution preset (Settings -> Runner groups -> Execution presets): + # what StackGuardian fills in for runners / Terraform version when the payload + # carries none. Read once here; steps 2 and 3 offer it as a choice. + W_PRESET_JSON='{}' + W_PRESET_READ=0 + W_PRESET_RUNNER="" + W_PRESET_TFVER="" + if [ "$W_SG_DISCOVERY" -eq 1 ]; then + if W_PRESET_JSON="$(sg_execution_preset)"; then + W_PRESET_READ=1 + W_PRESET_RUNNER="$(sg_preset_runner_desc "$W_PRESET_JSON")" + W_PRESET_TFVER="$(sg_preset_version_desc "$W_PRESET_JSON")" + if [ "$W_PRESET_JSON" = "{}" ]; then + sg_dim "execution preset: none configured in $ORG — StackGuardian's platform defaults apply ($W_PRESET_TFVER on $W_PRESET_RUNNER)" + else + sg_dim "execution preset of $ORG: $W_PRESET_TFVER on $W_PRESET_RUNNER" + fi + fi fi - # Runner constraints. - runner="$(sg_select "Where should the workflows run?" \ - "shared|StackGuardian-hosted shared runners" \ - "private|a private runner group in your own network")" || return 1 - if [ "$runner" = "private" ]; then + # Runner constraints: the org's execution preset (first when we could read it + # and nothing explicit was configured before), SG shared runners, or a private + # runner group. + local -a runner_items=("shared|StackGuardian-hosted shared runners" "private|a private runner group in your own network") + local preset_item="preset|whatever the org's execution preset says${W_PRESET_RUNNER:+ (now: $W_PRESET_RUNNER)}" + if tfvars_is_null SGDefaultRunnerConstraints || { [ "$W_PRESET_READ" -eq 1 ] && ! tfvars_has SGDefaultRunnerConstraints; }; then + runner_items=("$preset_item" "${runner_items[@]}") + else + runner_items+=("$preset_item") + fi + runner="$(sg_select "Where should the workflows run?" "${runner_items[@]}")" || return 1 + if [ "$runner" = "preset" ]; then + W_RUNNER_JSON="null" + W_RUNNER_DESC="from the org's execution preset${W_PRESET_RUNNER:+ (now: $W_PRESET_RUNNER)}" + elif [ "$runner" = "private" ]; then groups="" [ "$W_SG_DISCOVERY" -eq 1 ] && groups="$(sg_list_runnergroups 2>/dev/null | "$jqb" -r 'sort | .[]' 2>/dev/null || true)" if [ -n "$groups" ]; then - # shellcheck disable=SC2046 - name="$(SG_SELECT_OTHER=1 sg_select "Which runner group?" $(printf '%s\n' "$groups" | tr '\n' ' '))" || return 1 + name="$(_w_select_lines "Which runner group?" "$groups")" || return 1 else name="$(sg_ask_required "Runner group name")" || return 1 fi @@ -183,8 +330,10 @@ wizard_sg() { sg_warn "runner group '$name' was not found in org '$ORG' — preflight will fail until it exists" fi W_RUNNER_JSON="$("$jqb" -nc --arg n "$name" '{type:"private", names:[$n]}')" + W_RUNNER_DESC="private runner group '$name'" else W_RUNNER_JSON='{"type":"shared"}' + W_RUNNER_DESC="StackGuardian shared runners" fi } @@ -195,7 +344,32 @@ wizard_policy() { # with sensible defaults — edit them in terraform.tfvars if needed. W_APPROVERS_JSON="$(tfvars_get_json .SGDefaultWfApprovers)" [ "$W_APPROVERS_JSON" = "null" ] && W_APPROVERS_JSON='[]' - W_TF_VERSION="$(_w_default .SGDefaultTerraformVersion TERRAFORM-1.5.7)" + + # Terraform version: carry the TFC pins (plus a fallback for the rest) or send + # none and let the org's execution preset decide. Previous choices come first. + local src fb prev_src prev_fb fixed above_hint="" + local -a src_items fb_items + [ "${W_WS_ABOVE_CEILING:-0}" -gt 0 ] && above_hint=" ($W_WS_ABOVE_CEILING pinned above SG's 1.5.7 ceiling would use the fallback)" + src_items=("carry|keep each workspace's pinned TFC version$above_hint" + "preset|no version at all: the org's execution preset applies${W_PRESET_TFVER:+ (now: $W_PRESET_TFVER)}") + prev_src="$(_w_default .SGTerraformVersionSource carry)" + [ "$prev_src" = "preset" ] && src_items=("${src_items[1]}" "${src_items[0]}") + src="$(sg_select "Which Terraform version should the migrated workflows run?" "${src_items[@]}")" || return 1 + W_TF_SOURCE="$src" + if [ "$src" = "preset" ]; then + W_TF_VERSION="null" + else + if tfvars_is_null SGDefaultTerraformVersion; then prev_fb="null"; else prev_fb="$(_w_default .SGDefaultTerraformVersion TERRAFORM-1.5.7)"; fi + [ "$prev_fb" = "null" ] && fixed="TERRAFORM-1.5.7" || fixed="TERRAFORM-${prev_fb#TERRAFORM-}" + fb_items=("${fixed#TERRAFORM-}|a fixed version ($fixed; StackGuardian bundles Terraform up to 1.5.7)" + "preset|the org's execution preset${W_PRESET_TFVER:+ (now: $W_PRESET_TFVER)}") + [ "$prev_fb" = "null" ] && fb_items=("${fb_items[1]}" "${fb_items[0]}") + fb="$(sg_select "Fallback for workspaces without a pinned version, and for pins StackGuardian rejects (above 1.5.7)?" "${fb_items[@]}")" || return 1 + case "$fb" in + preset) W_TF_VERSION="null" ;; + *) W_TF_VERSION="TERRAFORM-${fb#TERRAFORM-}" ;; + esac + fi if sg_confirm "Export Terraform state for each workspace?" "$([ "$(_w_default .exportStateFiles true)" = "false" ] && echo N || echo Y)"; then W_EXPORT_STATE=true; else W_EXPORT_STATE=false; fi if sg_confirm "Pre-configure VCS triggers (push / pull-request runs) from the TFC settings?" "$([ "$(_w_default .SGDefaultEnableVCSTriggers true)" = "false" ] && echo N || echo Y)"; then W_TRIGGERS=true; else W_TRIGGERS=false; fi sg_dim "TFC_* / TFE_* variables (e.g. TFC_WORKSPACE_NAME, TFC_AWS_RUN_ROLE_ARN) only mean something inside Terraform Cloud." @@ -208,29 +382,55 @@ wizard_policy() { # --- step 4: review + write ---------------------------------------------------- wizard_review() { + local scope tf yn_state yn_trig strip sg_step "4/4 Review" - row() { printf ' %s%-28s%s %s\n' "$C_BOLD" "$1" "$C_RESET" "$2" >&2; } - row "TFC host / org" "$W_TFHOST / $W_TFORG" - row "Workspaces" "$W_WSNAMES_JSON tags=$W_TAGS_JSON ignore=$W_IGNORE_TAGS_JSON${W_WS_COUNT:+ ($W_WS_COUNT found)}" - row "SG org" "$ORG" - row "VCS connector" "$W_VCS_INTEGRATION ($W_DEST_KIND, $W_REPO_PREFIX)" - row "Cloud connector" "$W_DPC_JSON" - row "Runners" "$W_RUNNER_JSON" - row "State export / triggers" "$W_EXPORT_STATE / $W_TRIGGERS" - row "Strip variables matching" "$W_IGNORE_PATTERNS_JSON" - row "Fallback Terraform" "$W_TF_VERSION${W_WS_ABOVE_CEILING:+ ($W_WS_ABOVE_CEILING workspace(s) pinned above 1.5.7 (the last FOSS runtime SG bundles) will use it)}" + sg_row "TFC" "$W_TFHOST / $W_TFORG" + case "${W_SCOPE:-all}" in + tags) scope="workspaces tagged $(_w_csv "$W_TAGS_JSON")" ;; + exclude) scope="all workspaces except those tagged $(_w_csv "$W_IGNORE_TAGS_JSON")" ;; + names) scope="workspaces named $(_w_csv "$W_WSNAMES_JSON")" ;; + *) scope="all workspaces" ;; + esac + if [ -n "${W_WS_COUNT:-}" ]; then + if [ "${W_SCOPE:-all}" = "all" ]; then scope="$scope ($W_WS_COUNT)"; else scope="$scope — $W_WS_COUNT of $W_WS_TOTAL match"; fi + fi + sg_row "Workspaces" "$scope" + [ -n "${W_GROUPS:-}" ] && sg_row "Workflow groups" "$W_GROUPS" + sg_row "StackGuardian org" "$ORG" + sg_row "VCS connector" "${W_VCS_INTEGRATION#/integrations/} ($W_DEST_KIND) — repositories under $W_REPO_PREFIX" + sg_row "Cloud connector" "$W_CLOUD_DESC" + sg_row "Runners" "$W_RUNNER_DESC" + [ "$W_EXPORT_STATE" = "true" ] && yn_state=yes || yn_state=no + [ "$W_TRIGGERS" = "true" ] && yn_trig="yes, from each workspace's TFC settings" || yn_trig=no + sg_row "State export" "$yn_state" + sg_row "VCS triggers" "$yn_trig" + [ "$W_IGNORE_PATTERNS_JSON" = "[]" ] && strip="none" || strip="TFC_*, TFE_*" + sg_row "Strip variables" "$strip" + if [ "${W_TF_SOURCE:-carry}" = "preset" ]; then + tf="from the org's execution preset${W_PRESET_TFVER:+ (now: $W_PRESET_TFVER)}" + else + if [ "${W_TF_VERSION:-null}" = "null" ]; then + tf="carried from TFC; unpinned workspaces and pins above 1.5.7 go to the execution preset${W_PRESET_TFVER:+ (now: $W_PRESET_TFVER)}" + else + tf="carried from TFC; fallback ${W_TF_VERSION#TERRAFORM-} for unpinned workspaces and pins above 1.5.7" + fi + [ "${W_WS_ABOVE_CEILING:-0}" -gt 0 ] && tf="$tf — $W_WS_ABOVE_CEILING workspace(s) affected" + fi + sg_row "Terraform version" "$tf" [ "${W_DPC_PLACEHOLDER:-0}" -eq 1 ] && sg_warn "cloud connector left as a placeholder — edit SGDefaultDeploymentPlatformConfig in $(sg_rel "$TFVARS") before 'apply'" - sg_dim "approvers, repo URL prefix and the fallback version can be edited in $(sg_rel "$TFVARS")" + [ "${W_TFC_VCS_OTHER:-0}" -gt 0 ] && sg_warn "$W_TFC_VCS_OTHER workspace(s) use a different VCS provider than the default above — give them their own connector/prefix via workspaceOverrides in $(sg_rel "$TFVARS")" + sg_dim "approvers and the repo URL prefix can be edited in $(sg_rel "$TFVARS")" sg_confirm "Write $(sg_rel "$TFVARS")?" Y } # wizard_run — the whole flow; returns non-zero when aborted. wizard_run() { + local kept="" wizard_tfc && wizard_sg && wizard_policy || { sg_err "init aborted"; return 1; } wizard_review || { sg_log "nothing written"; return 1; } if [ -f "$TFVARS" ]; then cp "$TFVARS" "$TFVARS.bak" - sg_log "previous file kept as $(sg_rel "$TFVARS.bak")" + kept=" (previous version kept as $(basename "$TFVARS").bak)" fi tfvars_write "$TFVARS" if ! tfvars_valid; then @@ -240,5 +440,5 @@ wizard_run() { # Remember the SG org (and API host) for later phases, so users don't have to # export SG_ORG again in a new shell. Tokens are never stored. state_update '.config = ((.config // {}) + {sg_org: $o, sg_base_url: $u})' --arg o "$ORG" --arg u "$SG_BASE_URL" - sg_success "wrote $(sg_rel "$TFVARS")" + sg_success "wrote $(sg_rel "$TFVARS")$kept" } diff --git a/scripts/migrate.sh b/scripts/migrate.sh index 6fad061..b9e8c9b 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -61,6 +61,12 @@ TF_PARALLELISM="${SG_TF_PARALLELISM:-20}" RETRIES="${SG_RETRIES:-4}" RETRY_BASE="${SG_RETRY_BASE:-2}" PF=() +# Phase bookkeeping: 'all' numbers its phases ("Phase 2/5: ...") and every +# phase reports how long it took. +PHASE_TOTAL=0 +PHASE_N=0 +PHASE_T0=$SECONDS +RUN_T0=$SECONDS # The init wizard remembers the SG org / API host in .sg/state.json so a new # shell without SG_ORG still works; flags and env always win. if [ -z "$ORG" ] && [ -f "$STATE_FILE" ]; then @@ -130,31 +136,73 @@ die() { exit 1 } -throttle() { while [ "$(jobs -rp | wc -l | tr -d ' ')" -ge "$1" ]; do sleep 0.2; done; } +# phase_begin <title> — "==> Phase 2/5: title" inside 'all', "==> Phase: title" +# for a single command; starts the phase timer. +phase_begin() { + PHASE_T0=$SECONDS + if [ "$PHASE_TOTAL" -gt 0 ]; then + PHASE_N=$((PHASE_N + 1)) + sg_step "Phase $PHASE_N/$PHASE_TOTAL: $1" + else + sg_step "Phase: $1" + fi +} +# phase_took — "12s" / "1m 04s" since phase_begin. +phase_took() { sg_fmt_secs $((SECONDS - PHASE_T0)); } + +# Ctrl-C: end the live progress line cleanly and say what happens next, instead +# of a bare "^C" (a background terraform/sg-cli gets the same SIGINT from the +# terminal, so nothing keeps running). +trap 'sg_spin_clear; printf "\n" >&2; sg_warn "interrupted — re-run the same command to pick up where this left off"; exit 130' INT -# run_parallel <fn> <max> <items...> — runs fn over items, up to max at a time. -# Each job's stdout+stderr is buffered to its own file (so concurrent tools see -# a non-TTY and don't scatter spinner output across the terminal), then flushed -# as a clean labeled block in submission order. Returns non-zero if any failed. +# rp_progress <statusdir> <label> <total> <t0> — one frame of the live +# "<label> — done/total (elapsed)" line while run_parallel waits. +rp_progress() { + local finished=0 f + for f in "$1"/*.rc; do [ -e "$f" ] && finished=$((finished + 1)); done + sg_spin_frame "$2 ${C_DIM}— $finished/$3 done ($(sg_fmt_secs $((SECONDS - $4))))${C_RESET}" +} + +# run_parallel <fn> <max> <label> <items...> — runs fn over items, up to max at +# a time, with a live progress line meanwhile. Each job's stdout+stderr is +# buffered to its own file (so concurrent tools see a non-TTY and don't scatter +# spinner output across the terminal), then flushed in submission order: a job +# that printed a single line is shown as-is, longer or failed output gets a +# "── <item> ──" header (always with -v). Returns non-zero if any job failed. run_parallel() { - local fn="$1" max="$2" - shift 2 - local statusdir i=0 rc=0 item + local fn="$1" max="$2" label="$3" + shift 3 + local statusdir i=0 rc=0 item total=$# t0=$SECONDS statusdir="$(mktemp -d)" + [ "$SG_ANIMATE" = "1" ] || sg_log "$label, up to $max in parallel..." for item in "$@"; do - throttle "$max" + while [ "$(jobs -rp | wc -l | tr -d ' ')" -ge "$max" ]; do + rp_progress "$statusdir" "$label" "$total" "$t0" + sleep 0.2 + done ( "$fn" "$item" >"$statusdir/$i.out" 2>&1 echo "$?" >"$statusdir/$i.rc" ) & i=$((i + 1)) done + while [ "$(jobs -rp | wc -l | tr -d ' ')" -gt 0 ]; do + rp_progress "$statusdir" "$label" "$total" "$t0" + sleep 0.2 + done wait + sg_spin_clear i=0 for item in "$@"; do - printf '%s── %s ──%s\n' "$C_CYAN" "$(basename "$item")" "$C_RESET" >&2 + if [ "$(cat "$statusdir/$i.rc" 2>/dev/null)" = "0" ]; then + if [ "$VERBOSE" -eq 1 ] || [ "$(wc -l <"$statusdir/$i.out" | tr -d ' ')" -gt 1 ]; then + printf '%s── %s ──%s\n' "$C_CYAN" "$(basename "$item")" "$C_RESET" >&2 + fi + else + printf '%s── %s ──%s\n' "$C_CYAN" "$(basename "$item")" "$C_RESET" >&2 + rc=1 + fi [ -s "$statusdir/$i.out" ] && cat "$statusdir/$i.out" >&2 - [ "$(cat "$statusdir/$i.rc" 2>/dev/null)" = "0" ] || rc=1 i=$((i + 1)) done rm -rf "$statusdir" @@ -169,17 +217,27 @@ payload_sha() { # run_phase <name> <input-sha> <fn> — in 'all', skip a phase that already ran # with identical inputs; otherwise run it and record the inputs it ran with. -# Phases that rewrite the payloads (enrich, convert) record the post-run hash, -# so an unchanged export is recognised on the next run. +# apply and enrich are keyed by the apply inputs (tfvars + workspace filter): +# enrich only depends on what apply produced, and the later convert phase +# rewrites the payloads, so a payload hash could never match again. convert and +# validate record the post-run payload hash, so an unchanged export is +# recognised on the next run. run_phase() { local name="$1" sha="$2" fn="$3" at if at="$(state_phase_done "$name" "$sha")" && [ -n "$at" ]; then - sg_log "skipping $name — unchanged since $at (--fresh to redo)" + at="${at%:*}" + at="${at/T/ } UTC" + if [ "$PHASE_TOTAL" -gt 0 ]; then + PHASE_N=$((PHASE_N + 1)) + sg_log "skipping phase $PHASE_N/$PHASE_TOTAL ($name) — inputs unchanged since $at (--fresh to redo)" + else + sg_log "skipping $name — inputs unchanged since $at (--fresh to redo)" + fi return 0 fi "$fn" || return $? case "$name" in - apply) state_mark_phase "$name" "$sha" ;; + apply | enrich) state_mark_phase "$name" "$sha" ;; *) state_mark_phase "$name" "$(payload_sha)" ;; esac } @@ -240,18 +298,21 @@ cmd_init() { wizard_run || return 1 fi fi - sg_log "workflow groups are created automatically as tfc-<project>; no mapping needed" - sg_log "next: ./sg-migrate.sh all" - completion_hint - sg_success "init complete" + # A standalone 'init' ends with what to do next; 'all' just carries on. + [ "${1:-}" = "standalone" ] && show_next_steps + return 0 } -# completion_hint — tell the user how to enable tab completion for their shell. -# A child process cannot register completions in the parent shell, so this only -# prints the one-liner (nothing is written to the user's rc files). -completion_hint() { - sg_log "tab completion for this shell session: source <(./sg-migrate.sh completion)" +# show_next_steps — the commands that make sense after init. (A child process +# cannot register completions in the parent shell, so the completion line is a +# hint only; nothing is written to the user's rc files.) +show_next_steps() { + sg_step "Next steps" + next_row "$PROG all" "run the migration — the import plan is shown and confirmed before anything is created" + next_row "$PROG apply" "export only: review the payloads in $(sg_rel "$EXPORT_DIR")/ first, then '$PROG import'" + next_row "source <($PROG completion)" "tab completion for this shell session" } +next_row() { printf ' %s%-38s%s %s%s%s\n' "$C_BOLD" "$1" "$C_RESET" "$C_DIM" "$2" "$C_RESET" >&2; } # current_shell — bash|zsh: the shell the user is typing in (detected by # sg-migrate.sh from its parent process), else the login shell. @@ -326,9 +387,8 @@ set_triggers_pass() { sg_log "no workflows carry VCS triggers — nothing to register" return 0 fi - sg_log "registering VCS triggers for $total workflow(s), up to $CONC in parallel (retries: $RETRIES)" local rc=0 f seg - run_parallel do_set_triggers "$CONC" "${PF[@]}" || rc=1 + run_parallel do_set_triggers "$CONC" "registering VCS triggers for $total workflow(s)" "${PF[@]}" || rc=1 for f in "${PF[@]}"; do seg="$(seg_of "$f")" if [ -f "$EXPORT_DIR/.triggers-result.$seg.json" ]; then @@ -337,7 +397,7 @@ set_triggers_pass() { fi done if [ "$rc" -eq 0 ]; then - sg_success "vcs triggers registered" + sg_success "VCS triggers registered for $total workflow(s)" else sg_err "one or more VCS trigger registrations failed (re-run: $PROG triggers)" return 1 @@ -345,7 +405,7 @@ set_triggers_pass() { } cmd_triggers() { - sg_step "Phase: vcs triggers" + phase_begin "VCS triggers" [ -n "${SG_API_TOKEN:-}" ] || die "SG_API_TOKEN is not set." [ -n "$ORG" ] || die "StackGuardian org not set (use --org or SG_ORG)." command -v curl >/dev/null 2>&1 || die "curl is required for VCS trigger registration." @@ -357,7 +417,7 @@ cmd_triggers() { } cmd_clean() { - sg_step "Phase: clean" + phase_begin "clean" sg_log "removing local working artifacts..." rm -rf "$EXPORT_DIR" rm -rf "$TRANSFORMER_DIR/.terraform" "$TRANSFORMER_DIR/.terraform.lock.hcl" \ @@ -372,17 +432,24 @@ cmd_clean() { sg_success "clean complete" } +# tf_init / tf_apply — terraform in the transformer dir, with jq on PATH for the +# state export's local-exec. Always run in a subshell (sg_run_quiet backgrounds +# them; the verbose path wraps them in parentheses). +# shellcheck disable=SC2120 # extra terraform flags (-no-color) come from the quiet path +tf_init() { cd "$TRANSFORMER_DIR" && export PATH="$TF_PATH" TF_IN_AUTOMATION=1 && terraform init -input=false "$@"; } +tf_apply() { cd "$TRANSFORMER_DIR" && export PATH="$TF_PATH" TF_IN_AUTOMATION=1 && terraform apply -auto-approve -compact-warnings -parallelism="$TF_PARALLELISM" -var-file=terraform.tfvars "$@"; } + cmd_apply() { - sg_step "Phase: apply (terraform)" + phase_begin "apply (terraform)" command -v terraform >/dev/null 2>&1 || die "terraform not found on PATH" [ -f "$TFVARS" ] || die "Missing $(sg_rel "$TFVARS"). Run: $PROG init" preflight_run apply # State export (TFC API) calls curl + jq from terraform's local-exec; make sure # both are on PATH for the apply (jq from cache if not already installed). command -v curl >/dev/null 2>&1 || die "curl is required for state export" - local jqdir tflog rc=0 + local tflog rc=0 local -a tfvar_args=() - jqdir="$(dirname "$(sg_resolve jq sg_ensure_jq)")" + TF_PATH="$(dirname "$(sg_resolve jq sg_ensure_jq)"):$PATH" if [ "${#WS_FILTER[@]}" -gt 0 ]; then JQ_BIN="${JQ_BIN:-$(sg_resolve jq sg_ensure_jq)}" tfvar_args=(-var "workspacenames=$(ws_filter_json)") @@ -390,36 +457,32 @@ cmd_apply() { fi if [ "$VERBOSE" -eq 1 ]; then - (cd "$TRANSFORMER_DIR" && export PATH="$jqdir:$PATH" TF_IN_AUTOMATION=1 && - terraform init -input=false && - terraform apply -auto-approve -compact-warnings -parallelism="$TF_PARALLELISM" -var-file=terraform.tfvars "${tfvar_args[@]}") || rc=$? + # shellcheck disable=SC2119 + (tf_init && tf_apply ${tfvar_args[@]+"${tfvar_args[@]}"}) || rc=$? else - # Quiet: capture terraform's verbose plan/output; surface only progress, the - # final summary, and (on failure) the captured log. + # Quiet: terraform's init/plan output goes to a log that is shown only on + # failure; the terminal gets a live progress line per step instead. tflog="$(mktemp)" - sg_log "initializing terraform (providers)..." - (cd "$TRANSFORMER_DIR" && export PATH="$jqdir:$PATH" TF_IN_AUTOMATION=1 && terraform init -input=false -no-color) >"$tflog" 2>&1 || rc=$? + sg_run_quiet "initializing terraform providers" "terraform providers ready" "$tflog" tf_init -no-color || rc=$? if [ "$rc" -eq 0 ]; then - sg_log "reading workspaces, generating payloads, exporting state..." - (cd "$TRANSFORMER_DIR" && export PATH="$jqdir:$PATH" TF_IN_AUTOMATION=1 && - terraform apply -auto-approve -compact-warnings -no-color -parallelism="$TF_PARALLELISM" -var-file=terraform.tfvars "${tfvar_args[@]}") >"$tflog" 2>&1 || rc=$? + sg_run_quiet "reading workspaces, generating payloads, exporting state" "workspaces read, payloads generated, state exported" "$tflog" \ + tf_apply -no-color ${tfvar_args[@]+"${tfvar_args[@]}"} || rc=$? fi if [ "$rc" -ne 0 ]; then sg_err "terraform failed (rc=$rc):" cat "$tflog" >&2 - else - grep -E '^(Apply complete|No changes)' "$tflog" | sed 's/^/ /' >&2 || true fi rm -f "$tflog" fi [ "$rc" -eq 0 ] || return "$rc" - sg_success "apply complete — payloads in $(sg_rel "$EXPORT_DIR")" + payload_files + sg_success "apply complete in $(phase_took) — ${#PF[@]} payload file(s) in $(sg_rel "$EXPORT_DIR")/" show_migration_summary } cmd_enrich() { - sg_step "Phase: variable sets" + phase_begin "variable sets" [ -f "$TFVARS" ] || die "Missing $(sg_rel "$TFVARS") (run: $PROG init)." payload_files [ "${#PF[@]}" -gt 0 ] || die "No payload files in $(sg_rel "$EXPORT_DIR") (run 'apply' first)." @@ -433,12 +496,11 @@ cmd_enrich() { do_convert() { "$SCRIPT_DIR/convert_hcl_to_json.sh" "$1"; } cmd_convert() { - sg_step "Phase: convert (HCL → JSON)" + phase_begin "convert (HCL → JSON)" payload_files [ "${#PF[@]}" -gt 0 ] || die "No payload files in $(sg_rel "$EXPORT_DIR") (run 'apply' first)." - sg_log "converting ${#PF[@]} payload(s), up to $CONC in parallel" - if run_parallel do_convert "$CONC" "${PF[@]}"; then - sg_success "converted ${#PF[@]} payload(s)" + if run_parallel do_convert "$CONC" "converting ${#PF[@]} payload file(s)" "${PF[@]}"; then + sg_success "converted ${#PF[@]} payload file(s) in $(phase_took)" else sg_err "conversion failed for one or more payloads" return 1 @@ -446,13 +508,13 @@ cmd_convert() { } cmd_validate() { - sg_step "Phase: validate" + phase_begin "validate" payload_files [ "${#PF[@]}" -gt 0 ] || die "No payload files in $(sg_rel "$EXPORT_DIR")." if "$SCRIPT_DIR/validate_payload.sh" "${PF[@]}"; then - sg_success "all ${#PF[@]} payload(s) valid" + sg_success "${#PF[@]} payload file(s) valid against schema/sg-payload.schema.json" else - sg_err "validation failed" + sg_err "validation failed — fix the payload(s) above (or the transformer) and re-run '$PROG validate'" return 1 fi } @@ -479,7 +541,7 @@ names_json() { printf '%s\n' "$@" | "$JQ_BIN" -R . | "$JQ_BIN" -s .; } # the trigger pass see what was actually imported); each fallback is appended to # terraform-version-fallbacks.log. Any other per-workflow failure fails the file. do_import() { - local f="$1" seg grp out rc=0 ceiling="" failed=() fb=() name line tmp names work all_names + local f="$1" seg grp out rc=0 ceiling="" failed=() fb=() name line tmp names work all_names patch seg="$(seg_of "$f")" grp="$(group_for "$seg")" # With --workspace, import only the selected workflows (a filtered copy). @@ -512,12 +574,18 @@ do_import() { rm -f "$out" if [ "${#fb[@]}" -gt 0 ]; then - sg_warn "${#fb[@]} workflow(s) pinned above SG's managed Terraform ceiling ($ceiling); re-importing with $SG_DEFAULT_TF_VERSION" + sg_warn "${#fb[@]} workflow(s) pinned above SG's managed Terraform ceiling ($ceiling); re-importing with ${SG_TF_FALLBACK_LABEL:-$SG_DEFAULT_TF_VERSION}" names="$(names_json "${fb[@]}")" tmp="$(mktemp "$EXPORT_DIR/.fallback.$seg.XXXXXX")" - # Patch the affected workflows in the payload and re-import only those. - "$JQ_BIN" --arg v "$SG_DEFAULT_TF_VERSION" --argjson names "$names" \ - 'map(if (.ResourceName as $n | $names | index($n)) != null then .TerraformConfig.terraformVersion = $v else . end)' "$f" >"$tmp.full" && + # Patch the affected workflows in the payload and re-import only those: a + # fixed fallback version, or (SGDefaultTerraformVersion = null) no version at + # all so the API fills it from the org's execution preset. + if [ -n "$SG_DEFAULT_TF_VERSION" ]; then + patch='map(if (.ResourceName as $n | $names | index($n)) != null then .TerraformConfig.terraformVersion = $v else . end)' + else + patch='map(if (.ResourceName as $n | $names | index($n)) != null then del(.TerraformConfig.terraformVersion) else . end)' + fi + "$JQ_BIN" --arg v "$SG_DEFAULT_TF_VERSION" --argjson names "$names" "$patch" "$f" >"$tmp.full" && "$JQ_BIN" --argjson names "$names" \ 'map(select(.ResourceName as $n | $names | index($n) != null))' "$tmp.full" >"$tmp" || { @@ -531,7 +599,7 @@ do_import() { failed+=("$name") else line="$("$JQ_BIN" -r --arg n "$name" '.[] | select(.ResourceName == $n) | .TerraformConfig.terraformVersion' "$f")" - printf '%s/%s: %s -> %s (above SG managed ceiling %s)\n' "$grp" "$name" "$line" "$SG_DEFAULT_TF_VERSION" "$ceiling" >>"$EXPORT_DIR/terraform-version-fallbacks.log" + printf '%s/%s: %s -> %s (above SG managed ceiling %s)\n' "$grp" "$name" "$line" "${SG_DEFAULT_TF_VERSION:-execution preset}" "$ceiling" >>"$EXPORT_DIR/terraform-version-fallbacks.log" fi done rm -f "$out" @@ -566,17 +634,18 @@ tf_fallback_notice() { cat >&2 <<NOTICE StackGuardian ships managed Terraform runtimes only up to the last MPL-licensed (FOSS) release; newer versions are BSL-licensed and are not bundled. The - workflows above were created with $SG_DEFAULT_TF_VERSION instead of the version - pinned in TFC, so they will run a different Terraform than before - verify the - configuration is compatible before the first run. To keep a newer version, set - workspaceOverrides[<name>].terraformVersion to a binary path mounted from a - private runner, or use a custom runtime container template - (wfStepTemplateRevisionId), and re-import. + workflows above were created with ${SG_TF_FALLBACK_LABEL:-$SG_DEFAULT_TF_VERSION} + instead of the version pinned in TFC, so they will run a different Terraform + than before - verify the configuration is compatible before the first run. To + keep a newer version, set workspaceOverrides[<name>].terraformVersion to a + binary path mounted from a private runner, or point the org's execution preset + (or the override) at a custom runtime image (wfStepTemplateRevisionId) that + ships it, and re-import. NOTICE } cmd_import() { - sg_step "Phase: import" + phase_begin "import" [ -n "${SG_API_TOKEN:-}" ] || die "SG_API_TOKEN is not set." [ -n "$ORG" ] || die "StackGuardian org not set (use --org or SG_ORG)." command -v curl >/dev/null 2>&1 || die "curl is required for workflow-group checks/creation." @@ -591,12 +660,12 @@ cmd_import() { # Build the plan: resolve each project's group, check existence, and decide # which groups need creating. Override groups (from the map) must already exist. - local fail=0 to_create=" " f seg grp count override is_override code status - printf '%sImport plan%s (org: %s%s%s, %s)\n' "$C_BOLD" "$C_RESET" "$C_CYAN" "$ORG" "$C_RESET" "$SG_BASE_URL" >&2 - printf ' %s%-34s %-26s %-9s %s%s\n' "$C_BOLD" "FILE" "WORKFLOW GROUP" "WORKFLOWS" "STATUS" "$C_RESET" >&2 + local fail=0 to_create=" " n_create=0 total_wf=0 f seg grp count override is_override code status fw gw q i + local -a files=() groups=() counts=() statuses=() for f in "${PF[@]}"; do seg="$(seg_of "$f")" - count="$("$JQ_BIN" 'length' "$f")" + count="$("$JQ_BIN" --argjson ws "$(ws_filter_json)" '[.[] | select(($ws | length) == 0 or (.ResourceName as $n | $ws | index($n) != null))] | length' "$f")" + total_wf=$((total_wf + count)) override="" [ -f "$MAPPING" ] && override="$("$JQ_BIN" -r --arg k "$seg" '.[$k] // empty' "$MAPPING" 2>/dev/null || true)" if [ -n "$override" ]; then @@ -616,7 +685,11 @@ cmd_import() { fail=1 elif [ "$CREATE_GROUPS" -eq 1 ]; then status="${C_YELLOW}create${C_RESET}" - case "$to_create" in *" $grp "*) ;; *) to_create="$to_create$grp " ;; esac + case "$to_create" in *" $grp "*) ;; *) + to_create="$to_create$grp " + n_create=$((n_create + 1)) + ;; + esac else status="${C_RED}missing!${C_RESET}" fail=1 @@ -626,15 +699,34 @@ cmd_import() { 000) die "could not reach $SG_BASE_URL" ;; *) die "unexpected HTTP $code checking group '$grp'" ;; esac - printf ' %-34s %-26s %-9s %s\n' "$(basename "$f")" "$grp" "$count" "$status" >&2 + files+=("$(basename "$f")") + groups+=("$grp") + counts+=("$count") + statuses+=("$status") + done + fw="$(sg_maxlen 4 "${files[@]}")" + gw="$(sg_maxlen 14 "${groups[@]}")" + printf '%sImport plan%s (org: %s%s%s, %s)\n' "$C_BOLD" "$C_RESET" "$C_CYAN" "$ORG" "$C_RESET" "$SG_BASE_URL" >&2 + printf " %s%-${fw}s %-${gw}s %-9s %s%s\n" "$C_BOLD" "FILE" "WORKFLOW GROUP" "WORKFLOWS" "STATUS" "$C_RESET" >&2 + for ((i = 0; i < ${#files[@]}; i++)); do + printf " %-${fw}s %-${gw}s %-9s %s\n" "${files[i]}" "${groups[i]}" "${counts[i]}" "${statuses[i]}" >&2 done if [ "$fail" -ne 0 ]; then die "some groups are missing (override groups are not auto-created; create them or remove the override)." fi - # Fallback Terraform version for workflows the API rejects as above the - # managed ceiling: SGDefaultTerraformVersion from terraform.tfvars, else 1.5.7. - SG_DEFAULT_TF_VERSION="${SG_DEFAULT_TF_VERSION:-$(tfvars_get '.SGDefaultTerraformVersion' TERRAFORM-1.5.7)}" + # Fallback for workflows the API rejects as above the managed ceiling: a fixed + # SGDefaultTerraformVersion (missing key = 1.5.7), or an explicit null in + # terraform.tfvars = drop the version so the org's execution preset decides. + # The preset itself is read so the plan can show what "preset" resolves to. + if tfvars_is_null SGDefaultTerraformVersion; then + SG_DEFAULT_TF_VERSION="" + else + SG_DEFAULT_TF_VERSION="${SG_DEFAULT_TF_VERSION:-$(tfvars_get '.SGDefaultTerraformVersion' TERRAFORM-1.5.7)}" + fi + # shellcheck disable=SC2034 # read by report.sh (plan) and checklist.sh + SG_PRESET_JSON="$(sg_execution_preset)" || SG_PRESET_JSON="" + preset_labels show_import_plan "${PF[@]}" if [ "$DRY_RUN" -eq 1 ]; then sg_success "dry run — nothing was created or changed" @@ -642,9 +734,14 @@ cmd_import() { fi if [ "$ASSUME_YES" -ne 1 ]; then - printf '%sProceed?%s This imports to %s and creates any "create" groups. [y/N] ' "$C_BOLD$C_YELLOW" "$C_RESET" "$ORG" >&2 - read -r ans || ans="" - case "$ans" in y | Y | yes | YES) ;; *) die "Aborted." ;; esac + q="Import $total_wf workflow(s) into $ORG" + [ "$n_create" -gt 0 ] && q="$q and create $n_create workflow group(s)" + sg_interactive || die "no terminal to confirm the import — re-run with -y to import without a prompt" + if ! sg_confirm "$q?" N; then + sg_warn "import cancelled — nothing was changed in $ORG" + sg_dim "re-run '$PROG all' (or '$PROG import') to come back to this plan; the export phases are saved and skipped" + return 1 + fi fi # Create the missing tfc-* groups before importing into them. @@ -660,7 +757,7 @@ cmd_import() { # (a changed payload or a previous failure re-imports the whole file; # sg-cli updates existing workflows in place). local -a todo=() - local skipped=0 seg + local skipped=0 import_rc=0 for f in "${PF[@]}"; do seg="$(seg_of "$f")" if [ "$FRESH" -eq 0 ] && [ "${#WS_FILTER[@]}" -eq 0 ] && state_import_done "$seg" "$(sg_sha_files "$f")"; then @@ -669,11 +766,9 @@ cmd_import() { fi todo+=("$f") done - [ "$skipped" -gt 0 ] && sg_log "skipping $skipped payload(s) already imported and unchanged (use --fresh to re-import)" - local import_rc=0 + [ "$skipped" -gt 0 ] && sg_log "skipping $skipped payload file(s) already imported and unchanged (--fresh to re-import)" if [ "${#todo[@]}" -gt 0 ]; then - sg_log "importing ${#todo[@]} payload(s), up to $CONC in parallel (retries: $RETRIES)" - run_parallel do_import "$CONC" "${todo[@]}" || import_rc=1 + run_parallel do_import "$CONC" "importing ${#todo[@]} payload file(s) (retries: $RETRIES)" "${todo[@]}" || import_rc=1 for f in "${todo[@]}"; do seg="$(seg_of "$f")" if [ -f "$EXPORT_DIR/.import-result.$seg.json" ]; then @@ -684,7 +779,7 @@ cmd_import() { fi tf_fallback_notice if [ "$import_rc" -eq 0 ]; then - sg_success "import complete (${#todo[@]} payload(s) imported, $skipped skipped)" + sg_success "import complete (${#todo[@]} payload file(s) imported, $skipped skipped)" else sg_err "one or more workflows failed to import (see above); VCS triggers are still registered for the ones that succeeded" fi @@ -696,9 +791,24 @@ cmd_import() { fi [ "$SECRET_STUBS" -eq 1 ] && create_secret_stubs write_checklist + finish_line "$import_rc" return "$import_rc" } +# finish_line <rc> — the last line of an import / all run: outcome, total time, +# and whether the checklist still has items for a human. +finish_line() { + local open="${CHECKLIST_OPEN:-0}" took + took="$(sg_fmt_secs $((SECONDS - RUN_T0)))" + if [ "$1" -ne 0 ]; then + sg_err "finished with failures in $took — fix what is reported above and re-run '$PROG import' (only failed or changed files are retried)" + elif [ "$open" -gt 0 ]; then + sg_success "migration complete in $took — $open item(s) still need a human, see $(sg_rel "$EXPORT_DIR")/post-import-checklist.md" + else + sg_success "migration complete in $took — nothing left to do by hand" + fi +} + # Single source of truth for shell completion (keep in sync with the parser below # and the host-only flags in sg-migrate.sh). SG_COMMANDS="init preflight apply enrich convert validate import triggers checklist all clean completion" @@ -873,7 +983,7 @@ main() { export SG_VERBOSE="$VERBOSE" case "$CMD" in - init) cmd_init ;; + init) cmd_init standalone ;; clean) cmd_clean ;; apply) cmd_apply ;; enrich) cmd_enrich ;; @@ -904,8 +1014,12 @@ main() { preflight_run all [ "$FRESH" -eq 1 ] && { state_reset; sg_log "--fresh: previous run state discarded"; } JQ_BIN="$(sg_resolve jq sg_ensure_jq)" - run_phase apply "$(sg_sha "$(sg_sha_files "$TFVARS")|$(ws_filter_json)")" cmd_apply - if [ "$ENRICH_VARSETS" -eq 1 ]; then run_phase enrich "$(payload_sha)" cmd_enrich; fi + PHASE_TOTAL=4 + [ "$ENRICH_VARSETS" -eq 1 ] && PHASE_TOTAL=5 + local apply_sha + apply_sha="$(sg_sha "$(sg_sha_files "$TFVARS")|$(ws_filter_json)")" + run_phase apply "$apply_sha" cmd_apply + if [ "$ENRICH_VARSETS" -eq 1 ]; then run_phase enrich "$apply_sha" cmd_enrich; fi run_phase convert "$(payload_sha)" cmd_convert run_phase validate "$(payload_sha)" cmd_validate cmd_import diff --git a/scripts/tools.sh b/scripts/tools.sh index 901a9ff..3100080 100755 --- a/scripts/tools.sh +++ b/scripts/tools.sh @@ -53,6 +53,65 @@ sg_err() { printf '%s[sg-migrate] ERROR%s %s\n' "$C_RED$C_BOLD" "$C_RESET" "$*" sg_success() { printf '%s[sg-migrate] ✓%s %s\n' "$C_GREEN$C_BOLD" "$C_RESET" "$*" >&2; } sg_step() { printf '\n%s==> %s%s\n' "$C_CYAN$C_BOLD" "$*" "$C_RESET" >&2; } sg_dim() { printf '%s %s%s\n' "$C_DIM" "$*" "$C_RESET" >&2; } +# sg_row <label> <value> — an aligned " label value" line (review/summary tables). +sg_row() { printf ' %s%-26s%s %s\n' "$C_BOLD" "$1" "$C_RESET" "$2" >&2; } + +# sg_fmt_secs <seconds> — 14s / 1m 12s / 1h 02m, for "done in ..." messages. +sg_fmt_secs() { + local s="${1:-0}" + if [ "$s" -ge 3600 ]; then printf '%dh %02dm' "$((s / 3600))" "$(((s % 3600) / 60))" + elif [ "$s" -ge 60 ]; then printf '%dm %02ds' "$((s / 60))" "$((s % 60))" + else printf '%ds' "$s"; fi +} + +# sg_maxlen <min> <string>... — the longest string's length, but at least <min> +# (used to size table columns to their content). +sg_maxlen() { + local w="$1" s + shift + for s in "$@"; do [ "${#s}" -gt "$w" ] && w="${#s}"; done + printf '%s' "$w" +} + +# In-place progress line for long-running steps. Animated only when stderr is a +# TTY with colors on (same switch as the colors); plain log lines otherwise, so +# CI logs never fill up with spinner frames. +SG_ANIMATE=0 +[ -n "$C_RESET" ] && SG_ANIMATE=1 +_SG_SPIN_FRAMES=('⠋' '⠙' '⠹' '⠸' '⠼' '⠴' '⠦' '⠧' '⠇' '⠏') +_SG_SPIN_I=0 +sg_spin_frame() { + [ "$SG_ANIMATE" = "1" ] || return 0 + printf '\r%s[sg-migrate]%s %s%s%s %s\033[K' "$C_CYAN" "$C_RESET" "$C_CYAN" "${_SG_SPIN_FRAMES[_SG_SPIN_I % 10]}" "$C_RESET" "$1" >&2 + _SG_SPIN_I=$((_SG_SPIN_I + 1)) +} +sg_spin_clear() { + [ "$SG_ANIMATE" = "1" ] && printf '\r\033[K' >&2 + return 0 +} + +# sg_run_quiet <running-label> <done-label> <logfile> <cmd...> — run cmd with +# stdout+stderr captured in logfile, showing "<running-label> (elapsed)" as a +# live line meanwhile, then a "<done-label> (took Ns)" log line. Returns the +# command's exit code; the caller decides what to do with the log. +sg_run_quiet() { + local running="$1" done_label="$2" log="$3" pid rc=0 t0=$SECONDS + shift 3 + "$@" >"$log" 2>&1 & + pid=$! + if [ "$SG_ANIMATE" = "1" ]; then + while kill -0 "$pid" 2>/dev/null; do + sg_spin_frame "$running $C_DIM($(sg_fmt_secs $((SECONDS - t0))))$C_RESET" + sleep 0.2 + done + sg_spin_clear + else + sg_log "$running..." + fi + wait "$pid" || rc=$? + [ "$rc" -eq 0 ] && sg_log "$done_label $C_DIM($(sg_fmt_secs $((SECONDS - t0))))$C_RESET" + return "$rc" +} sg_arch() { case "$(uname -m)" in diff --git a/scripts/validate_payload.sh b/scripts/validate_payload.sh index c5fbd4a..d8545ba 100755 --- a/scripts/validate_payload.sh +++ b/scripts/validate_payload.sh @@ -20,10 +20,21 @@ fi # Resolve yajsv from PATH (Docker image) or download+cache (native). YAJSV_BIN=$(sg_resolve yajsv sg_ensure_yajsv) -sg_log "validating $# file(s) against schema/sg-payload.schema.json" -# yajsv prints "<file>: valid" per file and exits non-zero if any file fails. -# Strip the repo-root prefix from its output for readable, relative paths -# (pipefail off so the pipeline's status is sed's; yajsv's status via PIPESTATUS). +# yajsv prints "<file>: pass" / "<file>: fail: <reason>" per file and exits +# non-zero if any file fails. Re-render its lines in the migrator's style +# (✓/✗ + file name; the raw lines with -v) — pipefail off so the pipeline's +# status is the loop's; yajsv's own status comes back via PIPESTATUS. set +o pipefail -"$YAJSV_BIN" -s "$SCHEMA" "$@" 2>&1 | sed "s#${SG_REPO_ROOT}/##g" | awk '!seen[$0]++' +"$YAJSV_BIN" -s "$SCHEMA" "$@" 2>&1 | sed "s#${SG_REPO_ROOT}/##g" | awk '!seen[$0]++' | while IFS= read -r line; do + if [ "${SG_VERBOSE:-0}" = "1" ]; then + printf ' %s\n' "$line" >&2 + continue + fi + case "$line" in + *": pass") printf ' %s✓%s %s\n' "$C_GREEN" "$C_RESET" "$(basename "${line%: pass}")" >&2 ;; + *": fail: "*) printf ' %s✗%s %s: %s\n' "$C_RED$C_BOLD" "$C_RESET" "$(basename "${line%%: fail: *}")" "${line#*: fail: }" >&2 ;; + *": error: "*) printf ' %s✗%s %s: %s\n' "$C_RED$C_BOLD" "$C_RESET" "$(basename "${line%%: error: *}")" "${line#*: error: }" >&2 ;; + *) printf ' %s\n' "$line" >&2 ;; + esac +done exit "${PIPESTATUS[0]}" diff --git a/transformer/terraform-cloud/example_payload.jsonc b/transformer/terraform-cloud/example_payload.jsonc index 60a67d9..7113182 100644 --- a/transformer/terraform-cloud/example_payload.jsonc +++ b/transformer/terraform-cloud/example_payload.jsonc @@ -34,11 +34,11 @@ "ResourceName": "", // workspace name "RunnerConstraints": { "type": "" - }, // type should be "shared" or "private" i.e. "RunnerConstraints": { "type": "private", "names":["runner-group-name"]} + }, // type should be "shared" or "private" i.e. "RunnerConstraints": { "type": "private", "names":["runner-group-name"]}. Omit the whole key (SGDefaultRunnerConstraints = null) and StackGuardian applies the org's execution preset at import. "Tags": [], // workflow tags "TerraformConfig": { "managedTerraformState": true, // managed state from StackGuardian - "terraformVersion": "TERRAFORM-1.1.6" // version of terraform + "terraformVersion": "TERRAFORM-1.1.6" // version of terraform; omit the key (SGTerraformVersionSource = "preset", or a null fallback) and the org's execution preset decides. The API only fills keys that are absent, never null/"". }, "UserSchedules": [], "VCSConfig": { diff --git a/transformer/terraform-cloud/locals.tf b/transformer/terraform-cloud/locals.tf index 5692f95..471f68e 100644 --- a/transformer/terraform-cloud/locals.tf +++ b/transformer/terraform-cloud/locals.tf @@ -42,13 +42,41 @@ locals { } # Workspaces whose terraform_version is not a pinned semver (e.g. "latest" or - # a constraint) and have no per-workspace override fall back to the default. + # a constraint) and have no per-workspace override fall back to the default + # (SGDefaultTerraformVersion, or the org's execution preset when that is null). versionFallbacks = { for name in local.workflowNames : name => data.tfe_workspace.data[name].terraform_version if !can(regex("^[0-9]+\\.[0-9]+\\.[0-9]+$", data.tfe_workspace.data[name].terraform_version)) && try(var.workspaceOverrides[name].terraformVersion, null) == null } + # Terraform version sent per workflow. An override is always sent as-is. + # Otherwise "carry" keeps a pinned TFC semver (TERRAFORM-x.y.z) and falls back + # to SGDefaultTerraformVersion for anything else; "preset" (or a null default) + # yields null, and the key is then left out of the payload so StackGuardian + # fills it from the org's execution preset at import time. + tfVersion = { + for name in local.workflowNames : + name => ( + try(var.workspaceOverrides[name].terraformVersion, null) != null ? var.workspaceOverrides[name].terraformVersion : + var.SGTerraformVersionSource == "preset" ? null : + can(regex("^[0-9]+\\.[0-9]+\\.[0-9]+$", data.tfe_workspace.data[name].terraform_version)) ? "TERRAFORM-${data.tfe_workspace.data[name].terraform_version}" : + var.SGDefaultTerraformVersion + ) + } + + # Runner constraints sent per workflow: override, else the global default, + # else null (key left out, the execution preset decides). Picked via a tuple + # rather than a conditional: an override with "names" and a default without + # it are different object types, which a conditional refuses to unify. + runnerConstraints = { + for name in local.workflowNames : + name => try([for c in [ + try(var.workspaceOverrides[name].RunnerConstraints, null), + var.SGDefaultRunnerConstraints == null ? null : { for k, v in var.SGDefaultRunnerConstraints : k => v if v != null } + ] : c if c != null][0], null) + } + # Workspaces not using "remote" execution may not store their state in TFC, so # the API state export can come back empty; flag them in the summary. nonRemoteModes = { @@ -67,7 +95,7 @@ locals { # One SG workflow payload per workspace. Per-workspace overrides win over the # SGDefault* values; everything else is derived from the TFC workspace. workflowPayload = { - for wsName, wsId in data.tfe_workspace_ids.data.ids : wsName => { + for wsName, wsId in data.tfe_workspace_ids.data.ids : wsName => merge({ CLIConfiguration = { "WorkflowGroup" : { # SG workflow group per TFC project: tfc-<project> (matches the group @@ -87,7 +115,6 @@ locals { ) DeploymentPlatformConfig = try(var.workspaceOverrides[wsName].DeploymentPlatformConfig, null) != null ? var.workspaceOverrides[wsName].DeploymentPlatformConfig : var.SGDefaultDeploymentPlatformConfig - RunnerConstraints = try(var.workspaceOverrides[wsName].RunnerConstraints, null) != null ? var.workspaceOverrides[wsName].RunnerConstraints : { for k, v in var.SGDefaultRunnerConstraints : k => v if v != null } VCSConfig = { "iacVCSConfig" : { @@ -166,18 +193,18 @@ locals { Approvers = try(var.workspaceOverrides[wsName].Approvers, null) != null ? var.workspaceOverrides[wsName].Approvers : (data.tfe_workspace.data[wsName].auto_apply ? [] : var.SGDefaultWfApprovers) - TerraformConfig = { + # terraformVersion is omitted (not null) when the execution preset should + # decide: the SG API only fills in keys that are absent from the payload. + TerraformConfig = { for k, v in { "managedTerraformState" : true, - "terraformVersion" : ( - try(var.workspaceOverrides[wsName].terraformVersion, null) != null ? var.workspaceOverrides[wsName].terraformVersion : - can(regex("^[0-9]+\\.[0-9]+\\.[0-9]+$", data.tfe_workspace.data[wsName].terraform_version)) ? "TERRAFORM-${data.tfe_workspace.data[wsName].terraform_version}" : var.SGDefaultTerraformVersion - ), + "terraformVersion" : local.tfVersion[wsName], "approvalPreApply" : !data.tfe_workspace.data[wsName].auto_apply - } + } : k => v if v != null } WfType = "TERRAFORM" UserSchedules = [] - } + # RunnerConstraints likewise: present only when we have a value to send. + }, { for k, v in { RunnerConstraints = local.runnerConstraints[wsName] } : k => v if v != null }) } # Group payloads by TFC project so each project imports into its own SG @@ -198,13 +225,18 @@ locals { # Machine-readable migration summary (also rendered to markdown). summary = { - organization = var.tfOrg - workspaceCount = length(local.workflowNames) - projectWorkspaceCounts = { for pid in local.projectsUsed : try(local.projectNames[pid], pid) => length(local.payloadByProject[pid]) } - skippedSensitiveVars = { for name, vars in local.sensitiveVars : name => vars if length(vars) > 0 } - strippedVars = { for name, vars in local.strippedVars : name => vars if length(vars) > 0 } - ignoreVarPatterns = var.ignoreVarPatterns - terraformVersionFallbacks = local.versionFallbacks + organization = var.tfOrg + workspaceCount = length(local.workflowNames) + projectWorkspaceCounts = { for pid in local.projectsUsed : try(local.projectNames[pid], pid) => length(local.payloadByProject[pid]) } + skippedSensitiveVars = { for name, vars in local.sensitiveVars : name => vars if length(vars) > 0 } + strippedVars = { for name, vars in local.strippedVars : name => vars if length(vars) > 0 } + ignoreVarPatterns = var.ignoreVarPatterns + # Version policy, so the later phases can explain what each workflow runs. + terraformVersionSource = var.SGTerraformVersionSource + terraformVersionDefault = var.SGDefaultTerraformVersion # null = the execution preset decides + runnerConstraintsSource = var.SGDefaultRunnerConstraints == null ? "preset" : "config" + tfcTerraformVersions = { for name in local.workflowNames : name => data.tfe_workspace.data[name].terraform_version } + terraformVersionFallbacks = var.SGTerraformVersionSource == "carry" ? local.versionFallbacks : {} nonRemoteExecutionModes = local.nonRemoteModes renamedWorkspaces = { for name in local.workflowNames : name => local.resourceNames[name] if local.resourceNames[name] != name } variableSetsReminder = "TFC Variable Set variables are merged by the 'enrich' step (non-sensitive only). Sensitive set vars can't be read from the API — recreate them as SG secrets; see the enrich step output." diff --git a/transformer/terraform-cloud/summary.tmpl b/transformer/terraform-cloud/summary.tmpl index 7e7f811..d7df0de 100644 --- a/transformer/terraform-cloud/summary.tmpl +++ b/transformer/terraform-cloud/summary.tmpl @@ -28,8 +28,14 @@ Not migrated because they only mean something inside Terraform Cloud (patterns: %{ endfor ~} %{ endif ~} -## Terraform version fallbacks -Workspaces whose version was not a pinned semver; the configured SGDefaultTerraformVersion was used. +## Terraform version +%{ if summary.terraformVersionSource == "preset" ~} +No version is set on the migrated workflows (SGTerraformVersionSource = "preset"): StackGuardian applies the organisation's execution preset (Settings -> Runner groups -> Execution presets), or its platform default (managed Terraform 1.5.7) when none is configured. The workspaces ran these versions in TFC: +%{ for ws, ver in summary.tfcTerraformVersions ~} +- ${ws}: ${ver} +%{ endfor ~} +%{ else ~} +Pinned TFC versions are carried over (SGTerraformVersionSource = "carry"). Workspaces whose version was not a pinned semver use %{ if summary.terraformVersionDefault == null }the organisation's execution preset (SGDefaultTerraformVersion = null)%{ else }the fallback ${summary.terraformVersionDefault}%{ endif }: %{ if length(summary.terraformVersionFallbacks) == 0 ~} - None. %{ else ~} @@ -37,6 +43,14 @@ Workspaces whose version was not a pinned semver; the configured SGDefaultTerraf - ${ws}: reported "${ver}" %{ endfor ~} %{ endif ~} +%{ endif ~} + +## Runners +%{ if summary.runnerConstraintsSource == "preset" ~} +- No runner constraints are set (SGDefaultRunnerConstraints = null); the organisation's execution preset decides (platform default: shared runners). +%{ else ~} +- Runner constraints come from terraform.tfvars (SGDefaultRunnerConstraints / workspaceOverrides). +%{ endif ~} ## Non-remote execution modes TFC may not hold state for these (local/agent execution); state export can be empty. diff --git a/transformer/terraform-cloud/terraform.tfvars.example b/transformer/terraform-cloud/terraform.tfvars.example index f588f18..e20789c 100644 --- a/transformer/terraform-cloud/terraform.tfvars.example +++ b/transformer/terraform-cloud/terraform.tfvars.example @@ -43,17 +43,28 @@ SGDefaultDeploymentPlatformConfig = [ # Runners for every workflow: SG-hosted shared runners (default), or put all # workflows behind a private runner group: # SGDefaultRunnerConstraints = { type = "private", names = ["sg-runner"] } +# Set to null to send no runner constraints, so StackGuardian applies the org's +# execution preset (Settings -> Runner groups -> Execution presets). SGDefaultRunnerConstraints = { type = "shared" } # Choose from: GITHUB_COM, BITBUCKET_ORG, GITLAB_COM, AZURE_DEVOPS, GIT_OTHER SGDefaultSourceConfigDestKind = "GITHUB_COM" -# SG Terraform version used when a workspace's terraform_version is not a pinned -# semver (e.g. "latest" or a constraint). Pinned versions are carried over as-is; -# if the SG API rejects one as above the managed ceiling (1.5.7 - the last MPL/ -# FOSS Terraform release; newer versions are BSL and not bundled), the importer -# re-creates that workflow with this version and logs it in -# export/terraform-version-fallbacks.log. +# Where the workflows get their Terraform version from: +# "carry" - keep each workspace's pinned TFC version; the rest (see below) use +# SGDefaultTerraformVersion +# "preset" - send no version at all: StackGuardian fills it from the org's +# execution preset (Settings -> Runner groups -> Execution presets), +# or its platform default (managed Terraform 1.5.7) +SGTerraformVersionSource = "carry" + +# Fallback for "carry": used when a workspace's terraform_version is not a pinned +# semver (e.g. "latest" or a constraint), and by the importer when the SG API +# rejects a pinned version as above the managed ceiling (1.5.7 - the last MPL/ +# FOSS Terraform release; newer versions are BSL and not bundled) - those +# workflows are re-created with this version and logged in +# export/terraform-version-fallbacks.log. Set to null to leave them to the org's +# execution preset instead. SGDefaultTerraformVersion = "TERRAFORM-1.5.7" # Pre-configure VCS triggers on each VCS-backed workflow, remapped from the diff --git a/transformer/terraform-cloud/variables.tf b/transformer/terraform-cloud/variables.tf index a8a103d..c265db6 100644 --- a/transformer/terraform-cloud/variables.tf +++ b/transformer/terraform-cloud/variables.tf @@ -79,17 +79,18 @@ variable "SGDefaultDeploymentPlatformConfig" { variable "SGDefaultRunnerConstraints" { default = { type = "shared" } - description = "Runner constraints applied to every workflow. Use { type = \"shared\" } for SG-hosted runners, or { type = \"private\", names = [\"<runner-group>\"] } to put every workflow behind a private runner group. Override per workspace via workspaceOverrides[name].RunnerConstraints." + nullable = true + description = "Runner constraints applied to every workflow. Use { type = \"shared\" } for SG-hosted runners, { type = \"private\", names = [\"<runner-group>\"] } to put every workflow behind a private runner group, or null to send no runner constraints at all so StackGuardian applies the org's execution preset (Settings -> Runner groups -> Execution presets; platform default: shared runners). Override per workspace via workspaceOverrides[name].RunnerConstraints." type = object({ type = string names = optional(list(string)) }) validation { - condition = contains(["shared", "private"], var.SGDefaultRunnerConstraints.type) - error_message = "SGDefaultRunnerConstraints.type must be \"shared\" or \"private\"." + condition = var.SGDefaultRunnerConstraints == null || contains(["shared", "private"], try(var.SGDefaultRunnerConstraints.type, "")) + error_message = "SGDefaultRunnerConstraints.type must be \"shared\" or \"private\" (or the whole variable null to defer to the execution preset)." } validation { - condition = var.SGDefaultRunnerConstraints.type != "private" || length(coalesce(var.SGDefaultRunnerConstraints.names, [])) > 0 + condition = var.SGDefaultRunnerConstraints == null || try(var.SGDefaultRunnerConstraints.type, "") != "private" || length(coalesce(try(var.SGDefaultRunnerConstraints.names, null), [])) > 0 error_message = "SGDefaultRunnerConstraints.names must list at least one runner group when type is \"private\"." } } @@ -100,9 +101,20 @@ variable "SGDefaultSourceConfigDestKind" { type = string } +variable "SGTerraformVersionSource" { + default = "carry" + description = "Where the migrated workflows get their Terraform version from. \"carry\": keep each workspace's pinned TFC version; workspaces without a pinned semver (e.g. 'latest') use SGDefaultTerraformVersion, and so does the importer when the SG API rejects a pinned version as above its managed ceiling (1.5.7). \"preset\": send no version at all, so StackGuardian fills it from the org's execution preset (Settings -> Runner groups -> Execution presets), or its platform default when none is configured; SGDefaultTerraformVersion is then ignored. A workspaceOverrides[name].terraformVersion is always sent as-is." + type = string + validation { + condition = contains(["carry", "preset"], var.SGTerraformVersionSource) + error_message = "SGTerraformVersionSource must be \"carry\" or \"preset\"." + } +} + variable "SGDefaultTerraformVersion" { default = "TERRAFORM-1.5.7" - description = "SG Terraform version used when a workspace's terraform_version is not a pinned semver (e.g. 'latest' or a version constraint). Also used by the importer when the SG API rejects a pinned version as above the managed ceiling (1.5.7, the last MPL/FOSS release; newer versions are BSL and not bundled). Use the SG-formatted value, e.g. TERRAFORM-1.5.7." + nullable = true + description = "Fallback SG Terraform version when SGTerraformVersionSource is \"carry\": used for workspaces whose terraform_version is not a pinned semver (e.g. 'latest' or a version constraint), and by the importer when the SG API rejects a pinned version as above the managed ceiling (1.5.7, the last MPL/FOSS release; newer versions are BSL and not bundled). Use the SG-formatted value, e.g. TERRAFORM-1.5.7, or null to leave those workflows to the org's execution preset instead." type = string }