feat: end-to-end TFC migration pipeline with guided init and import

.dockerignore

@@ -0,0 +1,9 @@
+.git
+.sg
+export
+out
+**/.terraform
+**/.terraform.lock.hcl
+*.tfstate
+*.tfstate.*
+*.tfvars

.gitignore

@@ -16,6 +16,8 @@ eggs/
 .eggs/
 lib/
 lib64/
+# The migrator's sourced shell libraries are not Python build output.
+!scripts/lib/
 parts/
 sdist/
 var/
@@ -146,6 +148,8 @@ crash.log
 # control as they are data points which are potentially sensitive and subject
 # to change depending on the environment.
 *.tfvars
+# The init wizard keeps the previous file as terraform.tfvars.bak
+*.tfvars.bak
 
 # Ignore override files as they are usually used to override resources locally and so
 # are not checked in
@@ -163,4 +167,7 @@ out/*
 zip
 zip/*
 
+# Migrator local cache + config (downloaded tool binaries, workflow-group map)
+.sg/
+
 .DS_Store

Dockerfile

@@ -0,0 +1,53 @@
+# Migrator runtime: bundles all pinned tooling so the flow behaves identically
+# on Linux/macOS/Windows hosts (anywhere Docker runs). The repo is bind-mounted
+# at /app at runtime; this image only provides the tools on PATH.
+
+# yajsv has no linux/arm64 release asset, so build it from source for the
+# image's target architecture.
+FROM golang:1.22-bookworm AS yajsv
+ARG YAJSV_VERSION=v1.4.1
+RUN go install "github.com/neilpa/yajsv@${YAJSV_VERSION}"
+
+FROM debian:bookworm-slim
+
+ARG TERRAFORM_VERSION=1.9.8
+ARG JQ_VERSION=1.8.1
+ARG HCL2JSON_VERSION=0.6.7
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+      bash curl ca-certificates git unzip tar coreutils \
+    && rm -rf /var/lib/apt/lists/*
+
+# terraform (arch from dpkg: amd64/arm64 — works with or without buildx)
+RUN arch="$(dpkg --print-architecture)" \
+    && curl -fsSL "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_${arch}.zip" -o /tmp/tf.zip \
+    && unzip /tmp/tf.zip -d /usr/local/bin \
+    && rm /tmp/tf.zip
+
+# jq
+RUN arch="$(dpkg --print-architecture)" \
+    && curl -fsSL "https://github.com/jqlang/jq/releases/download/jq-${JQ_VERSION}/jq-linux-${arch}" -o /usr/local/bin/jq \
+    && chmod +x /usr/local/bin/jq
+
+# hcl2json
+RUN arch="$(dpkg --print-architecture)" \
+    && curl -fsSL "https://github.com/tmccombs/hcl2json/releases/download/v${HCL2JSON_VERSION}/hcl2json_linux_${arch}" -o /usr/local/bin/hcl2json \
+    && chmod +x /usr/local/bin/hcl2json
+
+# yajsv (from the build stage above)
+COPY --from=yajsv /go/bin/yajsv /usr/local/bin/yajsv
+
+# sg-cli (latest release, Go binary). Assets: sg-cli_<OS>_<ARCH>.tar.gz.
+RUN set -eu; \
+    arch="$(dpkg --print-architecture)"; \
+    case "$arch" in amd64) arch=x86_64 ;; arm64) arch=arm64 ;; esac; \
+    curl -fsSL "https://github.com/StackGuardian/sg-cli/releases/latest/download/sg-cli_Linux_${arch}.tar.gz" -o /tmp/sg-cli.tar.gz; \
+    mkdir -p /tmp/sgcli; \
+    tar -xzf /tmp/sg-cli.tar.gz -C /tmp/sgcli; \
+    realcli="$(find /tmp/sgcli -maxdepth 2 -type f -name sg-cli | head -1)"; \
+    test -n "$realcli"; \
+    install -m 0755 "$realcli" /usr/local/bin/sg-cli; \
+    rm -rf /tmp/sg-cli.tar.gz /tmp/sgcli
+
+WORKDIR /app
+ENTRYPOINT ["/bin/bash"]

schema/sg-payload.schema.json

@@ -0,0 +1,118 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "StackGuardian bulk workflow payload",
+  "description": "Validates the array of workflow objects produced by the transformer (sg-payload.<project>.json). Constraints (ResourceName length, kind/sourceConfigDestKind enums) are derived from schema/sg-openapi.json (#/components/schemas/Workflow and related). Lenient by design: only the fields the transformer emits are checked; unknown fields are allowed so the schema does not need to track every optional API field.",
+  "type": "array",
+  "items": { "$ref": "#/definitions/workflow" },
+  "definitions": {
+    "workflow": {
+      "type": "object",
+      "required": ["ResourceName", "WfType", "VCSConfig"],
+      "properties": {
+        "ResourceName": { "type": "string", "minLength": 1, "maxLength": 100 },
+        "WfType": { "type": "string", "minLength": 1 },
+        "Description": { "type": "string" },
+        "Tags": { "type": "array", "items": { "type": "string" } },
+        "Approvers": { "type": "array", "items": { "type": "string" } },
+        "UserSchedules": { "type": "array" },
+        "EnvironmentVariables": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "required": ["kind", "config"],
+            "properties": {
+              "kind": { "type": "string" },
+              "config": {
+                "type": "object",
+                "required": ["varName"],
+                "properties": {
+                  "varName": { "type": "string", "minLength": 1 },
+                  "textValue": { "type": "string" }
+                }
+              }
+            }
+          }
+        },
+        "DeploymentPlatformConfig": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "required": ["kind", "config"],
+            "properties": {
+              "kind": {
+                "enum": ["AWS_STATIC", "AWS_RBAC", "AWS_OIDC", "AZURE_STATIC", "AZURE_OIDC", "AZURE_MANAGED_ID_OIDC", "GCP_STATIC", "GCP_OIDC"]
+              },
+              "config": { "type": "object" }
+            }
+          }
+        },
+        "RunnerConstraints": {
+          "type": "object",
+          "required": ["type"],
+          "properties": {
+            "type": { "type": "string" },
+            "names": { "type": "array", "items": { "type": "string" } }
+          }
+        },
+        "VCSConfig": {
+          "type": "object",
+          "required": ["iacVCSConfig", "iacInputData"],
+          "properties": {
+            "iacVCSConfig": {
+              "type": "object",
+              "properties": {
+                "customSource": {
+                  "type": "object",
+                  "properties": {
+                    "sourceConfigDestKind": {
+                      "enum": ["GITHUB_COM", "GITHUB_APP_CUSTOM", "GIT_OTHER", "INLINE", "BITBUCKET_ORG", "GITLAB_COM", "AZURE_DEVOPS"]
+                    }
+                  }
+                }
+              }
+            },
+            "iacInputData": {
+              "type": "object",
+              "required": ["schemaType", "data"],
+              "properties": {
+                "schemaType": { "type": "string" },
+                "data": { "type": "object" }
+              }
+            }
+          }
+        },
+        "TerraformConfig": {
+          "type": "object",
+          "properties": {
+            "terraformVersion": { "type": "string", "minLength": 1 },
+            "managedTerraformState": { "type": "boolean" },
+            "approvalPreApply": { "type": "boolean" }
+          }
+        },
+        "VCSTriggers": {
+          "type": ["object", "null"],
+          "required": ["type"],
+          "properties": {
+            "type": {
+              "enum": ["GITHUB_COM", "GITHUB_APP_CUSTOM", "GITLAB_OAUTH_SSH", "BITBUCKET_ORG", "GITLAB_COM", "AZURE_DEVOPS", "AZURE_DEVOPS_SP"]
+            },
+            "tracked_branch": { "type": ["string", "null"] },
+            "approval_pre_apply": { "type": "boolean" },
+            "plan_only": { "type": "boolean" },
+            "gh_check": { "type": "boolean" },
+            "gl_pipeline": { "type": "boolean" },
+            "post_comments": { "type": "boolean" },
+            "file_triggers_enabled": { "type": "boolean" },
+            "file_trigger_patterns": { "type": "array", "items": { "type": "string" } },
+            "tags_regex": { "type": ["string", "null"] },
+            "push": { "type": "object" },
+            "pull_request_opened": { "type": "object" },
+            "pull_request_modified": { "type": "object" },
+            "all_pull_requests": { "type": "object" },
+            "create_tag": { "type": "object" }
+          }
+        }
+      }
+    }
+  }
+}

convert_hcl_to_json.sh → scripts/convert_hcl_to_json.sh

@@ -1,49 +1,14 @@
 #!/bin/bash
 set -euo pipefail
 
-log() { echo "[convert_hcl_to_json] $*" >&2; }
-
-WORKDIR=$(mktemp -d)
-cleanup() { rm -rf "$WORKDIR"; }
-trap cleanup EXIT
-
-# Normalize OS/arch to the names used by the jq and hcl2json release assets.
-OS=$(uname -s)
-case "$OS" in
-  Darwin) OS="macos" ;;
-  Linux) OS="linux" ;;
-  *) echo "Unsupported OS: $OS" >&2; exit 1 ;;
-esac
-
-ARCH=$(uname -m)
-case "$ARCH" in
-  x86_64 | amd64) ARCH="amd64" ;;
-  aarch64 | arm64) ARCH="arm64" ;;
-  *) echo "Unsupported architecture: $ARCH" >&2; exit 1 ;;
-esac
-
-JQ_BIN="$WORKDIR/jq"
-HCL2JSON_BIN="$WORKDIR/hcl2json"
-
-install_jq() {
-  local url="https://github.com/jqlang/jq/releases/download/jq-1.8.1/jq-${OS}-${ARCH}"
-  if ! curl -fsSL -o "$JQ_BIN" "$url"; then
-    echo "Failed to download jq from $url" >&2
-    exit 1
-  fi
-  chmod +x "$JQ_BIN"
-}
-
-install_hcl2json() {
-  # hcl2json uses "darwin" rather than "macos" for the OS segment.
-  local hcl_os="$OS"
-  [[ "$hcl_os" == "macos" ]] && hcl_os="darwin"
-  local url="https://github.com/tmccombs/hcl2json/releases/download/v0.6.7/hcl2json_${hcl_os}_${ARCH}"
-  if ! curl -fsSL -o "$HCL2JSON_BIN" "$url"; then
-    echo "Failed to download hcl2json from $url" >&2
-    exit 1
-  fi
-  chmod +x "$HCL2JSON_BIN"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=tools.sh
+source "$SCRIPT_DIR/tools.sh"
+
+# Detail lines are shown only in verbose mode; warnings always show.
+log() {
+  [ "${SG_VERBOSE:-0}" = "1" ] || return 0
+  printf '%s[convert]%s %s\n' "$C_CYAN" "$C_RESET" "$*" >&2
 }
 
 INPUT_FILE_JSON="${1:-}"
@@ -56,24 +21,31 @@ if [ ! -f "$INPUT_FILE_JSON" ]; then
   exit 1
 fi
 
-log "Downloading jq and hcl2json..."
-install_jq
-install_hcl2json
+WORKDIR=$(mktemp -d)
+cleanup() { rm -rf "$WORKDIR"; }
+trap cleanup EXIT
+
+# Resolve tooling from PATH (Docker image) or download+cache (native).
+JQ_BIN=$(sg_resolve jq sg_ensure_jq)
+HCL2JSON_BIN=$(sg_resolve hcl2json sg_ensure_hcl2json)
 
 # Read entire JSON array into a variable
 json_data=$(cat "$INPUT_FILE_JSON")
 
 # Use jq to get the length of array
 length=$($JQ_BIN length <<<"$json_data")
-log "Processing $length workflow(s) from $INPUT_FILE_JSON"
+log "Processing $length workflow(s) from $(sg_rel "$INPUT_FILE_JSON")"
 
 # Accumulate updated objects as newline-delimited JSON
 tmpfile="$WORKDIR/updated.ndjson"
 : >"$tmpfile"
 
 JSON_PATH=".VCSConfig.iacInputData.data"
+converted=0
+touched=0
 
 for ((i = 0; i < length; i++)); do
+  wf_converted=0
   # Extract ith object
   obj=$($JQ_BIN ".[$i]" <<<"$json_data")
 
@@ -117,13 +89,16 @@ for ((i = 0; i < length; i++)); do
     if [[ -n "$parsed" && "$parsed" != "null" ]]; then
       log "  workflow $((i + 1)): converted '$key' from HCL to JSON"
       new_val=$($JQ_BIN --arg k "$key" --argjson v "$parsed" '. + {($k): $v}' <<<"$new_val")
+      converted=$((converted + 1))
+      wf_converted=1
     else
-      log "  workflow $((i + 1)): parsing failed, keeping original value for '$key'"
+      sg_warn "$(sg_rel "$INPUT_FILE_JSON") workflow $((i + 1)): could not parse '$key' as HCL; keeping original value"
     fi
   done < <($JQ_BIN -r 'keys[]' <<<"$val")
 
   # Assign the converted data back at JSON_PATH
   updated_obj=$($JQ_BIN --argjson nv "$new_val" "$JSON_PATH = \$nv" <<<"$obj")
+  touched=$((touched + wf_converted))
 
   echo "$updated_obj" >>"$tmpfile"
 done
@@ -133,4 +108,9 @@ done
 outfile="$WORKDIR/output.json"
 $JQ_BIN -s '.' "$tmpfile" >"$outfile"
 mv "$outfile" "$INPUT_FILE_JSON"
-log "Done. Updated $INPUT_FILE_JSON in place."
+# One result line per file (the orchestrator shows it as-is).
+if [ "$converted" -gt 0 ]; then
+  sg_log "$(basename "$INPUT_FILE_JSON"): $converted HCL value(s) converted to JSON in $touched of $length workflow(s)"
+else
+  sg_log "$(basename "$INPUT_FILE_JSON"): nothing to convert ($length workflow(s), values already JSON)"
+fi