ci: resolve full Rust toolchain from manifest

.github/scripts/test_rust_ci_full_nextest_platform_workflow.py

@@ -6,10 +6,13 @@
 
 REPO_ROOT = Path(__file__).resolve().parents[2]
 WORKFLOW = REPO_ROOT / ".github" / "workflows" / "rust-ci-full-nextest-platform.yml"
+FULL_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "rust-ci-full.yml"
 HOSTED_CLEANUP_IF = (
     "${{ inputs.hosted_linux_preinstalled_tool_cleanup && runner.os == 'Linux' && "
     "runner.environment == 'github-hosted' && runner.arch == 'X64' }}"
 )
+LINUX_REMOTE_RUNNER = "blacksmith-16vcpu-ubuntu-2404"
+REMOTE_CAPABILITY_FILTER = "binary_id(codex-core::all) and test(/^suite::remote_env::/)"
 
 
 class RustCiFullNextestPlatformWorkflowTest(unittest.TestCase):
@@ -38,6 +41,120 @@ def test_hosted_cleanup_is_limited_to_x64_linux_runners(self) -> None:
                 self.assertNotIn("docker system prune", run)
                 self.assertNotIn("apt-get clean", run)
 
+    def test_linux_remote_archive_and_shards_use_capacity_runner(self) -> None:
+        workflow = yaml.safe_load(FULL_WORKFLOW.read_text(encoding="utf-8"))
+
+        self.assertEqual(
+            {
+                "runner": LINUX_REMOTE_RUNNER,
+                "target": "x86_64-unknown-linux-gnu",
+                "profile": "ci-test",
+                "artifact_id": "linux-x64-remote",
+                "remote_test_filter": REMOTE_CAPABILITY_FILTER,
+                "use_sccache": True,
+                "hosted_linux_preinstalled_tool_cleanup": True,
+            },
+            workflow["jobs"]["tests_linux_x64_remote"]["with"],
+        )
+
+    def test_remote_environment_is_scoped_to_capability_tests(self) -> None:
+        workflow = yaml.safe_load(WORKFLOW.read_text(encoding="utf-8"))
+        workflow_call = workflow.get("on", workflow.get(True))["workflow_call"]
+        inputs = workflow_call["inputs"]
+        shard_steps = workflow["jobs"]["shard"]["steps"]
+        setup_step = next(
+            step
+            for step in shard_steps
+            if step.get("name") == "Set up remote test env (Docker)"
+        )
+        ordinary_test_step = next(
+            step for step in shard_steps if step.get("id") == "test"
+        )
+        remote_test_step = next(
+            step for step in shard_steps if step.get("id") == "remote_test"
+        )
+        junit_steps = [
+            step
+            for step in shard_steps
+            if step.get("name") == "Upload nextest JUnit report"
+        ]
+        verify_step = next(
+            step for step in shard_steps if step.get("name") == "verify tests passed"
+        )
+
+        self.assertNotIn("remote_env", inputs)
+        self.assertEqual("string", inputs["remote_test_filter"]["type"])
+        self.assertNotIn("$GITHUB_ENV", setup_step["run"])
+        self.assertIn("$GITHUB_OUTPUT", setup_step["run"])
+        self.assertNotIn("CODEX_TEST_REMOTE_ENV", ordinary_test_step["run"])
+        self.assertEqual(
+            "${{ inputs.remote_test_filter == '' }}",
+            ordinary_test_step["if"],
+        )
+        self.assertLess(
+            shard_steps.index(ordinary_test_step),
+            shard_steps.index(setup_step),
+        )
+        self.assertLess(
+            shard_steps.index(setup_step),
+            shard_steps.index(remote_test_step),
+        )
+        expected_remote_env = {
+            "CODEX_TEST_REMOTE_ENV": "${{ steps.remote_env.outputs.container_name }}",
+            "CODEX_TEST_REMOTE_EXEC_SERVER_URL": (
+                "${{ steps.remote_env.outputs.exec_server_url }}"
+            ),
+            "CODEX_TEST_REMOTE_CODEX_PATH": (
+                "${{ steps.remote_env.outputs.codex_path }}"
+            ),
+            "REMOTE_TEST_FILTER": "${{ inputs.remote_test_filter }}",
+        }
+        self.assertEqual(
+            expected_remote_env,
+            {key: remote_test_step["env"][key] for key in expected_remote_env},
+        )
+        self.assertIn("--filterset", remote_test_step["run"])
+        self.assertIn('"${REMOTE_TEST_FILTER}"', remote_test_step["run"])
+        self.assertIn("--no-tests pass", remote_test_step["run"])
+        self.assertNotIn("--skip", remote_test_step["run"])
+        self.assertEqual(1, len(junit_steps))
+        self.assertEqual("always()", junit_steps[0]["if"])
+        self.assertLess(
+            shard_steps.index(remote_test_step),
+            shard_steps.index(junit_steps[0]),
+        )
+        self.assertIn("!cancelled()", setup_step["if"])
+        self.assertEqual(
+            "${{ !cancelled() && (steps.test.outcome == 'failure' || "
+            "steps.remote_test.outcome == 'failure') }}",
+            verify_step["if"],
+        )
+
+    def test_user_namespace_setup_is_shard_only(self) -> None:
+        workflow = yaml.safe_load(WORKFLOW.read_text(encoding="utf-8"))
+        user_namespace_step = "Enable unprivileged user namespaces (Linux)"
+        shard_steps = workflow["jobs"]["shard"]["steps"]
+        shard_user_namespace_step = next(
+            step for step in shard_steps if step.get("name") == user_namespace_step
+        )
+
+        self.assertNotIn(
+            user_namespace_step,
+            [step.get("name") for step in workflow["jobs"]["archive"]["steps"]],
+        )
+        self.assertIn(
+            user_namespace_step,
+            [step.get("name") for step in shard_steps],
+        )
+        self.assertIn(
+            "/proc/sys/kernel/unprivileged_userns_clone",
+            shard_user_namespace_step["run"],
+        )
+        self.assertIn(
+            "/proc/sys/kernel/apparmor_restrict_unprivileged_userns",
+            shard_user_namespace_step["run"],
+        )
+
 
 if __name__ == "__main__":
     unittest.main()

.github/workflows/rust-ci-full-nextest-platform.yml

@@ -35,10 +35,10 @@ on:
       artifact_id:
         required: true
         type: string
-      remote_env:
+      remote_test_filter:
         required: false
-        default: false
-        type: boolean
+        default: ""
+        type: string
       test_threads:
         required: false
         default: 0
@@ -152,9 +152,14 @@ jobs:
       - name: Install DotSlash
         uses: facebook/install-dotslash@1e4e7b3e07eaca387acb98f1d4720e0bee8dbb6a # v2
 
-      - uses: dtolnay/rust-toolchain@e081816240890017053eacbb1bdf337761dc5582 # 1.95.0
-        with:
-          targets: ${{ inputs.target }}
+      - name: Install Rust toolchain from rust-toolchain.toml
+        shell: bash
+        run: |
+          set -euo pipefail
+          rustc --version
+          rustup target add "${{ inputs.target }}"
+          rustup show active-toolchain
+          rustup target list --installed
 
       - name: Expose MSVC SDK environment (Windows)
         if: ${{ runner.os == 'Windows' && inputs.target == 'aarch64-pc-windows-msvc' }}
@@ -234,14 +239,6 @@ jobs:
         with:
           tool: [email protected]
 
-      - name: Enable unprivileged user namespaces (Linux)
-        if: runner.os == 'Linux'
-        run: |
-          sudo sysctl -w kernel.unprivileged_userns_clone=1
-          if sudo sysctl -a 2>/dev/null | grep -q '^kernel.apparmor_restrict_unprivileged_userns'; then
-            sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
-          fi
-
       - name: Build nextest archive
         shell: bash
         run: |
@@ -416,9 +413,14 @@ jobs:
       - name: Install DotSlash
         uses: facebook/install-dotslash@1e4e7b3e07eaca387acb98f1d4720e0bee8dbb6a # v2
 
-      - uses: dtolnay/rust-toolchain@e081816240890017053eacbb1bdf337761dc5582 # 1.95.0
-        with:
-          targets: ${{ inputs.target }}
+      - name: Install Rust toolchain from rust-toolchain.toml
+        shell: bash
+        run: |
+          set -euo pipefail
+          rustc --version
+          rustup target add "${{ inputs.target }}"
+          rustup show active-toolchain
+          rustup target list --installed
 
       - uses: taiki-e/install-action@43aecc8d72668fbcfe75c31400bc4f890f1c5853 # v2.83.2
         with:
@@ -427,21 +429,14 @@ jobs:
       - name: Enable unprivileged user namespaces (Linux)
         if: runner.os == 'Linux'
         run: |
-          sudo sysctl -w kernel.unprivileged_userns_clone=1
-          if sudo sysctl -a 2>/dev/null | grep -q '^kernel.apparmor_restrict_unprivileged_userns'; then
+          set -euo pipefail
+          if [[ -e /proc/sys/kernel/unprivileged_userns_clone ]]; then
+            sudo sysctl -w kernel.unprivileged_userns_clone=1
+          fi
+          if [[ -e /proc/sys/kernel/apparmor_restrict_unprivileged_userns ]]; then
             sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
           fi
 
-      - name: Set up remote test env (Docker)
-        if: ${{ runner.os == 'Linux' && inputs.remote_env }}
-        shell: bash
-        run: |
-          set -euo pipefail
-          export CODEX_TEST_REMOTE_ENV_CONTAINER_NAME="codex-remote-test-env-${{ github.run_id }}-${{ matrix.shard }}"
-          source "${GITHUB_WORKSPACE}/scripts/test-remote-env.sh"
-          echo "CODEX_TEST_REMOTE_ENV=${CODEX_TEST_REMOTE_ENV}" >> "$GITHUB_ENV"
-          echo "CODEX_TEST_REMOTE_EXEC_SERVER_URL=${CODEX_TEST_REMOTE_EXEC_SERVER_URL}" >> "$GITHUB_ENV"
-
       - name: Download nextest archive
         uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
@@ -463,6 +458,7 @@ jobs:
 
       - name: tests
         id: test
+        if: ${{ inputs.remote_test_filter == '' }}
         shell: bash
         run: |
           set -euo pipefail
@@ -558,6 +554,58 @@ jobs:
           RUST_MIN_STACK: "8388608" # 8 MiB
           NEXTEST_STATUS_LEVEL: leak
 
+      - name: Set up remote test env (Docker)
+        id: remote_env
+        if: ${{ !cancelled() && runner.os == 'Linux' && inputs.remote_test_filter != '' }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          export CODEX_TEST_REMOTE_ENV_CONTAINER_NAME="codex-remote-test-env-${{ github.run_id }}-${{ matrix.shard }}"
+          source "${GITHUB_WORKSPACE}/scripts/test-remote-env.sh"
+          {
+            echo "container_name=${CODEX_TEST_REMOTE_ENV}"
+            echo "exec_server_url=${CODEX_TEST_REMOTE_EXEC_SERVER_URL}"
+            echo "codex_path=${CODEX_TEST_REMOTE_CODEX_PATH}"
+          } >> "$GITHUB_OUTPUT"
+
+      - name: Run remote-exec capability tests
+        id: remote_test
+        if: ${{ !cancelled() && runner.os == 'Linux' && inputs.remote_test_filter != '' }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          archive_file="${RUNNER_TEMP}/nextest-archive/${NEXTEST_ARCHIVE_FILE}"
+          workspace_root="$(pwd)"
+          helper_target_dir="${workspace_root}/target/${{ inputs.target }}/${{ inputs.profile }}"
+          sandbox_helper="${helper_target_dir}/codex-linux-sandbox"
+
+          remote_nextest_args=(
+            run
+            --no-fail-fast
+            --archive-file "${archive_file}"
+            --workspace-remap "${workspace_root}"
+            --filterset "${REMOTE_TEST_FILTER}"
+            --partition "hash:${{ matrix.shard }}/4"
+            --no-tests pass
+          )
+          if [[ "${{ inputs.test_threads }}" != "0" ]]; then
+            remote_nextest_args+=(--test-threads "${{ inputs.test_threads }}")
+          fi
+
+          env \
+            "INSTA_WORKSPACE_ROOT=${workspace_root}" \
+            "CARGO_BIN_EXE_codex-linux-sandbox=${sandbox_helper}" \
+            "CARGO_BIN_EXE_codex_linux_sandbox=${sandbox_helper}" \
+            cargo nextest "${remote_nextest_args[@]}"
+        env:
+          CODEX_TEST_REMOTE_ENV: ${{ steps.remote_env.outputs.container_name }}
+          CODEX_TEST_REMOTE_EXEC_SERVER_URL: ${{ steps.remote_env.outputs.exec_server_url }}
+          CODEX_TEST_REMOTE_CODEX_PATH: ${{ steps.remote_env.outputs.codex_path }}
+          REMOTE_TEST_FILTER: ${{ inputs.remote_test_filter }}
+          RUST_BACKTRACE: 1
+          RUST_MIN_STACK: "8388608" # 8 MiB
+          NEXTEST_STATUS_LEVEL: leak
+
       - name: Upload nextest JUnit report
         if: always()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
@@ -567,19 +615,20 @@ jobs:
           if-no-files-found: warn
 
       - name: Tear down remote test env
-        if: ${{ always() && runner.os == 'Linux' && inputs.remote_env }}
+        if: ${{ always() && runner.os == 'Linux' && inputs.remote_test_filter != '' }}
         shell: bash
         run: |
           set +e
-          if [[ "${STEPS_TEST_OUTCOME}" != "success" ]]; then
-            docker logs "${CODEX_TEST_REMOTE_ENV}" || true
+          if [[ "${REMOTE_TEST_OUTCOME}" != "success" ]]; then
+            docker logs "${REMOTE_ENV_CONTAINER_NAME}" || true
           fi
-          docker rm -f "${CODEX_TEST_REMOTE_ENV}" >/dev/null 2>&1 || true
+          docker rm -f "${REMOTE_ENV_CONTAINER_NAME}" >/dev/null 2>&1 || true
         env:
-          STEPS_TEST_OUTCOME: ${{ steps.test.outcome }}
+          REMOTE_ENV_CONTAINER_NAME: ${{ steps.remote_env.outputs.container_name }}
+          REMOTE_TEST_OUTCOME: ${{ steps.remote_test.outcome }}
 
       - name: verify tests passed
-        if: steps.test.outcome == 'failure'
+        if: ${{ !cancelled() && (steps.test.outcome == 'failure' || steps.remote_test.outcome == 'failure') }}
         run: |
           echo "Tests failed. See logs for details."
           exit 1

.github/workflows/rust-ci-full.yml

@@ -232,10 +232,14 @@ jobs:
             sudo apt-get update -y
             sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends pkg-config libcap-dev
           fi
-      - uses: dtolnay/rust-toolchain@e081816240890017053eacbb1bdf337761dc5582 # 1.95.0
-        with:
-          targets: ${{ matrix.target }}
-          components: clippy
+      - name: Install Rust toolchain from rust-toolchain.toml
+        shell: bash
+        run: |
+          set -euo pipefail
+          rustc --version
+          rustup target add "${{ matrix.target }}"
+          rustup show active-toolchain
+          rustup target list --installed
 
       - if: ${{ matrix.target == 'x86_64-unknown-linux-musl' || matrix.target == 'aarch64-unknown-linux-musl'}}
         name: Use hermetic Cargo home (musl)
@@ -445,11 +449,11 @@ jobs:
     name: Tests — ubuntu-24.04 - x86_64-unknown-linux-gnu (remote)
     uses: ./.github/workflows/rust-ci-full-nextest-platform.yml
     with:
-      runner: ubuntu-24.04
+      runner: blacksmith-16vcpu-ubuntu-2404
       target: x86_64-unknown-linux-gnu
       profile: ci-test
       artifact_id: linux-x64-remote
-      remote_env: true
+      remote_test_filter: "binary_id(codex-core::all) and test(/^suite::remote_env::/)"
       use_sccache: true
       hosted_linux_preinstalled_tool_cleanup: true
     secrets: inherit