diff --git a/.github/scripts/test_rust_ci_full_nextest_platform_workflow.py b/.github/scripts/test_rust_ci_full_nextest_platform_workflow.py index 815f33669b..c6fecccfd7 100644 --- a/.github/scripts/test_rust_ci_full_nextest_platform_workflow.py +++ b/.github/scripts/test_rust_ci_full_nextest_platform_workflow.py @@ -6,10 +6,13 @@ REPO_ROOT = Path(__file__).resolve().parents[2] WORKFLOW = REPO_ROOT / ".github" / "workflows" / "rust-ci-full-nextest-platform.yml" +FULL_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "rust-ci-full.yml" HOSTED_CLEANUP_IF = ( "${{ inputs.hosted_linux_preinstalled_tool_cleanup && runner.os == 'Linux' && " "runner.environment == 'github-hosted' && runner.arch == 'X64' }}" ) +LINUX_REMOTE_RUNNER = "blacksmith-16vcpu-ubuntu-2404" +REMOTE_CAPABILITY_FILTER = "binary_id(codex-core::all) and test(/^suite::remote_env::/)" class RustCiFullNextestPlatformWorkflowTest(unittest.TestCase): @@ -38,6 +41,120 @@ def test_hosted_cleanup_is_limited_to_x64_linux_runners(self) -> None: self.assertNotIn("docker system prune", run) self.assertNotIn("apt-get clean", run) + def test_linux_remote_archive_and_shards_use_capacity_runner(self) -> None: + workflow = yaml.safe_load(FULL_WORKFLOW.read_text(encoding="utf-8")) + + self.assertEqual( + { + "runner": LINUX_REMOTE_RUNNER, + "target": "x86_64-unknown-linux-gnu", + "profile": "ci-test", + "artifact_id": "linux-x64-remote", + "remote_test_filter": REMOTE_CAPABILITY_FILTER, + "use_sccache": True, + "hosted_linux_preinstalled_tool_cleanup": True, + }, + workflow["jobs"]["tests_linux_x64_remote"]["with"], + ) + + def test_remote_environment_is_scoped_to_capability_tests(self) -> None: + workflow = yaml.safe_load(WORKFLOW.read_text(encoding="utf-8")) + workflow_call = workflow.get("on", workflow.get(True))["workflow_call"] + inputs = workflow_call["inputs"] + shard_steps = workflow["jobs"]["shard"]["steps"] + setup_step = next( + step + for step in shard_steps + if step.get("name") == "Set up remote test env (Docker)" + ) + ordinary_test_step = next( + step for step in shard_steps if step.get("id") == "test" + ) + remote_test_step = next( + step for step in shard_steps if step.get("id") == "remote_test" + ) + junit_steps = [ + step + for step in shard_steps + if step.get("name") == "Upload nextest JUnit report" + ] + verify_step = next( + step for step in shard_steps if step.get("name") == "verify tests passed" + ) + + self.assertNotIn("remote_env", inputs) + self.assertEqual("string", inputs["remote_test_filter"]["type"]) + self.assertNotIn("$GITHUB_ENV", setup_step["run"]) + self.assertIn("$GITHUB_OUTPUT", setup_step["run"]) + self.assertNotIn("CODEX_TEST_REMOTE_ENV", ordinary_test_step["run"]) + self.assertEqual( + "${{ inputs.remote_test_filter == '' }}", + ordinary_test_step["if"], + ) + self.assertLess( + shard_steps.index(ordinary_test_step), + shard_steps.index(setup_step), + ) + self.assertLess( + shard_steps.index(setup_step), + shard_steps.index(remote_test_step), + ) + expected_remote_env = { + "CODEX_TEST_REMOTE_ENV": "${{ steps.remote_env.outputs.container_name }}", + "CODEX_TEST_REMOTE_EXEC_SERVER_URL": ( + "${{ steps.remote_env.outputs.exec_server_url }}" + ), + "CODEX_TEST_REMOTE_CODEX_PATH": ( + "${{ steps.remote_env.outputs.codex_path }}" + ), + "REMOTE_TEST_FILTER": "${{ inputs.remote_test_filter }}", + } + self.assertEqual( + expected_remote_env, + {key: remote_test_step["env"][key] for key in expected_remote_env}, + ) + self.assertIn("--filterset", remote_test_step["run"]) + self.assertIn('"${REMOTE_TEST_FILTER}"', remote_test_step["run"]) + self.assertIn("--no-tests pass", remote_test_step["run"]) + self.assertNotIn("--skip", remote_test_step["run"]) + self.assertEqual(1, len(junit_steps)) + self.assertEqual("always()", junit_steps[0]["if"]) + self.assertLess( + shard_steps.index(remote_test_step), + shard_steps.index(junit_steps[0]), + ) + self.assertIn("!cancelled()", setup_step["if"]) + self.assertEqual( + "${{ !cancelled() && (steps.test.outcome == 'failure' || " + "steps.remote_test.outcome == 'failure') }}", + verify_step["if"], + ) + + def test_user_namespace_setup_is_shard_only(self) -> None: + workflow = yaml.safe_load(WORKFLOW.read_text(encoding="utf-8")) + user_namespace_step = "Enable unprivileged user namespaces (Linux)" + shard_steps = workflow["jobs"]["shard"]["steps"] + shard_user_namespace_step = next( + step for step in shard_steps if step.get("name") == user_namespace_step + ) + + self.assertNotIn( + user_namespace_step, + [step.get("name") for step in workflow["jobs"]["archive"]["steps"]], + ) + self.assertIn( + user_namespace_step, + [step.get("name") for step in shard_steps], + ) + self.assertIn( + "/proc/sys/kernel/unprivileged_userns_clone", + shard_user_namespace_step["run"], + ) + self.assertIn( + "/proc/sys/kernel/apparmor_restrict_unprivileged_userns", + shard_user_namespace_step["run"], + ) + if __name__ == "__main__": unittest.main() diff --git a/.github/workflows/rust-ci-full-nextest-platform.yml b/.github/workflows/rust-ci-full-nextest-platform.yml index 2181271aae..80aa2fe51f 100644 --- a/.github/workflows/rust-ci-full-nextest-platform.yml +++ b/.github/workflows/rust-ci-full-nextest-platform.yml @@ -35,10 +35,10 @@ on: artifact_id: required: true type: string - remote_env: + remote_test_filter: required: false - default: false - type: boolean + default: "" + type: string test_threads: required: false default: 0 @@ -152,9 +152,14 @@ jobs: - name: Install DotSlash uses: facebook/install-dotslash@1e4e7b3e07eaca387acb98f1d4720e0bee8dbb6a # v2 - - uses: dtolnay/rust-toolchain@e081816240890017053eacbb1bdf337761dc5582 # 1.95.0 - with: - targets: ${{ inputs.target }} + - name: Install Rust toolchain from rust-toolchain.toml + shell: bash + run: | + set -euo pipefail + rustc --version + rustup target add "${{ inputs.target }}" + rustup show active-toolchain + rustup target list --installed - name: Expose MSVC SDK environment (Windows) if: ${{ runner.os == 'Windows' && inputs.target == 'aarch64-pc-windows-msvc' }} @@ -234,14 +239,6 @@ jobs: with: tool: nextest@0.9.103 - - name: Enable unprivileged user namespaces (Linux) - if: runner.os == 'Linux' - run: | - sudo sysctl -w kernel.unprivileged_userns_clone=1 - if sudo sysctl -a 2>/dev/null | grep -q '^kernel.apparmor_restrict_unprivileged_userns'; then - sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 - fi - - name: Build nextest archive shell: bash run: | @@ -416,9 +413,14 @@ jobs: - name: Install DotSlash uses: facebook/install-dotslash@1e4e7b3e07eaca387acb98f1d4720e0bee8dbb6a # v2 - - uses: dtolnay/rust-toolchain@e081816240890017053eacbb1bdf337761dc5582 # 1.95.0 - with: - targets: ${{ inputs.target }} + - name: Install Rust toolchain from rust-toolchain.toml + shell: bash + run: | + set -euo pipefail + rustc --version + rustup target add "${{ inputs.target }}" + rustup show active-toolchain + rustup target list --installed - uses: taiki-e/install-action@43aecc8d72668fbcfe75c31400bc4f890f1c5853 # v2.83.2 with: @@ -427,21 +429,14 @@ jobs: - name: Enable unprivileged user namespaces (Linux) if: runner.os == 'Linux' run: | - sudo sysctl -w kernel.unprivileged_userns_clone=1 - if sudo sysctl -a 2>/dev/null | grep -q '^kernel.apparmor_restrict_unprivileged_userns'; then + set -euo pipefail + if [[ -e /proc/sys/kernel/unprivileged_userns_clone ]]; then + sudo sysctl -w kernel.unprivileged_userns_clone=1 + fi + if [[ -e /proc/sys/kernel/apparmor_restrict_unprivileged_userns ]]; then sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 fi - - name: Set up remote test env (Docker) - if: ${{ runner.os == 'Linux' && inputs.remote_env }} - shell: bash - run: | - set -euo pipefail - export CODEX_TEST_REMOTE_ENV_CONTAINER_NAME="codex-remote-test-env-${{ github.run_id }}-${{ matrix.shard }}" - source "${GITHUB_WORKSPACE}/scripts/test-remote-env.sh" - echo "CODEX_TEST_REMOTE_ENV=${CODEX_TEST_REMOTE_ENV}" >> "$GITHUB_ENV" - echo "CODEX_TEST_REMOTE_EXEC_SERVER_URL=${CODEX_TEST_REMOTE_EXEC_SERVER_URL}" >> "$GITHUB_ENV" - - name: Download nextest archive uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: @@ -463,6 +458,7 @@ jobs: - name: tests id: test + if: ${{ inputs.remote_test_filter == '' }} shell: bash run: | set -euo pipefail @@ -558,6 +554,58 @@ jobs: RUST_MIN_STACK: "8388608" # 8 MiB NEXTEST_STATUS_LEVEL: leak + - name: Set up remote test env (Docker) + id: remote_env + if: ${{ !cancelled() && runner.os == 'Linux' && inputs.remote_test_filter != '' }} + shell: bash + run: | + set -euo pipefail + export CODEX_TEST_REMOTE_ENV_CONTAINER_NAME="codex-remote-test-env-${{ github.run_id }}-${{ matrix.shard }}" + source "${GITHUB_WORKSPACE}/scripts/test-remote-env.sh" + { + echo "container_name=${CODEX_TEST_REMOTE_ENV}" + echo "exec_server_url=${CODEX_TEST_REMOTE_EXEC_SERVER_URL}" + echo "codex_path=${CODEX_TEST_REMOTE_CODEX_PATH}" + } >> "$GITHUB_OUTPUT" + + - name: Run remote-exec capability tests + id: remote_test + if: ${{ !cancelled() && runner.os == 'Linux' && inputs.remote_test_filter != '' }} + shell: bash + run: | + set -euo pipefail + archive_file="${RUNNER_TEMP}/nextest-archive/${NEXTEST_ARCHIVE_FILE}" + workspace_root="$(pwd)" + helper_target_dir="${workspace_root}/target/${{ inputs.target }}/${{ inputs.profile }}" + sandbox_helper="${helper_target_dir}/codex-linux-sandbox" + + remote_nextest_args=( + run + --no-fail-fast + --archive-file "${archive_file}" + --workspace-remap "${workspace_root}" + --filterset "${REMOTE_TEST_FILTER}" + --partition "hash:${{ matrix.shard }}/4" + --no-tests pass + ) + if [[ "${{ inputs.test_threads }}" != "0" ]]; then + remote_nextest_args+=(--test-threads "${{ inputs.test_threads }}") + fi + + env \ + "INSTA_WORKSPACE_ROOT=${workspace_root}" \ + "CARGO_BIN_EXE_codex-linux-sandbox=${sandbox_helper}" \ + "CARGO_BIN_EXE_codex_linux_sandbox=${sandbox_helper}" \ + cargo nextest "${remote_nextest_args[@]}" + env: + CODEX_TEST_REMOTE_ENV: ${{ steps.remote_env.outputs.container_name }} + CODEX_TEST_REMOTE_EXEC_SERVER_URL: ${{ steps.remote_env.outputs.exec_server_url }} + CODEX_TEST_REMOTE_CODEX_PATH: ${{ steps.remote_env.outputs.codex_path }} + REMOTE_TEST_FILTER: ${{ inputs.remote_test_filter }} + RUST_BACKTRACE: 1 + RUST_MIN_STACK: "8388608" # 8 MiB + NEXTEST_STATUS_LEVEL: leak + - name: Upload nextest JUnit report if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 @@ -567,19 +615,20 @@ jobs: if-no-files-found: warn - name: Tear down remote test env - if: ${{ always() && runner.os == 'Linux' && inputs.remote_env }} + if: ${{ always() && runner.os == 'Linux' && inputs.remote_test_filter != '' }} shell: bash run: | set +e - if [[ "${STEPS_TEST_OUTCOME}" != "success" ]]; then - docker logs "${CODEX_TEST_REMOTE_ENV}" || true + if [[ "${REMOTE_TEST_OUTCOME}" != "success" ]]; then + docker logs "${REMOTE_ENV_CONTAINER_NAME}" || true fi - docker rm -f "${CODEX_TEST_REMOTE_ENV}" >/dev/null 2>&1 || true + docker rm -f "${REMOTE_ENV_CONTAINER_NAME}" >/dev/null 2>&1 || true env: - STEPS_TEST_OUTCOME: ${{ steps.test.outcome }} + REMOTE_ENV_CONTAINER_NAME: ${{ steps.remote_env.outputs.container_name }} + REMOTE_TEST_OUTCOME: ${{ steps.remote_test.outcome }} - name: verify tests passed - if: steps.test.outcome == 'failure' + if: ${{ !cancelled() && (steps.test.outcome == 'failure' || steps.remote_test.outcome == 'failure') }} run: | echo "Tests failed. See logs for details." exit 1 diff --git a/.github/workflows/rust-ci-full.yml b/.github/workflows/rust-ci-full.yml index df327e2705..c67097a286 100644 --- a/.github/workflows/rust-ci-full.yml +++ b/.github/workflows/rust-ci-full.yml @@ -232,10 +232,14 @@ jobs: sudo apt-get update -y sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends pkg-config libcap-dev fi - - uses: dtolnay/rust-toolchain@e081816240890017053eacbb1bdf337761dc5582 # 1.95.0 - with: - targets: ${{ matrix.target }} - components: clippy + - name: Install Rust toolchain from rust-toolchain.toml + shell: bash + run: | + set -euo pipefail + rustc --version + rustup target add "${{ matrix.target }}" + rustup show active-toolchain + rustup target list --installed - if: ${{ matrix.target == 'x86_64-unknown-linux-musl' || matrix.target == 'aarch64-unknown-linux-musl'}} name: Use hermetic Cargo home (musl) @@ -445,11 +449,11 @@ jobs: name: Tests — ubuntu-24.04 - x86_64-unknown-linux-gnu (remote) uses: ./.github/workflows/rust-ci-full-nextest-platform.yml with: - runner: ubuntu-24.04 + runner: blacksmith-16vcpu-ubuntu-2404 target: x86_64-unknown-linux-gnu profile: ci-test artifact_id: linux-x64-remote - remote_env: true + remote_test_filter: "binary_id(codex-core::all) and test(/^suite::remote_env::/)" use_sccache: true hosted_linux_preinstalled_tool_cleanup: true secrets: inherit