From b09a14340b8f0b5bcaf14e6934e228c51c577706 Mon Sep 17 00:00:00 2001 From: Cheng-Chin Chiang Date: Fri, 21 Aug 2026 09:49:37 +0800 Subject: [PATCH 01/15] Check that the backend is listening instead of that it started pgrep matches as soon as carta_backend forks, which is well before it binds the port, and `pgrep ... | head -n 1` reports head's exit status, so a backend which died on startup still looked like a success. The macOS action ran `pgrep carta_backend` after each test file but only printed the result. Neither action could tell a dead backend from a failing test. Connect to the port instead, which is the only check that answers the question the tests ask, using bash's own /dev/tcp so no container needs a tool it may not have. Wait up to 120 s for it before the stage starts, and probe again after each test file: a backend which dies part way through takes every remaining file with it, each reporting a connection failure rather than the crash which caused it. The stage now names the file it died on and tails the backend log where it failed, instead of leaving the reason in an uploaded artifact. --- .github/actions/run-apptainer/action.yml | 19 ++++++++++++++++--- .github/actions/run-macos/action.yml | 16 +++++++++++++++- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/.github/actions/run-apptainer/action.yml b/.github/actions/run-apptainer/action.yml index a2b3e4ca..9ac91026 100644 --- a/.github/actions/run-apptainer/action.yml +++ b/.github/actions/run-apptainer/action.yml @@ -56,24 +56,37 @@ runs: --no_database \ --no_log \ --verbosity=5 >> $LOG_FILE 2>&1 & \ - CARTA_BACKEND_PID=\$(pgrep -f 'carta_backend.*${{ inputs.port }}' | head -n 1) && \ - echo 'carta_backend started with PID' \$CARTA_BACKEND_PID && \ + # pgrep matches as soon as the backend forks, which is before it binds the port + bash $BUILD_DIR/ICD-RxJS/scripts/wait_for_backend.sh ${{ inputs.port }} 120 $LOG_FILE && \ + echo 'carta_backend is listening with PID' \$(pgrep -f 'carta_backend.*${{ inputs.port }}' | head -n 1) && \ # Run the ICD tests cd $BUILD_DIR/ICD-RxJS && \ pwd && \ cat $TEST_STAGE && \ failed_tests=() && \ + backend_died='' && \ mapfile -t test_files < $TEST_STAGE && \ for test_file in \"\${test_files[@]}\"; do if [ -n \"\$test_file\" ]; then # Skip empty lines if ! CI=true npm test -- \"\$test_file\"; then failed_tests+=(\"\$test_file\") fi + # A backend which has died takes every remaining file in the stage with it, each + # reporting a connection failure instead of the crash which actually caused it. + if ! bash scripts/wait_for_backend.sh ${{ inputs.port }} 0 $LOG_FILE; then + backend_died=\"\$test_file\" + break + fi fi done && \ + if [ -n \"\$backend_died\" ]; then + echo \"carta_backend stopped listening during \$backend_died; abandoning the rest of the stage\" + fi && \ if [ \${#failed_tests[@]} -ne 0 ]; then echo \"The following tests failed:\" && \ - printf '%s\n' \"\${failed_tests[@]}\" && \ + printf '%s\n' \"\${failed_tests[@]}\" + fi && \ + if [ \${#failed_tests[@]} -ne 0 ] || [ -n \"\$backend_died\" ]; then exit 1 fi" shell: bash diff --git a/.github/actions/run-macos/action.yml b/.github/actions/run-macos/action.yml index bd6bcd7e..df35f8fb 100644 --- a/.github/actions/run-macos/action.yml +++ b/.github/actions/run-macos/action.yml @@ -30,6 +30,8 @@ runs: --verbosity=5 >> "$LOG_FILE" 2>&1 & echo "CARTA_BACKEND_PID=$!" >> $GITHUB_ENV echo "CARTA_LOG_FILE=$LOG_FILE" >> $GITHUB_ENV + # A PID only says the backend forked; the tests need the port to be bound. + bash $GITHUB_WORKSPACE/ICD-RxJS/scripts/wait_for_backend.sh ${{ inputs.port }} 120 "$LOG_FILE" shell: bash - name: ICD tests @@ -37,18 +39,30 @@ runs: ICD_DIR=$GITHUB_WORKSPACE/ICD-RxJS cd $ICD_DIR failed_tests=() + backend_died='' mapfile -t test_files < ICD_test_stages/${{ inputs.test_stage_name }}.tests for test_file in "${test_files[@]}"; do if [ -n "$test_file" ]; then if ! CI=true npm test -- "$test_file"; then failed_tests+=("$test_file") fi - sleep 3 && pgrep carta_backend + sleep 3 + # A backend which has died takes every remaining file in the stage with it, each + # reporting a connection failure instead of the crash which actually caused it. + if ! bash scripts/wait_for_backend.sh ${{ inputs.port }} 0 "$CARTA_LOG_FILE"; then + backend_died="$test_file" + break + fi fi done + if [ -n "$backend_died" ]; then + echo "carta_backend stopped listening during $backend_died; abandoning the rest of the stage" + fi if [ ${#failed_tests[@]} -ne 0 ]; then echo "The following tests failed:" printf '%s\n' "${failed_tests[@]}" + fi + if [ ${#failed_tests[@]} -ne 0 ] || [ -n "$backend_died" ]; then exit 1 fi shell: bash From ffda9c5b13e5bd35a64c5b241ad8e0fe39738605 Mon Sep 17 00:00:00 2001 From: Cheng-Chin Chiang Date: Fri, 21 Aug 2026 10:23:18 +0800 Subject: [PATCH 02/15] Add the CI guard script --- scripts/wait_for_backend.sh | 58 +++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100755 scripts/wait_for_backend.sh diff --git a/scripts/wait_for_backend.sh b/scripts/wait_for_backend.sh new file mode 100755 index 00000000..26c72cd6 --- /dev/null +++ b/scripts/wait_for_backend.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# +# Wait until carta_backend is accepting connections on a port. +# +# The CI actions used to take `pgrep` as proof that the backend was up, but pgrep matches as soon +# as the process has forked, which is well before it has bound the port -- and `pgrep ... | head` +# reports the exit status of head, so a backend which never started at all still looked like a +# success. The tests then opened the stage against a port nobody was listening on and reported it +# as a connection failure in the first test file, or, if the backend died part way through a +# stage, as the same failure in every file after it. +# +# Connecting is the only check which answers the question the tests actually ask, so that is what +# this does, using bash's own /dev/tcp redirection rather than a tool which may be missing from a +# container. +# +# Usage: wait_for_backend.sh PORT [TIMEOUT_SECONDS] [LOG_FILE] +# +# A timeout of 0 (the default) makes a single immediate check, which is how a stage asks whether +# the backend it started is still alive. LOG_FILE, if given, is tailed to stderr on failure, so +# that a crash is reported where the stage failed instead of only in an uploaded artifact. + +set -u + +port=${1:-} +timeout=${2:-0} +log_file=${3:-} + +if [ -z "$port" ]; then + echo "usage: $0 PORT [TIMEOUT_SECONDS] [LOG_FILE]" >&2 + exit 2 +fi + +deadline=$((SECONDS + timeout)) + +while :; do + # The subshell keeps the descriptor from leaking into the caller and turns a refused + # connection into an ordinary non-zero status rather than a message on stderr. + if (exec 3<>"/dev/tcp/127.0.0.1/$port") 2>/dev/null; then + exit 0 + fi + if [ "$SECONDS" -ge "$deadline" ]; then + break + fi + sleep 1 +done + +if [ "$timeout" -eq 0 ]; then + echo "carta_backend is not listening on port $port" >&2 +else + echo "carta_backend did not start listening on port $port within ${timeout}s" >&2 +fi + +if [ -n "$log_file" ] && [ -f "$log_file" ]; then + echo "--- last 100 lines of $log_file ---" >&2 + tail -n 100 "$log_file" >&2 +fi + +exit 1 From cb591d7681efe233fe58959e7f697d606c82657d Mon Sep 17 00:00:00 2001 From: Cheng-Chin Chiang Date: Fri, 21 Aug 2026 11:32:40 +0800 Subject: [PATCH 03/15] Name the tests a crashed backend stopped from running --- .github/actions/run-apptainer/action.yml | 17 +++++++++++++++-- .github/actions/run-macos/action.yml | 17 +++++++++++++++-- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/.github/actions/run-apptainer/action.yml b/.github/actions/run-apptainer/action.yml index 9ac91026..b4d27c09 100644 --- a/.github/actions/run-apptainer/action.yml +++ b/.github/actions/run-apptainer/action.yml @@ -65,8 +65,10 @@ runs: cat $TEST_STAGE && \ failed_tests=() && \ backend_died='' && \ + skipped_tests=() && \ mapfile -t test_files < $TEST_STAGE && \ - for test_file in \"\${test_files[@]}\"; do + for test_index in \"\${!test_files[@]}\"; do + test_file=\"\${test_files[\$test_index]}\" if [ -n \"\$test_file\" ]; then # Skip empty lines if ! CI=true npm test -- \"\$test_file\"; then failed_tests+=(\"\$test_file\") @@ -75,12 +77,23 @@ runs: # reporting a connection failure instead of the crash which actually caused it. if ! bash scripts/wait_for_backend.sh ${{ inputs.port }} 0 $LOG_FILE; then backend_died=\"\$test_file\" + skipped_tests=(\"\${test_files[@]:\$((test_index + 1))}\") break fi fi done && \ if [ -n \"\$backend_died\" ]; then - echo \"carta_backend stopped listening during \$backend_died; abandoning the rest of the stage\" + echo \"carta_backend stopped listening during \$backend_died\" + remaining_tests=() + for skipped_test in \"\${skipped_tests[@]}\"; do + if [ -n \"\$skipped_test\" ]; then + remaining_tests+=(\"\$skipped_test\") + fi + done + if [ \${#remaining_tests[@]} -ne 0 ]; then + echo \"The following tests did not run:\" + printf ' %s\n' \"\${remaining_tests[@]}\" + fi fi && \ if [ \${#failed_tests[@]} -ne 0 ]; then echo \"The following tests failed:\" && \ diff --git a/.github/actions/run-macos/action.yml b/.github/actions/run-macos/action.yml index df35f8fb..73a0fa1f 100644 --- a/.github/actions/run-macos/action.yml +++ b/.github/actions/run-macos/action.yml @@ -40,8 +40,10 @@ runs: cd $ICD_DIR failed_tests=() backend_died='' + skipped_tests=() mapfile -t test_files < ICD_test_stages/${{ inputs.test_stage_name }}.tests - for test_file in "${test_files[@]}"; do + for test_index in "${!test_files[@]}"; do + test_file="${test_files[$test_index]}" if [ -n "$test_file" ]; then if ! CI=true npm test -- "$test_file"; then failed_tests+=("$test_file") @@ -51,12 +53,23 @@ runs: # reporting a connection failure instead of the crash which actually caused it. if ! bash scripts/wait_for_backend.sh ${{ inputs.port }} 0 "$CARTA_LOG_FILE"; then backend_died="$test_file" + skipped_tests=("${test_files[@]:$((test_index + 1))}") break fi fi done if [ -n "$backend_died" ]; then - echo "carta_backend stopped listening during $backend_died; abandoning the rest of the stage" + echo "carta_backend stopped listening during $backend_died" + remaining_tests=() + for skipped_test in "${skipped_tests[@]}"; do + if [ -n "$skipped_test" ]; then + remaining_tests+=("$skipped_test") + fi + done + if [ ${#remaining_tests[@]} -ne 0 ]; then + echo "The following tests did not run:" + printf ' %s\n' "${remaining_tests[@]}" + fi fi if [ ${#failed_tests[@]} -ne 0 ]; then echo "The following tests failed:" From e354ea69b68a6a9cf7ae76243a17258579b1d3f6 Mon Sep 17 00:00:00 2001 From: Cheng-Chin Chiang Date: Fri, 21 Aug 2026 15:28:42 +0800 Subject: [PATCH 04/15] Edit the crashed backend message --- .github/actions/run-apptainer/action.yml | 2 +- .github/actions/run-macos/action.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/run-apptainer/action.yml b/.github/actions/run-apptainer/action.yml index b4d27c09..fc60dde1 100644 --- a/.github/actions/run-apptainer/action.yml +++ b/.github/actions/run-apptainer/action.yml @@ -83,7 +83,7 @@ runs: fi done && \ if [ -n \"\$backend_died\" ]; then - echo \"carta_backend stopped listening during \$backend_died\" + echo \"carta_backend stopped listening during \$backend_died; abandoning the rest of the stage\" remaining_tests=() for skipped_test in \"\${skipped_tests[@]}\"; do if [ -n \"\$skipped_test\" ]; then diff --git a/.github/actions/run-macos/action.yml b/.github/actions/run-macos/action.yml index 73a0fa1f..18aa1b5f 100644 --- a/.github/actions/run-macos/action.yml +++ b/.github/actions/run-macos/action.yml @@ -59,7 +59,7 @@ runs: fi done if [ -n "$backend_died" ]; then - echo "carta_backend stopped listening during $backend_died" + echo "carta_backend stopped listening during $backend_died; abandoning the rest of the stage" remaining_tests=() for skipped_test in "${skipped_tests[@]}"; do if [ -n "$skipped_test" ]; then From cc4dd4b34ff35abca81dbbd2bf0f1d7818340535 Mon Sep 17 00:00:00 2001 From: Cheng-Chin Chiang Date: Fri, 21 Aug 2026 16:21:11 +0800 Subject: [PATCH 05/15] Restart a crashed backend instead of abandoning the stage --- .github/actions/run-apptainer/action.yml | 64 +++++++++++----------- .github/actions/run-macos/action.yml | 70 +++++++++++++----------- scripts/start_backend.sh | 60 ++++++++++++++++++++ 3 files changed, 130 insertions(+), 64 deletions(-) create mode 100755 scripts/start_backend.sh diff --git a/.github/actions/run-apptainer/action.yml b/.github/actions/run-apptainer/action.yml index fc60dde1..248eed2f 100644 --- a/.github/actions/run-apptainer/action.yml +++ b/.github/actions/run-apptainer/action.yml @@ -43,29 +43,16 @@ runs: --bind /images:/images \ --pwd $BUILD_DIR \ ${{ inputs.image }} /bin/bash -c "\ - # Start the carta_backend - ASAN_OPTIONS=suppressions=$SRC_DIR/debug/asan/myasan.supp \ - LSAN_OPTIONS=suppressions=$SRC_DIR/debug/asan/myasan-leaks.supp \ - ASAN_SYMBOLIZER_PATH=llvm-symbolizer \ - ./carta_backend /images \ - --top_level_folder /images \ - --port ${{ inputs.port }} \ - --omp_threads=4 \ - --debug_no_auth \ - --no_frontend \ - --no_database \ - --no_log \ - --verbosity=5 >> $LOG_FILE 2>&1 & \ - # pgrep matches as soon as the backend forks, which is before it binds the port - bash $BUILD_DIR/ICD-RxJS/scripts/wait_for_backend.sh ${{ inputs.port }} 120 $LOG_FILE && \ - echo 'carta_backend is listening with PID' \$(pgrep -f 'carta_backend.*${{ inputs.port }}' | head -n 1) && \ + bash $BUILD_DIR/ICD-RxJS/scripts/start_backend.sh $SRC_DIR $BUILD_DIR ${{ inputs.port }} $LOG_FILE && \ # Run the ICD tests cd $BUILD_DIR/ICD-RxJS && \ pwd && \ cat $TEST_STAGE && \ failed_tests=() && \ - backend_died='' && \ + crash_sites=() && \ skipped_tests=() && \ + restarts=0 && \ + max_restarts=3 && \ mapfile -t test_files < $TEST_STAGE && \ for test_index in \"\${!test_files[@]}\"; do test_file=\"\${test_files[\$test_index]}\" @@ -75,31 +62,44 @@ runs: fi # A backend which has died takes every remaining file in the stage with it, each # reporting a connection failure instead of the crash which actually caused it. + # The tests hold no state in the backend -- every file connects and loads for + # itself -- so a fresh backend lets the rest of the stage run for real. if ! bash scripts/wait_for_backend.sh ${{ inputs.port }} 0 $LOG_FILE; then - backend_died=\"\$test_file\" - skipped_tests=(\"\${test_files[@]:\$((test_index + 1))}\") - break + crash_sites+=(\"\$test_file\") + if [ \"\$restarts\" -ge \"\$max_restarts\" ]; then + echo \"carta_backend crashed after \$test_file and has already been restarted \$restarts times; abandoning the rest of the stage\" + skipped_tests=(\"\${test_files[@]:\$((test_index + 1))}\") + break + fi + restarts=\$((restarts + 1)) + echo \"carta_backend crashed after \$test_file; restarting (\$restarts of \$max_restarts)\" + if ! bash scripts/start_backend.sh $SRC_DIR $BUILD_DIR ${{ inputs.port }} $LOG_FILE; then + echo \"carta_backend could not be restarted; abandoning the rest of the stage\" + skipped_tests=(\"\${test_files[@]:\$((test_index + 1))}\") + break + fi fi fi done && \ - if [ -n \"\$backend_died\" ]; then - echo \"carta_backend stopped listening during \$backend_died; abandoning the rest of the stage\" - remaining_tests=() - for skipped_test in \"\${skipped_tests[@]}\"; do - if [ -n \"\$skipped_test\" ]; then - remaining_tests+=(\"\$skipped_test\") - fi - done - if [ \${#remaining_tests[@]} -ne 0 ]; then - echo \"The following tests did not run:\" - printf ' %s\n' \"\${remaining_tests[@]}\" + if [ \${#crash_sites[@]} -ne 0 ]; then + echo \"carta_backend crashed after:\" + printf ' %s\n' \"\${crash_sites[@]}\" + fi && \ + remaining_tests=() && \ + for skipped_test in \"\${skipped_tests[@]}\"; do + if [ -n \"\$skipped_test\" ]; then + remaining_tests+=(\"\$skipped_test\") fi + done && \ + if [ \${#remaining_tests[@]} -ne 0 ]; then + echo \"The following tests did not run:\" + printf ' %s\n' \"\${remaining_tests[@]}\" fi && \ if [ \${#failed_tests[@]} -ne 0 ]; then echo \"The following tests failed:\" && \ printf '%s\n' \"\${failed_tests[@]}\" fi && \ - if [ \${#failed_tests[@]} -ne 0 ] || [ -n \"\$backend_died\" ]; then + if [ \${#failed_tests[@]} -ne 0 ] || [ \${#crash_sites[@]} -ne 0 ]; then exit 1 fi" shell: bash diff --git a/.github/actions/run-macos/action.yml b/.github/actions/run-macos/action.yml index 18aa1b5f..b78f5fc2 100644 --- a/.github/actions/run-macos/action.yml +++ b/.github/actions/run-macos/action.yml @@ -17,21 +17,10 @@ runs: - name: Start the carta-backend run: | - SRC_DIR=$GITHUB_WORKSPACE/source - BUILD_DIR=$GITHUB_WORKSPACE/build LOG_FILE="/tmp/carta_icd_macos_${{ inputs.test_stage_name }}.log" - cd $BUILD_DIR - ASAN_OPTIONS=suppressions=$SRC_DIR/debug/asan/myasan.supp \ - LSAN_OPTIONS=suppressions=$SRC_DIR/debug/asan/myasan-leaks.supp \ - ASAN_SYMBOLIZER_PATH=llvm-symbolizer \ - ./carta_backend /images --top_level_folder /images \ - --port ${{ inputs.port }} \ - --omp_threads=4 --debug_no_auth --no_frontend --no_database --no_log \ - --verbosity=5 >> "$LOG_FILE" 2>&1 & - echo "CARTA_BACKEND_PID=$!" >> $GITHUB_ENV echo "CARTA_LOG_FILE=$LOG_FILE" >> $GITHUB_ENV - # A PID only says the backend forked; the tests need the port to be bound. - bash $GITHUB_WORKSPACE/ICD-RxJS/scripts/wait_for_backend.sh ${{ inputs.port }} 120 "$LOG_FILE" + bash $GITHUB_WORKSPACE/ICD-RxJS/scripts/start_backend.sh \ + $GITHUB_WORKSPACE/source $GITHUB_WORKSPACE/build ${{ inputs.port }} "$LOG_FILE" shell: bash - name: ICD tests @@ -39,8 +28,10 @@ runs: ICD_DIR=$GITHUB_WORKSPACE/ICD-RxJS cd $ICD_DIR failed_tests=() - backend_died='' + crash_sites=() skipped_tests=() + restarts=0 + max_restarts=3 mapfile -t test_files < ICD_test_stages/${{ inputs.test_stage_name }}.tests for test_index in "${!test_files[@]}"; do test_file="${test_files[$test_index]}" @@ -51,31 +42,45 @@ runs: sleep 3 # A backend which has died takes every remaining file in the stage with it, each # reporting a connection failure instead of the crash which actually caused it. + # The tests hold no state in the backend -- every file connects and loads for + # itself -- so a fresh backend lets the rest of the stage run for real. if ! bash scripts/wait_for_backend.sh ${{ inputs.port }} 0 "$CARTA_LOG_FILE"; then - backend_died="$test_file" - skipped_tests=("${test_files[@]:$((test_index + 1))}") - break + crash_sites+=("$test_file") + if [ "$restarts" -ge "$max_restarts" ]; then + echo "carta_backend crashed after $test_file and has already been restarted $restarts times; abandoning the rest of the stage" + skipped_tests=("${test_files[@]:$((test_index + 1))}") + break + fi + restarts=$((restarts + 1)) + echo "carta_backend crashed after $test_file; restarting ($restarts of $max_restarts)" + if ! bash scripts/start_backend.sh \ + $GITHUB_WORKSPACE/source $GITHUB_WORKSPACE/build ${{ inputs.port }} "$CARTA_LOG_FILE"; then + echo "carta_backend could not be restarted; abandoning the rest of the stage" + skipped_tests=("${test_files[@]:$((test_index + 1))}") + break + fi fi fi done - if [ -n "$backend_died" ]; then - echo "carta_backend stopped listening during $backend_died; abandoning the rest of the stage" - remaining_tests=() - for skipped_test in "${skipped_tests[@]}"; do - if [ -n "$skipped_test" ]; then - remaining_tests+=("$skipped_test") - fi - done - if [ ${#remaining_tests[@]} -ne 0 ]; then - echo "The following tests did not run:" - printf ' %s\n' "${remaining_tests[@]}" + if [ ${#crash_sites[@]} -ne 0 ]; then + echo "carta_backend crashed after:" + printf ' %s\n' "${crash_sites[@]}" + fi + remaining_tests=() + for skipped_test in "${skipped_tests[@]}"; do + if [ -n "$skipped_test" ]; then + remaining_tests+=("$skipped_test") fi + done + if [ ${#remaining_tests[@]} -ne 0 ]; then + echo "The following tests did not run:" + printf ' %s\n' "${remaining_tests[@]}" fi if [ ${#failed_tests[@]} -ne 0 ]; then echo "The following tests failed:" printf '%s\n' "${failed_tests[@]}" fi - if [ ${#failed_tests[@]} -ne 0 ] || [ -n "$backend_died" ]; then + if [ ${#failed_tests[@]} -ne 0 ] || [ ${#crash_sites[@]} -ne 0 ]; then exit 1 fi shell: bash @@ -91,7 +96,8 @@ runs: - name: Stop carta-backend if: always() run: | - if [ -n "${{ env.CARTA_BACKEND_PID }}" ]; then - kill ${{ env.CARTA_BACKEND_PID }} || true - fi + # Matched by port rather than by a PID recorded at startup: the stage may have restarted + # the backend after a crash, in which case that PID is long gone and killing it would + # leave the replacement running on the runner. + pkill -f "carta_backend.*${{ inputs.port }}" || true shell: bash diff --git a/scripts/start_backend.sh b/scripts/start_backend.sh new file mode 100755 index 00000000..0d277f76 --- /dev/null +++ b/scripts/start_backend.sh @@ -0,0 +1,60 @@ +#!/bin/bash +# +# Start carta_backend and wait until it is accepting connections. +# +# Both CI actions used to carry their own copy of this command line, which made it easy for the +# two to drift and impossible to bring the backend back up from inside a test loop. With one +# script a stage can restart a crashed backend using exactly the invocation it was started with, +# so the files after the crash still run against a real backend instead of all reporting the same +# connection failure. +# +# Usage: start_backend.sh SRC_DIR BUILD_DIR PORT LOG_FILE [TIMEOUT_SECONDS] +# +# The backend's output is appended to LOG_FILE, so a restart leaves the previous crash report in +# place rather than overwriting the very thing we started the backend again to explain. Exits +# non-zero, with the tail of the log, if the backend does not start listening within +# TIMEOUT_SECONDS (default 120). + +set -u + +src_dir=${1:-} +build_dir=${2:-} +port=${3:-} +log_file=${4:-} +timeout=${5:-120} + +if [ -z "$src_dir" ] || [ -z "$build_dir" ] || [ -z "$port" ] || [ -z "$log_file" ]; then + echo "usage: $0 SRC_DIR BUILD_DIR PORT LOG_FILE [TIMEOUT_SECONDS]" >&2 + exit 2 +fi + +# Resolved before the cd below, so that wait_for_backend.sh is found next to this script whatever +# directory the caller happens to be in. +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) + +if ! cd "$build_dir"; then + echo "no such build directory: $build_dir" >&2 + exit 1 +fi + +ASAN_OPTIONS=suppressions=$src_dir/debug/asan/myasan.supp \ +LSAN_OPTIONS=suppressions=$src_dir/debug/asan/myasan-leaks.supp \ +ASAN_SYMBOLIZER_PATH=llvm-symbolizer \ +./carta_backend /images \ + --top_level_folder /images \ + --port "$port" \ + --omp_threads=4 \ + --debug_no_auth \ + --no_frontend \ + --no_database \ + --no_log \ + --verbosity=5 >> "$log_file" 2>&1 & + +backend_pid=$! + +# A PID only says the backend forked; the tests need the port to be bound. +if ! bash "$script_dir/wait_for_backend.sh" "$port" "$timeout" "$log_file"; then + exit 1 +fi + +echo "carta_backend is listening on port $port with PID $backend_pid" From 2c81f86299e31df00480bb5a1d0c6a2a48a86387 Mon Sep 17 00:00:00 2001 From: Cheng-Chin Chiang Date: Fri, 21 Aug 2026 16:39:14 +0800 Subject: [PATCH 06/15] Clear the port before starting the backend --- scripts/start_backend.sh | 128 +++++++++++++++++++++++++++++++++------ 1 file changed, 109 insertions(+), 19 deletions(-) diff --git a/scripts/start_backend.sh b/scripts/start_backend.sh index 0d277f76..f5090904 100755 --- a/scripts/start_backend.sh +++ b/scripts/start_backend.sh @@ -14,9 +14,18 @@ # place rather than overwriting the very thing we started the backend again to explain. Exits # non-zero, with the tail of the log, if the backend does not start listening within # TIMEOUT_SECONDS (default 120). +# +# The port is cleared before launching. A backend which has crashed can still hold the bind while +# it is being torn down, and a replacement which cannot bind would sit out the whole start-up +# timeout and then be reported as a backend that failed to start, hiding the real reason. set -u +# How long to wait for a dying backend to let go of the port before giving up on it. +PORT_FREE_TIMEOUT=30 +# How long to wait after SIGTERM before resorting to SIGKILL. +PORT_FREE_GRACE=5 + src_dir=${1:-} build_dir=${2:-} port=${3:-} @@ -32,29 +41,110 @@ fi # directory the caller happens to be in. script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# pgrep is not guaranteed to be in a container image, so fall back to plain ps. +backend_pids() { + if command -v pgrep >/dev/null 2>&1; then + pgrep -f "carta_backend.*$port" + else + ps -eo pid=,args= 2>/dev/null | awk -v pat="carta_backend.*$port" '$0 ~ pat {print $1}' + fi +} + +port_is_busy() { + (exec 3<>"/dev/tcp/127.0.0.1/$port") 2>/dev/null +} + +clear_port() { + local pids waited=0 signalled='' + + pids=$(backend_pids) + if [ -n "$pids" ]; then + echo "stopping carta_backend still holding port $port (PID $(echo $pids | tr '\n' ' '))" + kill $pids 2>/dev/null + signalled=yes + fi + + while [ "$waited" -lt "$PORT_FREE_TIMEOUT" ]; do + if ! port_is_busy; then + return 0 + fi + # Something is still listening. Give a signalled backend a few seconds to exit on its own + # terms -- it may be writing the sanitizer report we want -- and only then force it. + if [ -n "$signalled" ] && [ "$waited" -eq "$PORT_FREE_GRACE" ]; then + pids=$(backend_pids) + if [ -n "$pids" ]; then + kill -9 $pids 2>/dev/null + fi + fi + sleep 1 + waited=$((waited + 1)) + done + + return 1 +} + +if ! clear_port; then + echo "port $port is still in use after ${PORT_FREE_TIMEOUT}s; not starting carta_backend" >&2 + exit 1 +fi + if ! cd "$build_dir"; then echo "no such build directory: $build_dir" >&2 exit 1 fi -ASAN_OPTIONS=suppressions=$src_dir/debug/asan/myasan.supp \ -LSAN_OPTIONS=suppressions=$src_dir/debug/asan/myasan-leaks.supp \ -ASAN_SYMBOLIZER_PATH=llvm-symbolizer \ -./carta_backend /images \ - --top_level_folder /images \ - --port "$port" \ - --omp_threads=4 \ - --debug_no_auth \ - --no_frontend \ - --no_database \ - --no_log \ - --verbosity=5 >> "$log_file" 2>&1 & - -backend_pid=$! - -# A PID only says the backend forked; the tests need the port to be bound. -if ! bash "$script_dir/wait_for_backend.sh" "$port" "$timeout" "$log_file"; then - exit 1 +launch_backend() { + ASAN_OPTIONS=suppressions=$src_dir/debug/asan/myasan.supp \ + LSAN_OPTIONS=suppressions=$src_dir/debug/asan/myasan-leaks.supp \ + ASAN_SYMBOLIZER_PATH=llvm-symbolizer \ + ./carta_backend /images \ + --top_level_folder /images \ + --port "$port" \ + --omp_threads=4 \ + --debug_no_auth \ + --no_frontend \ + --no_database \ + --no_log \ + --verbosity=5 >> "$log_file" 2>&1 & + + backend_pid=$! +} + +max_attempts=3 +attempt=1 +attempt_timeout=$((timeout / max_attempts)) +if [ "$attempt_timeout" -lt 10 ]; then + max_attempts=1 + attempt_timeout=$timeout fi -echo "carta_backend is listening on port $port with PID $backend_pid" +while :; do + launch_backend + + # A PID only says the backend forked; the tests need the port to be bound. The log is left out + # here so that a retried attempt does not tail it once per try. + if bash "$script_dir/wait_for_backend.sh" "$port" "$attempt_timeout"; then + echo "carta_backend is listening on port $port with PID $backend_pid" + exit 0 + fi + + # Still running, just not listening yet: it is slow or wedged, and launching a second copy + # would only fight the first one for the port. + if kill -0 "$backend_pid" 2>/dev/null; then + break + fi + + # Gone already, so it never got as far as listening. The usual reason after a crash is that it + # could not bind: connections from the previous backend can sit in TIME_WAIT for a while after + # it has stopped accepting, which is invisible to a connect-based check. That clears by itself, + # so try again instead of spending the rest of the budget waiting on a process which has exited. + if [ "$attempt" -ge "$max_attempts" ]; then + break + fi + attempt=$((attempt + 1)) + echo "carta_backend exited before it started listening; retrying ($attempt of $max_attempts)" +done + +# Report the failure the same way every other check does, with the tail of the log attached. +bash "$script_dir/wait_for_backend.sh" "$port" 0 "$log_file" +exit 1 From dcafbc184a8e6f3abe5502c93e0f5dad285c179d Mon Sep 17 00:00:00 2001 From: Cheng-Chin Chiang Date: Fri, 21 Aug 2026 17:32:08 +0800 Subject: [PATCH 07/15] Drop the unexplained sleep between test files on macOS --- .github/actions/run-macos/action.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/actions/run-macos/action.yml b/.github/actions/run-macos/action.yml index b78f5fc2..9acee256 100644 --- a/.github/actions/run-macos/action.yml +++ b/.github/actions/run-macos/action.yml @@ -39,7 +39,6 @@ runs: if ! CI=true npm test -- "$test_file"; then failed_tests+=("$test_file") fi - sleep 3 # A backend which has died takes every remaining file in the stage with it, each # reporting a connection failure instead of the crash which actually caused it. # The tests hold no state in the backend -- every file connects and loads for From 6fc5c889535afdb63d04458d924873525ff23d4f Mon Sep 17 00:00:00 2001 From: Cheng-Chin Chiang Date: Fri, 21 Aug 2026 20:15:42 +0800 Subject: [PATCH 08/15] Load the ICD test actions from this repository --- .github/workflows/all_stages_icd_tests.yml | 56 ++++++++++---------- .github/workflows/icd_tests.yml | 56 ++++++++++---------- .github/workflows/single_stage_icd_tests.yml | 56 ++++++++++---------- 3 files changed, 84 insertions(+), 84 deletions(-) diff --git a/.github/workflows/all_stages_icd_tests.yml b/.github/workflows/all_stages_icd_tests.yml index fb1e8e81..de2180f3 100644 --- a/.github/workflows/all_stages_icd_tests.yml +++ b/.github/workflows/all_stages_icd_tests.yml @@ -270,13 +270,13 @@ jobs: # macOS steps - name: File Browser ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: test_stage_name: 'file_browser' # Linux steps - name: File Browser ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -325,13 +325,13 @@ jobs: # macOS steps - name: Animator ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: test_stage_name: 'animator' # Linux steps - name: Animator ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -380,13 +380,13 @@ jobs: # macOS steps - name: Region-Statistics ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: test_stage_name: 'region_statistics' # Linux steps - name: Region-Statistics ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -435,13 +435,13 @@ jobs: # macOS steps - name: Region Manipulation ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: test_stage_name: 'region_manipulation' # Linux steps - name: Region Manipulation ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -490,13 +490,13 @@ jobs: # macOS steps - name: Cube Histogram ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: test_stage_name: 'cube_histogram' # Linux steps - name: Cube Histogram ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -545,13 +545,13 @@ jobs: # macOS steps - name: PV Generator ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: test_stage_name: 'pv_generator' # Linux steps - name: PV Generator ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -600,13 +600,13 @@ jobs: # macOS steps - name: Raster Tiles ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: test_stage_name: 'raster_tiles' # Linux steps - name: Raster Tiles ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -655,13 +655,13 @@ jobs: # macOS steps - name: Catalog ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: test_stage_name: 'catalog' # Linux steps - name: Catalog ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -710,13 +710,13 @@ jobs: # macOS steps - name: Moment ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: test_stage_name: 'moment' # Linux steps - name: Moment ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -765,13 +765,13 @@ jobs: # macOS steps - name: Match ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: test_stage_name: 'match' # Linux steps - name: Match ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -820,13 +820,13 @@ jobs: # macOS steps - name: Close File ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: test_stage_name: 'close_file' # Linux steps - name: Close File ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -875,13 +875,13 @@ jobs: # macOS steps - name: Image Fitting ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: test_stage_name: 'image_fitting' # Linux steps - name: Image Fitting ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -930,13 +930,13 @@ jobs: # macOS steps - name: Vector Overlay ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: test_stage_name: 'vector_overlay' # Linux steps - name: Vector Overlay ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -985,13 +985,13 @@ jobs: # macOS steps - name: Resume ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: test_stage_name: 'resume' # Linux steps - name: Resume ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} diff --git a/.github/workflows/icd_tests.yml b/.github/workflows/icd_tests.yml index 76d4f207..5ec44446 100644 --- a/.github/workflows/icd_tests.yml +++ b/.github/workflows/icd_tests.yml @@ -300,13 +300,13 @@ jobs: # macOS steps - name: File Browser ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: test_stage_name: 'file_browser' # Linux steps - name: File Browser ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -356,13 +356,13 @@ jobs: # macOS steps - name: Animator ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: test_stage_name: 'animator' # Linux steps - name: Animator ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -412,13 +412,13 @@ jobs: # macOS steps - name: Region-Statistics ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: test_stage_name: 'region_statistics' # Linux steps - name: Region-Statistics ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -468,13 +468,13 @@ jobs: # macOS steps - name: Region Manipulation ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: test_stage_name: 'region_manipulation' # Linux steps - name: Region Manipulation ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -524,13 +524,13 @@ jobs: # macOS steps - name: Cube Histogram ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: test_stage_name: 'cube_histogram' # Linux steps - name: Cube Histogram ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -580,13 +580,13 @@ jobs: # macOS steps - name: PV Generator ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: test_stage_name: 'pv_generator' # Linux steps - name: PV Generator ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -636,13 +636,13 @@ jobs: # macOS steps - name: Raster Tiles ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: test_stage_name: 'raster_tiles' # Linux steps - name: Raster Tiles ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -692,13 +692,13 @@ jobs: # macOS steps - name: Catalog ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: test_stage_name: 'catalog' # Linux steps - name: Catalog ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -748,13 +748,13 @@ jobs: # macOS steps - name: Moment ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: test_stage_name: 'moment' # Linux steps - name: Moment ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -804,13 +804,13 @@ jobs: # macOS steps - name: Match ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: test_stage_name: 'match' # Linux steps - name: Match ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -860,13 +860,13 @@ jobs: # macOS steps - name: Close File ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: test_stage_name: 'close_file' # Linux steps - name: Close File ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -916,13 +916,13 @@ jobs: # macOS steps - name: Image Fitting ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: test_stage_name: 'image_fitting' # Linux steps - name: Image Fitting ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -972,13 +972,13 @@ jobs: # macOS steps - name: Vector Overlay ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: test_stage_name: 'vector_overlay' # Linux steps - name: Vector Overlay ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -1028,13 +1028,13 @@ jobs: # macOS steps - name: Resume ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: test_stage_name: 'resume' # Linux steps - name: Resume ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} diff --git a/.github/workflows/single_stage_icd_tests.yml b/.github/workflows/single_stage_icd_tests.yml index c71883c2..f94576bb 100644 --- a/.github/workflows/single_stage_icd_tests.yml +++ b/.github/workflows/single_stage_icd_tests.yml @@ -290,13 +290,13 @@ jobs: # macOS steps - name: File Browser ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: test_stage_name: 'file_browser' # Linux steps - name: File Browser ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -346,13 +346,13 @@ jobs: # macOS steps - name: Animator ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: test_stage_name: 'animator' # Linux steps - name: Animator ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -402,13 +402,13 @@ jobs: # macOS steps - name: Region-Statistics ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: test_stage_name: 'region_statistics' # Linux steps - name: Region-Statistics ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -458,13 +458,13 @@ jobs: # macOS steps - name: Region Manipulation ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: test_stage_name: 'region_manipulation' # Linux steps - name: Region Manipulation ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -514,13 +514,13 @@ jobs: # macOS steps - name: Cube Histogram ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: test_stage_name: 'cube_histogram' # Linux steps - name: Cube Histogram ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -570,13 +570,13 @@ jobs: # macOS steps - name: PV Generator ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: test_stage_name: 'pv_generator' # Linux steps - name: PV Generator ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -626,13 +626,13 @@ jobs: # macOS steps - name: Raster Tiles ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: test_stage_name: 'raster_tiles' # Linux steps - name: Raster Tiles ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -682,13 +682,13 @@ jobs: # macOS steps - name: Catalog ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: test_stage_name: 'catalog' # Linux steps - name: Catalog ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -738,13 +738,13 @@ jobs: # macOS steps - name: Moment ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: test_stage_name: 'moment' # Linux steps - name: Moment ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -794,13 +794,13 @@ jobs: # macOS steps - name: Match ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: test_stage_name: 'match' # Linux steps - name: Match ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -850,13 +850,13 @@ jobs: # macOS steps - name: Close File ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: test_stage_name: 'close_file' # Linux steps - name: Close File ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -906,13 +906,13 @@ jobs: # macOS steps - name: Image Fitting ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: test_stage_name: 'image_fitting' # Linux steps - name: Image Fitting ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -962,13 +962,13 @@ jobs: # macOS steps - name: Vector Overlay ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: test_stage_name: 'vector_overlay' # Linux steps - name: Vector Overlay ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -1018,13 +1018,13 @@ jobs: # macOS steps - name: Resume ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: test_stage_name: 'resume' # Linux steps - name: Resume ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} From 06a0fcd7ba93741300421e59c89b9dca9b788cba Mon Sep 17 00:00:00 2001 From: Cheng-Chin Chiang Date: Fri, 21 Aug 2026 20:59:07 +0800 Subject: [PATCH 09/15] Read the stage file without mapfile for bash 3.2 --- .github/actions/run-macos/action.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/actions/run-macos/action.yml b/.github/actions/run-macos/action.yml index 9acee256..8108ce30 100644 --- a/.github/actions/run-macos/action.yml +++ b/.github/actions/run-macos/action.yml @@ -32,7 +32,12 @@ runs: skipped_tests=() restarts=0 max_restarts=3 - mapfile -t test_files < ICD_test_stages/${{ inputs.test_stage_name }}.tests + # Read with a loop rather than mapfile: macOS ships bash 3.2, which does not have it. The + # `|| [ -n "$line" ]` keeps the last entry of a file with no trailing newline. + test_files=() + while IFS= read -r line || [ -n "$line" ]; do + test_files+=("$line") + done < ICD_test_stages/${{ inputs.test_stage_name }}.tests for test_index in "${!test_files[@]}"; do test_file="${test_files[$test_index]}" if [ -n "$test_file" ]; then From 9995c2658e830fdf80b553fb3c3c6f5bf6343b77 Mon Sep 17 00:00:00 2001 From: Cheng-Chin Chiang Date: Fri, 21 Aug 2026 21:13:13 +0800 Subject: [PATCH 10/15] Give a restart a budget the job cap can absorb --- .github/actions/run-apptainer/action.yml | 7 ++++++- .github/actions/run-macos/action.yml | 6 +++++- .github/workflows/all_stages_icd_tests.yml | 22 ++++++++++---------- .github/workflows/icd_tests.yml | 22 ++++++++++---------- .github/workflows/single_stage_icd_tests.yml | 22 ++++++++++---------- 5 files changed, 44 insertions(+), 35 deletions(-) diff --git a/.github/actions/run-apptainer/action.yml b/.github/actions/run-apptainer/action.yml index 248eed2f..e6273f3c 100644 --- a/.github/actions/run-apptainer/action.yml +++ b/.github/actions/run-apptainer/action.yml @@ -73,7 +73,12 @@ runs: fi restarts=\$((restarts + 1)) echo \"carta_backend crashed after \$test_file; restarting (\$restarts of \$max_restarts)\" - if ! bash scripts/start_backend.sh $SRC_DIR $BUILD_DIR ${{ inputs.port }} $LOG_FILE; then + # A restart gets a shorter budget than the 120s cold start. The binary is warm by + # now, and the job itself is capped: three restarts at the full budget would spend + # the cap on waiting and get the stage cancelled before it could report anything, + # which is exactly how this showed up -- an exit 143 in the middle of a restart. + # 90s still leaves 30s an attempt, which is what the old harness allowed a restart. + if ! bash scripts/start_backend.sh $SRC_DIR $BUILD_DIR ${{ inputs.port }} $LOG_FILE 90; then echo \"carta_backend could not be restarted; abandoning the rest of the stage\" skipped_tests=(\"\${test_files[@]:\$((test_index + 1))}\") break diff --git a/.github/actions/run-macos/action.yml b/.github/actions/run-macos/action.yml index 8108ce30..94a37d1b 100644 --- a/.github/actions/run-macos/action.yml +++ b/.github/actions/run-macos/action.yml @@ -57,8 +57,12 @@ runs: fi restarts=$((restarts + 1)) echo "carta_backend crashed after $test_file; restarting ($restarts of $max_restarts)" + # A restart gets a shorter budget than the 120s cold start. The binary is warm by now, + # and the job itself is capped: three restarts at the full budget would spend the cap + # on waiting and get the stage cancelled before it could report anything. 90s still + # leaves 30s an attempt, which is what the old harness allowed a restart. if ! bash scripts/start_backend.sh \ - $GITHUB_WORKSPACE/source $GITHUB_WORKSPACE/build ${{ inputs.port }} "$CARTA_LOG_FILE"; then + $GITHUB_WORKSPACE/source $GITHUB_WORKSPACE/build ${{ inputs.port }} "$CARTA_LOG_FILE" 90; then echo "carta_backend could not be restarted; abandoning the rest of the stage" skipped_tests=("${test_files[@]:$((test_index + 1))}") break diff --git a/.github/workflows/all_stages_icd_tests.yml b/.github/workflows/all_stages_icd_tests.yml index de2180f3..3d65c8a1 100644 --- a/.github/workflows/all_stages_icd_tests.yml +++ b/.github/workflows/all_stages_icd_tests.yml @@ -231,7 +231,7 @@ jobs: File-Browser-ICD-Tests: name: File-Browser ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -286,7 +286,7 @@ jobs: Animator-ICD-Tests: name: Animator ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -341,7 +341,7 @@ jobs: Region-Statistics-ICD-Tests: name: Region Statistics ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -396,7 +396,7 @@ jobs: Region-Manipulation-ICD-Tests: name: Region Manipulation ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -561,7 +561,7 @@ jobs: Raster-Tiles-ICD-Tests: name: Raster Tiles ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -616,7 +616,7 @@ jobs: Catalog-ICD-Tests: name: Catalog ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -671,7 +671,7 @@ jobs: Moment-ICD-Tests: name: Moment ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -726,7 +726,7 @@ jobs: Match-ICD-Tests: name: Match ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -781,7 +781,7 @@ jobs: Close-File-ICD-Tests: name: Close File ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -891,7 +891,7 @@ jobs: Vector-Overlay-ICD-Tests: name: Vector Overlay ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -946,7 +946,7 @@ jobs: Resume-ICD-Tests: name: Resume ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: diff --git a/.github/workflows/icd_tests.yml b/.github/workflows/icd_tests.yml index 5ec44446..c443a06f 100644 --- a/.github/workflows/icd_tests.yml +++ b/.github/workflows/icd_tests.yml @@ -260,7 +260,7 @@ jobs: File-Browser-ICD-Tests: name: File-Browser ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -316,7 +316,7 @@ jobs: Animator-ICD-Tests: name: Animator ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -372,7 +372,7 @@ jobs: Region-Statistics-ICD-Tests: name: Region Statistics ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -428,7 +428,7 @@ jobs: Region-Manipulation-ICD-Tests: name: Region Manipulation ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -596,7 +596,7 @@ jobs: Raster-Tiles-ICD-Tests: name: Raster Tiles ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -652,7 +652,7 @@ jobs: Catalog-ICD-Tests: name: Catalog ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -708,7 +708,7 @@ jobs: Moment-ICD-Tests: name: Moment ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -764,7 +764,7 @@ jobs: Match-ICD-Tests: name: Match ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -820,7 +820,7 @@ jobs: Close-File-ICD-Tests: name: Close File ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -932,7 +932,7 @@ jobs: Vector-Overlay-ICD-Tests: name: Vector Overlay ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -988,7 +988,7 @@ jobs: Resume-ICD-Tests: name: Resume ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: diff --git a/.github/workflows/single_stage_icd_tests.yml b/.github/workflows/single_stage_icd_tests.yml index f94576bb..0f2883b1 100644 --- a/.github/workflows/single_stage_icd_tests.yml +++ b/.github/workflows/single_stage_icd_tests.yml @@ -250,7 +250,7 @@ jobs: File-Browser-ICD-Tests: name: File-Browser ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -306,7 +306,7 @@ jobs: Animator-ICD-Tests: name: Animator ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -362,7 +362,7 @@ jobs: Region-Statistics-ICD-Tests: name: Region Statistics ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -418,7 +418,7 @@ jobs: Region-Manipulation-ICD-Tests: name: Region Manipulation ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -586,7 +586,7 @@ jobs: Raster-Tiles-ICD-Tests: name: Raster Tiles ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -642,7 +642,7 @@ jobs: Catalog-ICD-Tests: name: Catalog ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -698,7 +698,7 @@ jobs: Moment-ICD-Tests: name: Moment ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -754,7 +754,7 @@ jobs: Match-ICD-Tests: name: Match ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -810,7 +810,7 @@ jobs: Close-File-ICD-Tests: name: Close File ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -922,7 +922,7 @@ jobs: Vector-Overlay-ICD-Tests: name: Vector Overlay ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -978,7 +978,7 @@ jobs: Resume-ICD-Tests: name: Resume ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: From ebf997de2cffa941fb8272a02b1c6a8c3da433ea Mon Sep 17 00:00:00 2001 From: Cheng-Chin Chiang Date: Fri, 21 Aug 2026 21:39:45 +0800 Subject: [PATCH 11/15] Match the backend by its port argument, not by any mention of it --- .github/actions/run-apptainer/action.yml | 16 +++++++++----- .github/actions/run-macos/action.yml | 6 ++++-- scripts/start_backend.sh | 27 ++++++++++++++++++++++-- 3 files changed, 40 insertions(+), 9 deletions(-) diff --git a/.github/actions/run-apptainer/action.yml b/.github/actions/run-apptainer/action.yml index e6273f3c..92030596 100644 --- a/.github/actions/run-apptainer/action.yml +++ b/.github/actions/run-apptainer/action.yml @@ -26,12 +26,18 @@ runs: TEST_STAGE="$BUILD_DIR/ICD-RxJS/ICD_test_stages/${{ inputs.test_stage_name }}.tests" LOG_FILE="/tmp/carta_icd_${{ inputs.os_version }}_${{ inputs.test_stage_name }}.log" - # Function to cleanup backend process + # Function to cleanup backend process. + # + # Matched on the executable and its --port argument, not on "carta_backend" and the port + # appearing anywhere in a command line: this scan also sees the host's own + # `apptainer exec ... /bin/bash -c ""`, whose command line mentions + # both, and killing that tears the container down instead of the backend. cleanup() { - local backend_pid=$(pgrep -f "carta_backend.*${{ inputs.port }}" | head -n 1) - if [ ! -z "$backend_pid" ]; then - echo "Cleaning up carta_backend (PID: $backend_pid)" - kill -9 "$backend_pid" || true + local backend_pids + backend_pids=$(pgrep -f "(^|[/[:space:]])carta_backend[[:space:]].*--port[[:space:]=]+${{ inputs.port }}([[:space:]]|$)") + if [ -n "$backend_pids" ]; then + echo "Cleaning up carta_backend (PID: $(echo $backend_pids | tr '\n' ' '))" + kill -9 $backend_pids || true fi } diff --git a/.github/actions/run-macos/action.yml b/.github/actions/run-macos/action.yml index 94a37d1b..9be092ac 100644 --- a/.github/actions/run-macos/action.yml +++ b/.github/actions/run-macos/action.yml @@ -106,6 +106,8 @@ runs: run: | # Matched by port rather than by a PID recorded at startup: the stage may have restarted # the backend after a crash, in which case that PID is long gone and killing it would - # leave the replacement running on the runner. - pkill -f "carta_backend.*${{ inputs.port }}" || true + # leave the replacement running on the runner. The executable and its --port argument are + # both required so that this can only match a backend, never a shell whose command line + # happens to mention carta_backend and the port. + pkill -f "(^|[/[:space:]])carta_backend[[:space:]].*--port[[:space:]=]+${{ inputs.port }}([[:space:]]|$)" || true shell: bash diff --git a/scripts/start_backend.sh b/scripts/start_backend.sh index f5090904..7e2553df 100755 --- a/scripts/start_backend.sh +++ b/scripts/start_backend.sh @@ -41,12 +41,35 @@ fi # directory the caller happens to be in. script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# Matched on the executable and its --port argument rather than on "carta_backend" and the port +# number appearing anywhere in a command line. Apptainer shares the host PID space, so this scan +# also sees `apptainer exec ... /bin/bash -c ""` -- and that script mentions +# carta_backend in one line and the port in another, which `carta_backend.*$port` matches, since a +# POSIX regex crosses newlines. The loose pattern therefore matched the very process running this +# script, and clear_port killed the stage it was restarting the backend for. The stage loop has no +# --port in it, and no shell is named carta_backend, so requiring both makes only a real backend +# match. (macOS never showed this: its pgrep does not match across the newlines.) +backend_pattern="(^|[/[:space:]])carta_backend[[:space:]].*--port[[:space:]=]+$port([[:space:]]|\$)" + # pgrep is not guaranteed to be in a container image, so fall back to plain ps. backend_pids() { if command -v pgrep >/dev/null 2>&1; then - pgrep -f "carta_backend.*$port" + pgrep -f "$backend_pattern" else - ps -eo pid=,args= 2>/dev/null | awk -v pat="carta_backend.*$port" '$0 ~ pat {print $1}' + # By field rather than by a regex over the whole line: the pattern would otherwise appear + # in awk's own command line, and awk would report the process doing the matching. + ps -eo pid=,args= 2>/dev/null | awk -v port="$port" ' + { + argv0 = 0 + for (i = 2; i <= NF; i++) + if ($i ~ /(^|\/)carta_backend$/) { argv0 = i; break } + if (argv0) + for (i = argv0 + 1; i <= NF; i++) + if (($i == "--port" && $(i + 1) == port) || $i == "--port=" port) { + print $1 + break + } + }' fi } From 8faf78f1141b6a467bae1afdaa266d463be63997 Mon Sep 17 00:00:00 2001 From: Cheng-Chin Chiang Date: Fri, 21 Aug 2026 23:00:23 +0800 Subject: [PATCH 12/15] Retry a failed test file once and say so --- .github/actions/run-apptainer/action.yml | 20 +++++++++++++++++++- .github/actions/run-macos/action.yml | 20 +++++++++++++++++++- 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/.github/actions/run-apptainer/action.yml b/.github/actions/run-apptainer/action.yml index 92030596..5a326adb 100644 --- a/.github/actions/run-apptainer/action.yml +++ b/.github/actions/run-apptainer/action.yml @@ -55,6 +55,7 @@ runs: pwd && \ cat $TEST_STAGE && \ failed_tests=() && \ + retried_tests=() && \ crash_sites=() && \ skipped_tests=() && \ restarts=0 && \ @@ -63,8 +64,9 @@ runs: for test_index in \"\${!test_files[@]}\"; do test_file=\"\${test_files[\$test_index]}\" if [ -n \"\$test_file\" ]; then # Skip empty lines + first_attempt_failed='' if ! CI=true npm test -- \"\$test_file\"; then - failed_tests+=(\"\$test_file\") + first_attempt_failed=yes fi # A backend which has died takes every remaining file in the stage with it, each # reporting a connection failure instead of the crash which actually caused it. @@ -90,12 +92,28 @@ runs: break fi fi + # The retry comes after the liveness check so that a file which failed because the + # backend died underneath it is retried against the replacement rather than against + # nothing. One retry, reported either way: the harness this replaces retried + # silently, which made a file that failed once and passed once look like a clean one. + if [ -n \"\$first_attempt_failed\" ]; then + echo \"\$test_file failed; retrying once\" + if CI=true npm test -- \"\$test_file\"; then + retried_tests+=(\"\$test_file\") + else + failed_tests+=(\"\$test_file\") + fi + fi fi done && \ if [ \${#crash_sites[@]} -ne 0 ]; then echo \"carta_backend crashed after:\" printf ' %s\n' \"\${crash_sites[@]}\" fi && \ + if [ \${#retried_tests[@]} -ne 0 ]; then + echo \"The following tests failed once and passed on a retry:\" + printf ' %s\n' \"\${retried_tests[@]}\" + fi && \ remaining_tests=() && \ for skipped_test in \"\${skipped_tests[@]}\"; do if [ -n \"\$skipped_test\" ]; then diff --git a/.github/actions/run-macos/action.yml b/.github/actions/run-macos/action.yml index 9be092ac..f6ff9d91 100644 --- a/.github/actions/run-macos/action.yml +++ b/.github/actions/run-macos/action.yml @@ -28,6 +28,7 @@ runs: ICD_DIR=$GITHUB_WORKSPACE/ICD-RxJS cd $ICD_DIR failed_tests=() + retried_tests=() crash_sites=() skipped_tests=() restarts=0 @@ -41,8 +42,9 @@ runs: for test_index in "${!test_files[@]}"; do test_file="${test_files[$test_index]}" if [ -n "$test_file" ]; then + first_attempt_failed='' if ! CI=true npm test -- "$test_file"; then - failed_tests+=("$test_file") + first_attempt_failed=yes fi # A backend which has died takes every remaining file in the stage with it, each # reporting a connection failure instead of the crash which actually caused it. @@ -68,12 +70,28 @@ runs: break fi fi + # The retry comes after the liveness check so that a file which failed because the + # backend died underneath it is retried against the replacement rather than against + # nothing. One retry, reported either way: the harness this replaces retried silently, + # which made a file that failed once and passed once look exactly like a clean one. + if [ -n "$first_attempt_failed" ]; then + echo "$test_file failed; retrying once" + if CI=true npm test -- "$test_file"; then + retried_tests+=("$test_file") + else + failed_tests+=("$test_file") + fi + fi fi done if [ ${#crash_sites[@]} -ne 0 ]; then echo "carta_backend crashed after:" printf ' %s\n' "${crash_sites[@]}" fi + if [ ${#retried_tests[@]} -ne 0 ]; then + echo "The following tests failed once and passed on a retry:" + printf ' %s\n' "${retried_tests[@]}" + fi remaining_tests=() for skipped_test in "${skipped_tests[@]}"; do if [ -n "$skipped_test" ]; then From 60a1f04a9e3a8296a62c05f785607ac263d01d6a Mon Sep 17 00:00:00 2001 From: Cheng-Chin Chiang Date: Sun, 23 Aug 2026 00:37:10 +0800 Subject: [PATCH 13/15] Run the test stage from one script instead of two copies --- .github/actions/run-apptainer/action.yml | 93 ++------------ .github/actions/run-macos/action.yml | 87 +------------ scripts/run_test_stage.sh | 153 +++++++++++++++++++++++ 3 files changed, 165 insertions(+), 168 deletions(-) create mode 100644 scripts/run_test_stage.sh diff --git a/.github/actions/run-apptainer/action.yml b/.github/actions/run-apptainer/action.yml index 5a326adb..90444905 100644 --- a/.github/actions/run-apptainer/action.yml +++ b/.github/actions/run-apptainer/action.yml @@ -23,7 +23,7 @@ runs: SRC_DIR=$GITHUB_WORKSPACE/source BUILD_DIR=$GITHUB_WORKSPACE/build-${{ inputs.os_version }} - TEST_STAGE="$BUILD_DIR/ICD-RxJS/ICD_test_stages/${{ inputs.test_stage_name }}.tests" + SCRIPT_DIR="$BUILD_DIR/ICD-RxJS/scripts" LOG_FILE="/tmp/carta_icd_${{ inputs.os_version }}_${{ inputs.test_stage_name }}.log" # Function to cleanup backend process. @@ -32,13 +32,18 @@ runs: # appearing anywhere in a command line: this scan also sees the host's own # `apptainer exec ... /bin/bash -c ""`, whose command line mentions # both, and killing that tears the container down instead of the backend. + # `|| true` and the final `return 0` because this runs from a trap under `set -e`: pgrep + # exits 1 when it matches nothing, and the status of the last command a trap runs becomes + # the status of the step. A stage which passed but left no backend to clean up would + # otherwise be reported as a failure. cleanup() { local backend_pids - backend_pids=$(pgrep -f "(^|[/[:space:]])carta_backend[[:space:]].*--port[[:space:]=]+${{ inputs.port }}([[:space:]]|$)") + backend_pids=$(pgrep -f "(^|[/[:space:]])carta_backend[[:space:]].*--port[[:space:]=]+${{ inputs.port }}([[:space:]]|$)" || true) if [ -n "$backend_pids" ]; then echo "Cleaning up carta_backend (PID: $(echo $backend_pids | tr '\n' ' '))" kill -9 $backend_pids || true fi + return 0 } # Set trap for cleanup @@ -49,88 +54,8 @@ runs: --bind /images:/images \ --pwd $BUILD_DIR \ ${{ inputs.image }} /bin/bash -c "\ - bash $BUILD_DIR/ICD-RxJS/scripts/start_backend.sh $SRC_DIR $BUILD_DIR ${{ inputs.port }} $LOG_FILE && \ - # Run the ICD tests - cd $BUILD_DIR/ICD-RxJS && \ - pwd && \ - cat $TEST_STAGE && \ - failed_tests=() && \ - retried_tests=() && \ - crash_sites=() && \ - skipped_tests=() && \ - restarts=0 && \ - max_restarts=3 && \ - mapfile -t test_files < $TEST_STAGE && \ - for test_index in \"\${!test_files[@]}\"; do - test_file=\"\${test_files[\$test_index]}\" - if [ -n \"\$test_file\" ]; then # Skip empty lines - first_attempt_failed='' - if ! CI=true npm test -- \"\$test_file\"; then - first_attempt_failed=yes - fi - # A backend which has died takes every remaining file in the stage with it, each - # reporting a connection failure instead of the crash which actually caused it. - # The tests hold no state in the backend -- every file connects and loads for - # itself -- so a fresh backend lets the rest of the stage run for real. - if ! bash scripts/wait_for_backend.sh ${{ inputs.port }} 0 $LOG_FILE; then - crash_sites+=(\"\$test_file\") - if [ \"\$restarts\" -ge \"\$max_restarts\" ]; then - echo \"carta_backend crashed after \$test_file and has already been restarted \$restarts times; abandoning the rest of the stage\" - skipped_tests=(\"\${test_files[@]:\$((test_index + 1))}\") - break - fi - restarts=\$((restarts + 1)) - echo \"carta_backend crashed after \$test_file; restarting (\$restarts of \$max_restarts)\" - # A restart gets a shorter budget than the 120s cold start. The binary is warm by - # now, and the job itself is capped: three restarts at the full budget would spend - # the cap on waiting and get the stage cancelled before it could report anything, - # which is exactly how this showed up -- an exit 143 in the middle of a restart. - # 90s still leaves 30s an attempt, which is what the old harness allowed a restart. - if ! bash scripts/start_backend.sh $SRC_DIR $BUILD_DIR ${{ inputs.port }} $LOG_FILE 90; then - echo \"carta_backend could not be restarted; abandoning the rest of the stage\" - skipped_tests=(\"\${test_files[@]:\$((test_index + 1))}\") - break - fi - fi - # The retry comes after the liveness check so that a file which failed because the - # backend died underneath it is retried against the replacement rather than against - # nothing. One retry, reported either way: the harness this replaces retried - # silently, which made a file that failed once and passed once look like a clean one. - if [ -n \"\$first_attempt_failed\" ]; then - echo \"\$test_file failed; retrying once\" - if CI=true npm test -- \"\$test_file\"; then - retried_tests+=(\"\$test_file\") - else - failed_tests+=(\"\$test_file\") - fi - fi - fi - done && \ - if [ \${#crash_sites[@]} -ne 0 ]; then - echo \"carta_backend crashed after:\" - printf ' %s\n' \"\${crash_sites[@]}\" - fi && \ - if [ \${#retried_tests[@]} -ne 0 ]; then - echo \"The following tests failed once and passed on a retry:\" - printf ' %s\n' \"\${retried_tests[@]}\" - fi && \ - remaining_tests=() && \ - for skipped_test in \"\${skipped_tests[@]}\"; do - if [ -n \"\$skipped_test\" ]; then - remaining_tests+=(\"\$skipped_test\") - fi - done && \ - if [ \${#remaining_tests[@]} -ne 0 ]; then - echo \"The following tests did not run:\" - printf ' %s\n' \"\${remaining_tests[@]}\" - fi && \ - if [ \${#failed_tests[@]} -ne 0 ]; then - echo \"The following tests failed:\" && \ - printf '%s\n' \"\${failed_tests[@]}\" - fi && \ - if [ \${#failed_tests[@]} -ne 0 ] || [ \${#crash_sites[@]} -ne 0 ]; then - exit 1 - fi" + bash $SCRIPT_DIR/start_backend.sh $SRC_DIR $BUILD_DIR ${{ inputs.port }} $LOG_FILE && \ + bash $SCRIPT_DIR/run_test_stage.sh $SRC_DIR $BUILD_DIR ${{ inputs.port }} ${{ inputs.test_stage_name }} $LOG_FILE" shell: bash - name: Upload backend log on failure diff --git a/.github/actions/run-macos/action.yml b/.github/actions/run-macos/action.yml index f6ff9d91..99de66ec 100644 --- a/.github/actions/run-macos/action.yml +++ b/.github/actions/run-macos/action.yml @@ -25,90 +25,9 @@ runs: - name: ICD tests run: | - ICD_DIR=$GITHUB_WORKSPACE/ICD-RxJS - cd $ICD_DIR - failed_tests=() - retried_tests=() - crash_sites=() - skipped_tests=() - restarts=0 - max_restarts=3 - # Read with a loop rather than mapfile: macOS ships bash 3.2, which does not have it. The - # `|| [ -n "$line" ]` keeps the last entry of a file with no trailing newline. - test_files=() - while IFS= read -r line || [ -n "$line" ]; do - test_files+=("$line") - done < ICD_test_stages/${{ inputs.test_stage_name }}.tests - for test_index in "${!test_files[@]}"; do - test_file="${test_files[$test_index]}" - if [ -n "$test_file" ]; then - first_attempt_failed='' - if ! CI=true npm test -- "$test_file"; then - first_attempt_failed=yes - fi - # A backend which has died takes every remaining file in the stage with it, each - # reporting a connection failure instead of the crash which actually caused it. - # The tests hold no state in the backend -- every file connects and loads for - # itself -- so a fresh backend lets the rest of the stage run for real. - if ! bash scripts/wait_for_backend.sh ${{ inputs.port }} 0 "$CARTA_LOG_FILE"; then - crash_sites+=("$test_file") - if [ "$restarts" -ge "$max_restarts" ]; then - echo "carta_backend crashed after $test_file and has already been restarted $restarts times; abandoning the rest of the stage" - skipped_tests=("${test_files[@]:$((test_index + 1))}") - break - fi - restarts=$((restarts + 1)) - echo "carta_backend crashed after $test_file; restarting ($restarts of $max_restarts)" - # A restart gets a shorter budget than the 120s cold start. The binary is warm by now, - # and the job itself is capped: three restarts at the full budget would spend the cap - # on waiting and get the stage cancelled before it could report anything. 90s still - # leaves 30s an attempt, which is what the old harness allowed a restart. - if ! bash scripts/start_backend.sh \ - $GITHUB_WORKSPACE/source $GITHUB_WORKSPACE/build ${{ inputs.port }} "$CARTA_LOG_FILE" 90; then - echo "carta_backend could not be restarted; abandoning the rest of the stage" - skipped_tests=("${test_files[@]:$((test_index + 1))}") - break - fi - fi - # The retry comes after the liveness check so that a file which failed because the - # backend died underneath it is retried against the replacement rather than against - # nothing. One retry, reported either way: the harness this replaces retried silently, - # which made a file that failed once and passed once look exactly like a clean one. - if [ -n "$first_attempt_failed" ]; then - echo "$test_file failed; retrying once" - if CI=true npm test -- "$test_file"; then - retried_tests+=("$test_file") - else - failed_tests+=("$test_file") - fi - fi - fi - done - if [ ${#crash_sites[@]} -ne 0 ]; then - echo "carta_backend crashed after:" - printf ' %s\n' "${crash_sites[@]}" - fi - if [ ${#retried_tests[@]} -ne 0 ]; then - echo "The following tests failed once and passed on a retry:" - printf ' %s\n' "${retried_tests[@]}" - fi - remaining_tests=() - for skipped_test in "${skipped_tests[@]}"; do - if [ -n "$skipped_test" ]; then - remaining_tests+=("$skipped_test") - fi - done - if [ ${#remaining_tests[@]} -ne 0 ]; then - echo "The following tests did not run:" - printf ' %s\n' "${remaining_tests[@]}" - fi - if [ ${#failed_tests[@]} -ne 0 ]; then - echo "The following tests failed:" - printf '%s\n' "${failed_tests[@]}" - fi - if [ ${#failed_tests[@]} -ne 0 ] || [ ${#crash_sites[@]} -ne 0 ]; then - exit 1 - fi + bash $GITHUB_WORKSPACE/ICD-RxJS/scripts/run_test_stage.sh \ + $GITHUB_WORKSPACE/source $GITHUB_WORKSPACE/build \ + ${{ inputs.port }} ${{ inputs.test_stage_name }} "$CARTA_LOG_FILE" shell: bash - name: Upload backend log on failure diff --git a/scripts/run_test_stage.sh b/scripts/run_test_stage.sh new file mode 100644 index 00000000..7d8ebde4 --- /dev/null +++ b/scripts/run_test_stage.sh @@ -0,0 +1,153 @@ +#!/bin/bash +# +# Run one ICD test stage against a running carta_backend, restarting the backend if it dies. +# +# Both CI actions used to carry their own copy of this loop. Keeping them in step meant editing the +# same logic twice, once as ordinary shell and once as a backslash-escaped string inside +# `apptainer exec ... /bin/bash -c "..."`, where every quote and every `$` has to be escaped by hand +# and a mistake is a runtime error on a self-hosted runner rather than something a reviewer can see. +# +# Usage: run_test_stage.sh SRC_DIR BUILD_DIR PORT STAGE_NAME LOG_FILE [MAX_RESTARTS] +# +# Runs every file listed in ICD_test_stages/STAGE_NAME.tests, one `npm test` each, and exits +# non-zero if any file failed twice or the backend crashed at any point. SRC_DIR and BUILD_DIR are +# needed only to restart the backend with the invocation it was started with. +# +# Deliberately no `set -u`, unlike its sibling scripts: bash 3.2 -- which is what macOS runs `run:` +# blocks with -- treats "${empty_array[@]}" as an unbound variable, and this script cannot avoid +# expanding arrays which are empty on a clean run. `set -e` is left out for the same reason it is +# left out of the actions: every command whose failure matters is already tested by hand, and the +# ones whose failure does not matter (a test file failing, a liveness probe) are the normal path. + +# A stage tolerates this many crashes and keeps testing. Past it the backend is not coming back in +# any useful sense, and the remaining files would each report the same connection failure. +DEFAULT_MAX_RESTARTS=3 +# A restart gets a shorter budget than the 120s cold start. The binary is warm by now, and the job +# itself is capped: three restarts at the full budget would spend the cap on waiting and get the +# stage cancelled before it could report anything, which is exactly how this showed up -- an +# exit 143 in the middle of a restart. 90s still leaves 30s an attempt, which is what the old +# harness allowed a restart. +RESTART_TIMEOUT=90 + +src_dir=${1:-} +build_dir=${2:-} +port=${3:-} +stage_name=${4:-} +log_file=${5:-} +max_restarts=${6:-$DEFAULT_MAX_RESTARTS} + +if [ -z "$src_dir" ] || [ -z "$build_dir" ] || [ -z "$port" ] || [ -z "$stage_name" ] || [ -z "$log_file" ]; then + echo "usage: $0 SRC_DIR BUILD_DIR PORT STAGE_NAME LOG_FILE [MAX_RESTARTS]" >&2 + exit 2 +fi + +# This script lives in ICD-RxJS/scripts/, so the checkout it belongs to is its own grandparent. +# Derived rather than passed in: the two actions put the checkout in different places, and a path +# which disagreed with the script actually running would be the kind of mistake that only shows up +# on a runner. +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +icd_dir=$(dirname "$script_dir") + +stage_file="$icd_dir/ICD_test_stages/$stage_name.tests" +if [ ! -f "$stage_file" ]; then + echo "no such test stage: $stage_file" >&2 + exit 2 +fi + +if ! cd "$icd_dir"; then + echo "no such ICD-RxJS directory: $icd_dir" >&2 + exit 2 +fi + +cat "$stage_file" + +# Read with a loop rather than mapfile: macOS ships bash 3.2, which does not have it. The +# `|| [ -n "$line" ]` keeps the last entry of a file with no trailing newline. +test_files=() +while IFS= read -r line || [ -n "$line" ]; do + test_files+=("$line") +done < "$stage_file" + +failed_tests=() +retried_tests=() +crash_sites=() +skipped_tests=() +restarts=0 + +for test_index in "${!test_files[@]}"; do + test_file="${test_files[$test_index]}" + [ -n "$test_file" ] || continue # Skip empty lines + + first_attempt_failed='' + if ! CI=true npm test -- "$test_file"; then + first_attempt_failed=yes + fi + + # A backend which has died takes every remaining file in the stage with it, each reporting a + # connection failure instead of the crash which actually caused it. The tests hold no state in + # the backend -- every file connects and loads for itself -- so a fresh backend lets the rest of + # the stage run for real. + if ! bash "$script_dir/wait_for_backend.sh" "$port" 0 "$log_file"; then + crash_sites+=("$test_file") + if [ "$restarts" -ge "$max_restarts" ]; then + echo "carta_backend crashed after $test_file and has already been restarted $restarts times; abandoning the rest of the stage" + skipped_tests=("${test_files[@]:$((test_index + 1))}") + break + fi + restarts=$((restarts + 1)) + echo "carta_backend crashed after $test_file; restarting ($restarts of $max_restarts)" + if ! bash "$script_dir/start_backend.sh" "$src_dir" "$build_dir" "$port" "$log_file" "$RESTART_TIMEOUT"; then + echo "carta_backend could not be restarted; abandoning the rest of the stage" + skipped_tests=("${test_files[@]:$((test_index + 1))}") + break + fi + fi + + # The retry comes after the liveness check so that a file which failed because the backend died + # underneath it is retried against the replacement rather than against nothing. One retry, + # reported either way: the harness this replaces retried silently, which made a file that failed + # once and passed once look exactly like a clean one. + if [ -n "$first_attempt_failed" ]; then + echo "$test_file failed; retrying once" + if CI=true npm test -- "$test_file"; then + retried_tests+=("$test_file") + else + failed_tests+=("$test_file") + fi + fi +done + +if [ ${#crash_sites[@]} -ne 0 ]; then + echo "carta_backend crashed after:" + printf ' %s\n' "${crash_sites[@]}" +fi + +if [ ${#retried_tests[@]} -ne 0 ]; then + echo "The following tests failed once and passed on a retry:" + printf ' %s\n' "${retried_tests[@]}" +fi + +# Filtered rather than printed directly: slicing a bash 3.2 array past its last element yields one +# empty string rather than nothing, which would otherwise be reported as a test that did not run. +remaining_tests=() +for skipped_test in "${skipped_tests[@]}"; do + if [ -n "$skipped_test" ]; then + remaining_tests+=("$skipped_test") + fi +done +if [ ${#remaining_tests[@]} -ne 0 ]; then + echo "The following tests did not run:" + printf ' %s\n' "${remaining_tests[@]}" +fi + +if [ ${#failed_tests[@]} -ne 0 ]; then + echo "The following tests failed:" + printf '%s\n' "${failed_tests[@]}" +fi + +# A crash fails the stage even when every test which managed to run passed: the point of this +# harness is that a backend dying is itself the result worth reporting. +if [ ${#failed_tests[@]} -ne 0 ] || [ ${#crash_sites[@]} -ne 0 ]; then + exit 1 +fi +exit 0 From 39eadd210f7a7f76f6e4e8e4e8d316eab2f8a28a Mon Sep 17 00:00:00 2001 From: Cheng-Chin Chiang Date: Sat, 29 Aug 2026 00:03:41 +0800 Subject: [PATCH 14/15] Check the backend after a retry, not only after the first attempt --- scripts/run_test_stage.sh | 81 +++++++++++++++++++++++++++++++-------- 1 file changed, 65 insertions(+), 16 deletions(-) mode change 100644 => 100755 scripts/run_test_stage.sh diff --git a/scripts/run_test_stage.sh b/scripts/run_test_stage.sh old mode 100644 new mode 100755 index 7d8ebde4..4cd40184 --- a/scripts/run_test_stage.sh +++ b/scripts/run_test_stage.sh @@ -74,6 +74,48 @@ crash_sites=() skipped_tests=() restarts=0 +# The index of the last file which will actually run, so that a crash on it can be reported +# without waiting for a replacement backend that nothing is left to use. +last_index=-1 +for test_index in "${!test_files[@]}"; do + if [ -n "${test_files[$test_index]}" ]; then + last_index=$test_index + fi +done + +# Whether the backend survived TEST_FILE. A crash is recorded either way; MORE_TO_RUN, if set, +# says that something is still going to be run against the backend and it is therefore worth +# bringing back. Returns non-zero when the stage has to be abandoned. +check_backend() { + local test_file=$1 more_to_run=$2 + + if bash "$script_dir/wait_for_backend.sh" "$port" 0 "$log_file"; then + return 0 + fi + + crash_sites+=("$test_file") + + # Nothing else was going to run, so the crash is the whole of the news. Restarting here would + # spend the restart budget -- and the job's, which is what once turned a reportable crash into + # an exit 143 -- on a stage that is already over. + if [ -z "$more_to_run" ]; then + echo "carta_backend crashed after $test_file" + return 0 + fi + + if [ "$restarts" -ge "$max_restarts" ]; then + echo "carta_backend crashed after $test_file and has already been restarted $restarts times; abandoning the rest of the stage" + return 1 + fi + restarts=$((restarts + 1)) + echo "carta_backend crashed after $test_file; restarting ($restarts of $max_restarts)" + if ! bash "$script_dir/start_backend.sh" "$src_dir" "$build_dir" "$port" "$log_file" "$RESTART_TIMEOUT"; then + echo "carta_backend could not be restarted; abandoning the rest of the stage" + return 1 + fi + return 0 +} + for test_index in "${!test_files[@]}"; do test_file="${test_files[$test_index]}" [ -n "$test_file" ] || continue # Skip empty lines @@ -86,21 +128,15 @@ for test_index in "${!test_files[@]}"; do # A backend which has died takes every remaining file in the stage with it, each reporting a # connection failure instead of the crash which actually caused it. The tests hold no state in # the backend -- every file connects and loads for itself -- so a fresh backend lets the rest of - # the stage run for real. - if ! bash "$script_dir/wait_for_backend.sh" "$port" 0 "$log_file"; then - crash_sites+=("$test_file") - if [ "$restarts" -ge "$max_restarts" ]; then - echo "carta_backend crashed after $test_file and has already been restarted $restarts times; abandoning the rest of the stage" - skipped_tests=("${test_files[@]:$((test_index + 1))}") - break - fi - restarts=$((restarts + 1)) - echo "carta_backend crashed after $test_file; restarting ($restarts of $max_restarts)" - if ! bash "$script_dir/start_backend.sh" "$src_dir" "$build_dir" "$port" "$log_file" "$RESTART_TIMEOUT"; then - echo "carta_backend could not be restarted; abandoning the rest of the stage" - skipped_tests=("${test_files[@]:$((test_index + 1))}") - break - fi + # the stage run for real. A retry still to come counts as a reason to bring it back, just as + # another file does. + more_to_run='' + if [ -n "$first_attempt_failed" ] || [ "$test_index" -ne "$last_index" ]; then + more_to_run=yes + fi + if ! check_backend "$test_file" "$more_to_run"; then + skipped_tests=("${test_files[@]:$((test_index + 1))}") + break fi # The retry comes after the liveness check so that a file which failed because the backend died @@ -114,6 +150,19 @@ for test_index in "${!test_files[@]}"; do else failed_tests+=("$test_file") fi + + # A retry can take the backend down exactly as a first attempt can, and checking here + # rather than leaving it to the next file keeps the crash on the file which caused it and + # keeps the next file from being run against a backend which is already gone. On the last + # file of a stage this is the only thing which reports the crash at all. + more_to_run='' + if [ "$test_index" -ne "$last_index" ]; then + more_to_run=yes + fi + if ! check_backend "$test_file" "$more_to_run"; then + skipped_tests=("${test_files[@]:$((test_index + 1))}") + break + fi fi done @@ -142,7 +191,7 @@ fi if [ ${#failed_tests[@]} -ne 0 ]; then echo "The following tests failed:" - printf '%s\n' "${failed_tests[@]}" + printf ' %s\n' "${failed_tests[@]}" fi # A crash fails the stage even when every test which managed to run passed: the point of this From 0524e947a485317360c36900ff667c4f8fb90d94 Mon Sep 17 00:00:00 2001 From: Cheng-Chin Chiang Date: Sat, 29 Aug 2026 10:56:16 +0800 Subject: [PATCH 15/15] Take the macOS port from the matrix, not from two literals --- .github/actions/run-macos/action.yml | 7 ++- .github/workflows/all_stages_icd_tests.yml | 64 +++++++++++++++++++- .github/workflows/icd_tests.yml | 64 +++++++++++++++++++- .github/workflows/single_stage_icd_tests.yml | 64 +++++++++++++++++++- 4 files changed, 194 insertions(+), 5 deletions(-) diff --git a/.github/actions/run-macos/action.yml b/.github/actions/run-macos/action.yml index 99de66ec..2097d447 100644 --- a/.github/actions/run-macos/action.yml +++ b/.github/actions/run-macos/action.yml @@ -4,10 +4,13 @@ inputs: test_stage_name: description: 'ICD test stage' required: true + # Required, and deliberately without a default: the caller has to rewrite config.json with the + # same port, and a default here would let the two drift apart silently. The tests would then dial + # a port nothing is listening on while every liveness check in the stage passes, which reports as + # a whole stage of connection failures with a backend that was healthy throughout. port: description: 'Port number for carta_backend' - required: false - default: '5555' + required: true runs: using: 'composite' steps: diff --git a/.github/workflows/all_stages_icd_tests.yml b/.github/workflows/all_stages_icd_tests.yml index 3d65c8a1..64591880 100644 --- a/.github/workflows/all_stages_icd_tests.yml +++ b/.github/workflows/all_stages_icd_tests.yml @@ -22,12 +22,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -159,12 +162,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -209,7 +215,7 @@ jobs: cd protobuf ./build_proto.sh cd ../src/test - perl -p -i -e 's/3002/5555/' config.json + perl -p -i -e 's/3002/${{ matrix.port }}/' config.json - name: Prepare ICD-RxJS (Linux) if: matrix.os == 'linux' @@ -239,12 +245,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -272,6 +281,7 @@ jobs: if: matrix.os == 'macos' uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'file_browser' # Linux steps - name: File Browser ICD tests @@ -294,12 +304,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -327,6 +340,7 @@ jobs: if: matrix.os == 'macos' uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'animator' # Linux steps - name: Animator ICD tests @@ -349,12 +363,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -382,6 +399,7 @@ jobs: if: matrix.os == 'macos' uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'region_statistics' # Linux steps - name: Region-Statistics ICD tests @@ -404,12 +422,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -437,6 +458,7 @@ jobs: if: matrix.os == 'macos' uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'region_manipulation' # Linux steps - name: Region Manipulation ICD tests @@ -459,12 +481,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -492,6 +517,7 @@ jobs: if: matrix.os == 'macos' uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'cube_histogram' # Linux steps - name: Cube Histogram ICD tests @@ -514,12 +540,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -547,6 +576,7 @@ jobs: if: matrix.os == 'macos' uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'pv_generator' # Linux steps - name: PV Generator ICD tests @@ -569,12 +599,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -602,6 +635,7 @@ jobs: if: matrix.os == 'macos' uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'raster_tiles' # Linux steps - name: Raster Tiles ICD tests @@ -624,12 +658,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -657,6 +694,7 @@ jobs: if: matrix.os == 'macos' uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'catalog' # Linux steps - name: Catalog ICD tests @@ -679,12 +717,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -712,6 +753,7 @@ jobs: if: matrix.os == 'macos' uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'moment' # Linux steps - name: Moment ICD tests @@ -734,12 +776,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -767,6 +812,7 @@ jobs: if: matrix.os == 'macos' uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'match' # Linux steps - name: Match ICD tests @@ -789,12 +835,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -822,6 +871,7 @@ jobs: if: matrix.os == 'macos' uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'close_file' # Linux steps - name: Close File ICD tests @@ -844,12 +894,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -877,6 +930,7 @@ jobs: if: matrix.os == 'macos' uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'image_fitting' # Linux steps - name: Image Fitting ICD tests @@ -899,12 +953,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -932,6 +989,7 @@ jobs: if: matrix.os == 'macos' uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'vector_overlay' # Linux steps - name: Vector Overlay ICD tests @@ -954,12 +1012,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -987,6 +1048,7 @@ jobs: if: matrix.os == 'macos' uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'resume' # Linux steps - name: Resume ICD tests diff --git a/.github/workflows/icd_tests.yml b/.github/workflows/icd_tests.yml index c443a06f..dc43a469 100644 --- a/.github/workflows/icd_tests.yml +++ b/.github/workflows/icd_tests.yml @@ -51,12 +51,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -188,12 +191,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -238,7 +244,7 @@ jobs: cd protobuf ./build_proto.sh cd ../src/test - perl -p -i -e 's/3002/5555/' config.json + perl -p -i -e 's/3002/${{ matrix.port }}/' config.json - name: Prepare ICD-RxJS (Linux) if: matrix.os == 'linux' @@ -268,12 +274,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -302,6 +311,7 @@ jobs: if: matrix.os == 'macos' uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'file_browser' # Linux steps - name: File Browser ICD tests @@ -324,12 +334,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -358,6 +371,7 @@ jobs: if: matrix.os == 'macos' uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'animator' # Linux steps - name: Animator ICD tests @@ -380,12 +394,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -414,6 +431,7 @@ jobs: if: matrix.os == 'macos' uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'region_statistics' # Linux steps - name: Region-Statistics ICD tests @@ -436,12 +454,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -470,6 +491,7 @@ jobs: if: matrix.os == 'macos' uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'region_manipulation' # Linux steps - name: Region Manipulation ICD tests @@ -492,12 +514,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -526,6 +551,7 @@ jobs: if: matrix.os == 'macos' uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'cube_histogram' # Linux steps - name: Cube Histogram ICD tests @@ -548,12 +574,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -582,6 +611,7 @@ jobs: if: matrix.os == 'macos' uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'pv_generator' # Linux steps - name: PV Generator ICD tests @@ -604,12 +634,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -638,6 +671,7 @@ jobs: if: matrix.os == 'macos' uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'raster_tiles' # Linux steps - name: Raster Tiles ICD tests @@ -660,12 +694,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -694,6 +731,7 @@ jobs: if: matrix.os == 'macos' uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'catalog' # Linux steps - name: Catalog ICD tests @@ -716,12 +754,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -750,6 +791,7 @@ jobs: if: matrix.os == 'macos' uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'moment' # Linux steps - name: Moment ICD tests @@ -772,12 +814,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -806,6 +851,7 @@ jobs: if: matrix.os == 'macos' uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'match' # Linux steps - name: Match ICD tests @@ -828,12 +874,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -862,6 +911,7 @@ jobs: if: matrix.os == 'macos' uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'close_file' # Linux steps - name: Close File ICD tests @@ -884,12 +934,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -918,6 +971,7 @@ jobs: if: matrix.os == 'macos' uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'image_fitting' # Linux steps - name: Image Fitting ICD tests @@ -940,12 +994,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -974,6 +1031,7 @@ jobs: if: matrix.os == 'macos' uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'vector_overlay' # Linux steps - name: Vector Overlay ICD tests @@ -996,12 +1054,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -1030,6 +1091,7 @@ jobs: if: matrix.os == 'macos' uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'resume' # Linux steps - name: Resume ICD tests diff --git a/.github/workflows/single_stage_icd_tests.yml b/.github/workflows/single_stage_icd_tests.yml index 0f2883b1..7f075f3e 100644 --- a/.github/workflows/single_stage_icd_tests.yml +++ b/.github/workflows/single_stage_icd_tests.yml @@ -41,12 +41,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -178,12 +181,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -228,7 +234,7 @@ jobs: cd protobuf ./build_proto.sh cd ../src/test - perl -p -i -e 's/3002/5555/' config.json + perl -p -i -e 's/3002/${{ matrix.port }}/' config.json - name: Prepare ICD-RxJS (Linux) if: matrix.os == 'linux' @@ -258,12 +264,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -292,6 +301,7 @@ jobs: if: matrix.os == 'macos' uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'file_browser' # Linux steps - name: File Browser ICD tests @@ -314,12 +324,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -348,6 +361,7 @@ jobs: if: matrix.os == 'macos' uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'animator' # Linux steps - name: Animator ICD tests @@ -370,12 +384,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -404,6 +421,7 @@ jobs: if: matrix.os == 'macos' uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'region_statistics' # Linux steps - name: Region-Statistics ICD tests @@ -426,12 +444,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -460,6 +481,7 @@ jobs: if: matrix.os == 'macos' uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'region_manipulation' # Linux steps - name: Region Manipulation ICD tests @@ -482,12 +504,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -516,6 +541,7 @@ jobs: if: matrix.os == 'macos' uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'cube_histogram' # Linux steps - name: Cube Histogram ICD tests @@ -538,12 +564,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -572,6 +601,7 @@ jobs: if: matrix.os == 'macos' uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'pv_generator' # Linux steps - name: PV Generator ICD tests @@ -594,12 +624,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -628,6 +661,7 @@ jobs: if: matrix.os == 'macos' uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'raster_tiles' # Linux steps - name: Raster Tiles ICD tests @@ -650,12 +684,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -684,6 +721,7 @@ jobs: if: matrix.os == 'macos' uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'catalog' # Linux steps - name: Catalog ICD tests @@ -706,12 +744,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -740,6 +781,7 @@ jobs: if: matrix.os == 'macos' uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'moment' # Linux steps - name: Moment ICD tests @@ -762,12 +804,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -796,6 +841,7 @@ jobs: if: matrix.os == 'macos' uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'match' # Linux steps - name: Match ICD tests @@ -818,12 +864,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -852,6 +901,7 @@ jobs: if: matrix.os == 'macos' uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'close_file' # Linux steps - name: Close File ICD tests @@ -874,12 +924,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -908,6 +961,7 @@ jobs: if: matrix.os == 'macos' uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'image_fitting' # Linux steps - name: Image Fitting ICD tests @@ -930,12 +984,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -964,6 +1021,7 @@ jobs: if: matrix.os == 'macos' uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'vector_overlay' # Linux steps - name: Vector Overlay ICD tests @@ -986,12 +1044,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -1020,6 +1081,7 @@ jobs: if: matrix.os == 'macos' uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'resume' # Linux steps - name: Resume ICD tests