diff --git a/.github/actions/run-apptainer/action.yml b/.github/actions/run-apptainer/action.yml index a2b3e4ca..90444905 100644 --- a/.github/actions/run-apptainer/action.yml +++ b/.github/actions/run-apptainer/action.yml @@ -23,16 +23,27 @@ runs: SRC_DIR=$GITHUB_WORKSPACE/source BUILD_DIR=$GITHUB_WORKSPACE/build-${{ inputs.os_version }} - TEST_STAGE="$BUILD_DIR/ICD-RxJS/ICD_test_stages/${{ inputs.test_stage_name }}.tests" + SCRIPT_DIR="$BUILD_DIR/ICD-RxJS/scripts" LOG_FILE="/tmp/carta_icd_${{ inputs.os_version }}_${{ inputs.test_stage_name }}.log" - # Function to cleanup backend process + # Function to cleanup backend process. + # + # Matched on the executable and its --port argument, not on "carta_backend" and the port + # appearing anywhere in a command line: this scan also sees the host's own + # `apptainer exec ... /bin/bash -c ""`, whose command line mentions + # both, and killing that tears the container down instead of the backend. + # `|| true` and the final `return 0` because this runs from a trap under `set -e`: pgrep + # exits 1 when it matches nothing, and the status of the last command a trap runs becomes + # the status of the step. A stage which passed but left no backend to clean up would + # otherwise be reported as a failure. cleanup() { - local backend_pid=$(pgrep -f "carta_backend.*${{ inputs.port }}" | head -n 1) - if [ ! -z "$backend_pid" ]; then - echo "Cleaning up carta_backend (PID: $backend_pid)" - kill -9 "$backend_pid" || true + local backend_pids + backend_pids=$(pgrep -f "(^|[/[:space:]])carta_backend[[:space:]].*--port[[:space:]=]+${{ inputs.port }}([[:space:]]|$)" || true) + if [ -n "$backend_pids" ]; then + echo "Cleaning up carta_backend (PID: $(echo $backend_pids | tr '\n' ' '))" + kill -9 $backend_pids || true fi + return 0 } # Set trap for cleanup @@ -43,39 +54,8 @@ runs: --bind /images:/images \ --pwd $BUILD_DIR \ ${{ inputs.image }} /bin/bash -c "\ - # Start the carta_backend - ASAN_OPTIONS=suppressions=$SRC_DIR/debug/asan/myasan.supp \ - LSAN_OPTIONS=suppressions=$SRC_DIR/debug/asan/myasan-leaks.supp \ - ASAN_SYMBOLIZER_PATH=llvm-symbolizer \ - ./carta_backend /images \ - --top_level_folder /images \ - --port ${{ inputs.port }} \ - --omp_threads=4 \ - --debug_no_auth \ - --no_frontend \ - --no_database \ - --no_log \ - --verbosity=5 >> $LOG_FILE 2>&1 & \ - CARTA_BACKEND_PID=\$(pgrep -f 'carta_backend.*${{ inputs.port }}' | head -n 1) && \ - echo 'carta_backend started with PID' \$CARTA_BACKEND_PID && \ - # Run the ICD tests - cd $BUILD_DIR/ICD-RxJS && \ - pwd && \ - cat $TEST_STAGE && \ - failed_tests=() && \ - mapfile -t test_files < $TEST_STAGE && \ - for test_file in \"\${test_files[@]}\"; do - if [ -n \"\$test_file\" ]; then # Skip empty lines - if ! CI=true npm test -- \"\$test_file\"; then - failed_tests+=(\"\$test_file\") - fi - fi - done && \ - if [ \${#failed_tests[@]} -ne 0 ]; then - echo \"The following tests failed:\" && \ - printf '%s\n' \"\${failed_tests[@]}\" && \ - exit 1 - fi" + bash $SCRIPT_DIR/start_backend.sh $SRC_DIR $BUILD_DIR ${{ inputs.port }} $LOG_FILE && \ + bash $SCRIPT_DIR/run_test_stage.sh $SRC_DIR $BUILD_DIR ${{ inputs.port }} ${{ inputs.test_stage_name }} $LOG_FILE" shell: bash - name: Upload backend log on failure diff --git a/.github/actions/run-macos/action.yml b/.github/actions/run-macos/action.yml index bd6bcd7e..2097d447 100644 --- a/.github/actions/run-macos/action.yml +++ b/.github/actions/run-macos/action.yml @@ -4,10 +4,13 @@ inputs: test_stage_name: description: 'ICD test stage' required: true + # Required, and deliberately without a default: the caller has to rewrite config.json with the + # same port, and a default here would let the two drift apart silently. The tests would then dial + # a port nothing is listening on while every liveness check in the stage passes, which reports as + # a whole stage of connection failures with a backend that was healthy throughout. port: description: 'Port number for carta_backend' - required: false - default: '5555' + required: true runs: using: 'composite' steps: @@ -17,40 +20,17 @@ runs: - name: Start the carta-backend run: | - SRC_DIR=$GITHUB_WORKSPACE/source - BUILD_DIR=$GITHUB_WORKSPACE/build LOG_FILE="/tmp/carta_icd_macos_${{ inputs.test_stage_name }}.log" - cd $BUILD_DIR - ASAN_OPTIONS=suppressions=$SRC_DIR/debug/asan/myasan.supp \ - LSAN_OPTIONS=suppressions=$SRC_DIR/debug/asan/myasan-leaks.supp \ - ASAN_SYMBOLIZER_PATH=llvm-symbolizer \ - ./carta_backend /images --top_level_folder /images \ - --port ${{ inputs.port }} \ - --omp_threads=4 --debug_no_auth --no_frontend --no_database --no_log \ - --verbosity=5 >> "$LOG_FILE" 2>&1 & - echo "CARTA_BACKEND_PID=$!" >> $GITHUB_ENV echo "CARTA_LOG_FILE=$LOG_FILE" >> $GITHUB_ENV + bash $GITHUB_WORKSPACE/ICD-RxJS/scripts/start_backend.sh \ + $GITHUB_WORKSPACE/source $GITHUB_WORKSPACE/build ${{ inputs.port }} "$LOG_FILE" shell: bash - name: ICD tests run: | - ICD_DIR=$GITHUB_WORKSPACE/ICD-RxJS - cd $ICD_DIR - failed_tests=() - mapfile -t test_files < ICD_test_stages/${{ inputs.test_stage_name }}.tests - for test_file in "${test_files[@]}"; do - if [ -n "$test_file" ]; then - if ! CI=true npm test -- "$test_file"; then - failed_tests+=("$test_file") - fi - sleep 3 && pgrep carta_backend - fi - done - if [ ${#failed_tests[@]} -ne 0 ]; then - echo "The following tests failed:" - printf '%s\n' "${failed_tests[@]}" - exit 1 - fi + bash $GITHUB_WORKSPACE/ICD-RxJS/scripts/run_test_stage.sh \ + $GITHUB_WORKSPACE/source $GITHUB_WORKSPACE/build \ + ${{ inputs.port }} ${{ inputs.test_stage_name }} "$CARTA_LOG_FILE" shell: bash - name: Upload backend log on failure @@ -64,7 +44,10 @@ runs: - name: Stop carta-backend if: always() run: | - if [ -n "${{ env.CARTA_BACKEND_PID }}" ]; then - kill ${{ env.CARTA_BACKEND_PID }} || true - fi + # Matched by port rather than by a PID recorded at startup: the stage may have restarted + # the backend after a crash, in which case that PID is long gone and killing it would + # leave the replacement running on the runner. The executable and its --port argument are + # both required so that this can only match a backend, never a shell whose command line + # happens to mention carta_backend and the port. + pkill -f "(^|[/[:space:]])carta_backend[[:space:]].*--port[[:space:]=]+${{ inputs.port }}([[:space:]]|$)" || true shell: bash diff --git a/.github/workflows/all_stages_icd_tests.yml b/.github/workflows/all_stages_icd_tests.yml index fb1e8e81..64591880 100644 --- a/.github/workflows/all_stages_icd_tests.yml +++ b/.github/workflows/all_stages_icd_tests.yml @@ -22,12 +22,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -159,12 +162,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -209,7 +215,7 @@ jobs: cd protobuf ./build_proto.sh cd ../src/test - perl -p -i -e 's/3002/5555/' config.json + perl -p -i -e 's/3002/${{ matrix.port }}/' config.json - name: Prepare ICD-RxJS (Linux) if: matrix.os == 'linux' @@ -231,7 +237,7 @@ jobs: File-Browser-ICD-Tests: name: File-Browser ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -239,12 +245,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -270,13 +279,14 @@ jobs: # macOS steps - name: File Browser ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'file_browser' # Linux steps - name: File Browser ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -286,7 +296,7 @@ jobs: Animator-ICD-Tests: name: Animator ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -294,12 +304,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -325,13 +338,14 @@ jobs: # macOS steps - name: Animator ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'animator' # Linux steps - name: Animator ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -341,7 +355,7 @@ jobs: Region-Statistics-ICD-Tests: name: Region Statistics ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -349,12 +363,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -380,13 +397,14 @@ jobs: # macOS steps - name: Region-Statistics ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'region_statistics' # Linux steps - name: Region-Statistics ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -396,7 +414,7 @@ jobs: Region-Manipulation-ICD-Tests: name: Region Manipulation ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -404,12 +422,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -435,13 +456,14 @@ jobs: # macOS steps - name: Region Manipulation ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'region_manipulation' # Linux steps - name: Region Manipulation ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -459,12 +481,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -490,13 +515,14 @@ jobs: # macOS steps - name: Cube Histogram ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'cube_histogram' # Linux steps - name: Cube Histogram ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -514,12 +540,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -545,13 +574,14 @@ jobs: # macOS steps - name: PV Generator ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'pv_generator' # Linux steps - name: PV Generator ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -561,7 +591,7 @@ jobs: Raster-Tiles-ICD-Tests: name: Raster Tiles ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -569,12 +599,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -600,13 +633,14 @@ jobs: # macOS steps - name: Raster Tiles ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'raster_tiles' # Linux steps - name: Raster Tiles ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -616,7 +650,7 @@ jobs: Catalog-ICD-Tests: name: Catalog ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -624,12 +658,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -655,13 +692,14 @@ jobs: # macOS steps - name: Catalog ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'catalog' # Linux steps - name: Catalog ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -671,7 +709,7 @@ jobs: Moment-ICD-Tests: name: Moment ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -679,12 +717,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -710,13 +751,14 @@ jobs: # macOS steps - name: Moment ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'moment' # Linux steps - name: Moment ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -726,7 +768,7 @@ jobs: Match-ICD-Tests: name: Match ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -734,12 +776,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -765,13 +810,14 @@ jobs: # macOS steps - name: Match ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'match' # Linux steps - name: Match ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -781,7 +827,7 @@ jobs: Close-File-ICD-Tests: name: Close File ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -789,12 +835,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -820,13 +869,14 @@ jobs: # macOS steps - name: Close File ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'close_file' # Linux steps - name: Close File ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -844,12 +894,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -875,13 +928,14 @@ jobs: # macOS steps - name: Image Fitting ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'image_fitting' # Linux steps - name: Image Fitting ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -891,7 +945,7 @@ jobs: Vector-Overlay-ICD-Tests: name: Vector Overlay ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -899,12 +953,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -930,13 +987,14 @@ jobs: # macOS steps - name: Vector Overlay ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'vector_overlay' # Linux steps - name: Vector Overlay ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -946,7 +1004,7 @@ jobs: Resume-ICD-Tests: name: Resume ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -954,12 +1012,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -985,13 +1046,14 @@ jobs: # macOS steps - name: Resume ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'resume' # Linux steps - name: Resume ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} diff --git a/.github/workflows/icd_tests.yml b/.github/workflows/icd_tests.yml index 76d4f207..dc43a469 100644 --- a/.github/workflows/icd_tests.yml +++ b/.github/workflows/icd_tests.yml @@ -51,12 +51,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -188,12 +191,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -238,7 +244,7 @@ jobs: cd protobuf ./build_proto.sh cd ../src/test - perl -p -i -e 's/3002/5555/' config.json + perl -p -i -e 's/3002/${{ matrix.port }}/' config.json - name: Prepare ICD-RxJS (Linux) if: matrix.os == 'linux' @@ -260,7 +266,7 @@ jobs: File-Browser-ICD-Tests: name: File-Browser ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -268,12 +274,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -300,13 +309,14 @@ jobs: # macOS steps - name: File Browser ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'file_browser' # Linux steps - name: File Browser ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -316,7 +326,7 @@ jobs: Animator-ICD-Tests: name: Animator ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -324,12 +334,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -356,13 +369,14 @@ jobs: # macOS steps - name: Animator ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'animator' # Linux steps - name: Animator ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -372,7 +386,7 @@ jobs: Region-Statistics-ICD-Tests: name: Region Statistics ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -380,12 +394,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -412,13 +429,14 @@ jobs: # macOS steps - name: Region-Statistics ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'region_statistics' # Linux steps - name: Region-Statistics ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -428,7 +446,7 @@ jobs: Region-Manipulation-ICD-Tests: name: Region Manipulation ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -436,12 +454,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -468,13 +489,14 @@ jobs: # macOS steps - name: Region Manipulation ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'region_manipulation' # Linux steps - name: Region Manipulation ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -492,12 +514,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -524,13 +549,14 @@ jobs: # macOS steps - name: Cube Histogram ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'cube_histogram' # Linux steps - name: Cube Histogram ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -548,12 +574,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -580,13 +609,14 @@ jobs: # macOS steps - name: PV Generator ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'pv_generator' # Linux steps - name: PV Generator ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -596,7 +626,7 @@ jobs: Raster-Tiles-ICD-Tests: name: Raster Tiles ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -604,12 +634,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -636,13 +669,14 @@ jobs: # macOS steps - name: Raster Tiles ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'raster_tiles' # Linux steps - name: Raster Tiles ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -652,7 +686,7 @@ jobs: Catalog-ICD-Tests: name: Catalog ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -660,12 +694,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -692,13 +729,14 @@ jobs: # macOS steps - name: Catalog ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'catalog' # Linux steps - name: Catalog ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -708,7 +746,7 @@ jobs: Moment-ICD-Tests: name: Moment ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -716,12 +754,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -748,13 +789,14 @@ jobs: # macOS steps - name: Moment ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'moment' # Linux steps - name: Moment ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -764,7 +806,7 @@ jobs: Match-ICD-Tests: name: Match ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -772,12 +814,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -804,13 +849,14 @@ jobs: # macOS steps - name: Match ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'match' # Linux steps - name: Match ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -820,7 +866,7 @@ jobs: Close-File-ICD-Tests: name: Close File ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -828,12 +874,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -860,13 +909,14 @@ jobs: # macOS steps - name: Close File ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'close_file' # Linux steps - name: Close File ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -884,12 +934,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -916,13 +969,14 @@ jobs: # macOS steps - name: Image Fitting ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'image_fitting' # Linux steps - name: Image Fitting ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -932,7 +986,7 @@ jobs: Vector-Overlay-ICD-Tests: name: Vector Overlay ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -940,12 +994,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -972,13 +1029,14 @@ jobs: # macOS steps - name: Vector Overlay ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'vector_overlay' # Linux steps - name: Vector Overlay ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -988,7 +1046,7 @@ jobs: Resume-ICD-Tests: name: Resume ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -996,12 +1054,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -1028,13 +1089,14 @@ jobs: # macOS steps - name: Resume ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'resume' # Linux steps - name: Resume ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} diff --git a/.github/workflows/single_stage_icd_tests.yml b/.github/workflows/single_stage_icd_tests.yml index c71883c2..7f075f3e 100644 --- a/.github/workflows/single_stage_icd_tests.yml +++ b/.github/workflows/single_stage_icd_tests.yml @@ -41,12 +41,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -178,12 +181,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -228,7 +234,7 @@ jobs: cd protobuf ./build_proto.sh cd ../src/test - perl -p -i -e 's/3002/5555/' config.json + perl -p -i -e 's/3002/${{ matrix.port }}/' config.json - name: Prepare ICD-RxJS (Linux) if: matrix.os == 'linux' @@ -250,7 +256,7 @@ jobs: File-Browser-ICD-Tests: name: File-Browser ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -258,12 +264,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -290,13 +299,14 @@ jobs: # macOS steps - name: File Browser ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'file_browser' # Linux steps - name: File Browser ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -306,7 +316,7 @@ jobs: Animator-ICD-Tests: name: Animator ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -314,12 +324,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -346,13 +359,14 @@ jobs: # macOS steps - name: Animator ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'animator' # Linux steps - name: Animator ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -362,7 +376,7 @@ jobs: Region-Statistics-ICD-Tests: name: Region Statistics ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -370,12 +384,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -402,13 +419,14 @@ jobs: # macOS steps - name: Region-Statistics ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'region_statistics' # Linux steps - name: Region-Statistics ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -418,7 +436,7 @@ jobs: Region-Manipulation-ICD-Tests: name: Region Manipulation ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -426,12 +444,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -458,13 +479,14 @@ jobs: # macOS steps - name: Region Manipulation ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'region_manipulation' # Linux steps - name: Region Manipulation ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -482,12 +504,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -514,13 +539,14 @@ jobs: # macOS steps - name: Cube Histogram ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'cube_histogram' # Linux steps - name: Cube Histogram ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -538,12 +564,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -570,13 +599,14 @@ jobs: # macOS steps - name: PV Generator ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'pv_generator' # Linux steps - name: PV Generator ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -586,7 +616,7 @@ jobs: Raster-Tiles-ICD-Tests: name: Raster Tiles ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -594,12 +624,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -626,13 +659,14 @@ jobs: # macOS steps - name: Raster Tiles ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'raster_tiles' # Linux steps - name: Raster Tiles ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -642,7 +676,7 @@ jobs: Catalog-ICD-Tests: name: Catalog ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -650,12 +684,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -682,13 +719,14 @@ jobs: # macOS steps - name: Catalog ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'catalog' # Linux steps - name: Catalog ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -698,7 +736,7 @@ jobs: Moment-ICD-Tests: name: Moment ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -706,12 +744,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -738,13 +779,14 @@ jobs: # macOS steps - name: Moment ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'moment' # Linux steps - name: Moment ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -754,7 +796,7 @@ jobs: Match-ICD-Tests: name: Match ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -762,12 +804,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -794,13 +839,14 @@ jobs: # macOS steps - name: Match ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'match' # Linux steps - name: Match ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -810,7 +856,7 @@ jobs: Close-File-ICD-Tests: name: Close File ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -818,12 +864,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -850,13 +899,14 @@ jobs: # macOS steps - name: Close File ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'close_file' # Linux steps - name: Close File ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -874,12 +924,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -906,13 +959,14 @@ jobs: # macOS steps - name: Image Fitting ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'image_fitting' # Linux steps - name: Image Fitting ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -922,7 +976,7 @@ jobs: Vector-Overlay-ICD-Tests: name: Vector Overlay ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -930,12 +984,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -962,13 +1019,14 @@ jobs: # macOS steps - name: Vector Overlay ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'vector_overlay' # Linux steps - name: Vector Overlay ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} @@ -978,7 +1036,7 @@ jobs: Resume-ICD-Tests: name: Resume ${{ matrix.os_version }} runs-on: ${{ matrix.runner }} - timeout-minutes: 5 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -986,12 +1044,15 @@ jobs: - os_version: macOS-14 os: macos runner: [macOS-14, ICD] + port: 5555 - os_version: macOS-15 os: macos runner: [macOS-15, ICD] + port: 5555 - os_version: macOS-26 os: macos runner: [macOS-26, ICD] + port: 5555 - os_version: ubuntu-22.04 os: linux runner: [self-hosted, Linux, Apptainer, ICD2] @@ -1018,13 +1079,14 @@ jobs: # macOS steps - name: Resume ICD tests if: matrix.os == 'macos' - uses: ./source/.github/actions/run-macos + uses: ./ICD-RxJS/.github/actions/run-macos with: + port: ${{ matrix.port }} test_stage_name: 'resume' # Linux steps - name: Resume ICD tests if: matrix.os == 'linux' - uses: ./source/.github/actions/run-apptainer + uses: ./ICD-RxJS/.github/actions/run-apptainer with: os_version: ${{ matrix.os_version }} image: ${{ matrix.image }} diff --git a/scripts/run_test_stage.sh b/scripts/run_test_stage.sh new file mode 100755 index 00000000..4cd40184 --- /dev/null +++ b/scripts/run_test_stage.sh @@ -0,0 +1,202 @@ +#!/bin/bash +# +# Run one ICD test stage against a running carta_backend, restarting the backend if it dies. +# +# Both CI actions used to carry their own copy of this loop. Keeping them in step meant editing the +# same logic twice, once as ordinary shell and once as a backslash-escaped string inside +# `apptainer exec ... /bin/bash -c "..."`, where every quote and every `$` has to be escaped by hand +# and a mistake is a runtime error on a self-hosted runner rather than something a reviewer can see. +# +# Usage: run_test_stage.sh SRC_DIR BUILD_DIR PORT STAGE_NAME LOG_FILE [MAX_RESTARTS] +# +# Runs every file listed in ICD_test_stages/STAGE_NAME.tests, one `npm test` each, and exits +# non-zero if any file failed twice or the backend crashed at any point. SRC_DIR and BUILD_DIR are +# needed only to restart the backend with the invocation it was started with. +# +# Deliberately no `set -u`, unlike its sibling scripts: bash 3.2 -- which is what macOS runs `run:` +# blocks with -- treats "${empty_array[@]}" as an unbound variable, and this script cannot avoid +# expanding arrays which are empty on a clean run. `set -e` is left out for the same reason it is +# left out of the actions: every command whose failure matters is already tested by hand, and the +# ones whose failure does not matter (a test file failing, a liveness probe) are the normal path. + +# A stage tolerates this many crashes and keeps testing. Past it the backend is not coming back in +# any useful sense, and the remaining files would each report the same connection failure. +DEFAULT_MAX_RESTARTS=3 +# A restart gets a shorter budget than the 120s cold start. The binary is warm by now, and the job +# itself is capped: three restarts at the full budget would spend the cap on waiting and get the +# stage cancelled before it could report anything, which is exactly how this showed up -- an +# exit 143 in the middle of a restart. 90s still leaves 30s an attempt, which is what the old +# harness allowed a restart. +RESTART_TIMEOUT=90 + +src_dir=${1:-} +build_dir=${2:-} +port=${3:-} +stage_name=${4:-} +log_file=${5:-} +max_restarts=${6:-$DEFAULT_MAX_RESTARTS} + +if [ -z "$src_dir" ] || [ -z "$build_dir" ] || [ -z "$port" ] || [ -z "$stage_name" ] || [ -z "$log_file" ]; then + echo "usage: $0 SRC_DIR BUILD_DIR PORT STAGE_NAME LOG_FILE [MAX_RESTARTS]" >&2 + exit 2 +fi + +# This script lives in ICD-RxJS/scripts/, so the checkout it belongs to is its own grandparent. +# Derived rather than passed in: the two actions put the checkout in different places, and a path +# which disagreed with the script actually running would be the kind of mistake that only shows up +# on a runner. +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +icd_dir=$(dirname "$script_dir") + +stage_file="$icd_dir/ICD_test_stages/$stage_name.tests" +if [ ! -f "$stage_file" ]; then + echo "no such test stage: $stage_file" >&2 + exit 2 +fi + +if ! cd "$icd_dir"; then + echo "no such ICD-RxJS directory: $icd_dir" >&2 + exit 2 +fi + +cat "$stage_file" + +# Read with a loop rather than mapfile: macOS ships bash 3.2, which does not have it. The +# `|| [ -n "$line" ]` keeps the last entry of a file with no trailing newline. +test_files=() +while IFS= read -r line || [ -n "$line" ]; do + test_files+=("$line") +done < "$stage_file" + +failed_tests=() +retried_tests=() +crash_sites=() +skipped_tests=() +restarts=0 + +# The index of the last file which will actually run, so that a crash on it can be reported +# without waiting for a replacement backend that nothing is left to use. +last_index=-1 +for test_index in "${!test_files[@]}"; do + if [ -n "${test_files[$test_index]}" ]; then + last_index=$test_index + fi +done + +# Whether the backend survived TEST_FILE. A crash is recorded either way; MORE_TO_RUN, if set, +# says that something is still going to be run against the backend and it is therefore worth +# bringing back. Returns non-zero when the stage has to be abandoned. +check_backend() { + local test_file=$1 more_to_run=$2 + + if bash "$script_dir/wait_for_backend.sh" "$port" 0 "$log_file"; then + return 0 + fi + + crash_sites+=("$test_file") + + # Nothing else was going to run, so the crash is the whole of the news. Restarting here would + # spend the restart budget -- and the job's, which is what once turned a reportable crash into + # an exit 143 -- on a stage that is already over. + if [ -z "$more_to_run" ]; then + echo "carta_backend crashed after $test_file" + return 0 + fi + + if [ "$restarts" -ge "$max_restarts" ]; then + echo "carta_backend crashed after $test_file and has already been restarted $restarts times; abandoning the rest of the stage" + return 1 + fi + restarts=$((restarts + 1)) + echo "carta_backend crashed after $test_file; restarting ($restarts of $max_restarts)" + if ! bash "$script_dir/start_backend.sh" "$src_dir" "$build_dir" "$port" "$log_file" "$RESTART_TIMEOUT"; then + echo "carta_backend could not be restarted; abandoning the rest of the stage" + return 1 + fi + return 0 +} + +for test_index in "${!test_files[@]}"; do + test_file="${test_files[$test_index]}" + [ -n "$test_file" ] || continue # Skip empty lines + + first_attempt_failed='' + if ! CI=true npm test -- "$test_file"; then + first_attempt_failed=yes + fi + + # A backend which has died takes every remaining file in the stage with it, each reporting a + # connection failure instead of the crash which actually caused it. The tests hold no state in + # the backend -- every file connects and loads for itself -- so a fresh backend lets the rest of + # the stage run for real. A retry still to come counts as a reason to bring it back, just as + # another file does. + more_to_run='' + if [ -n "$first_attempt_failed" ] || [ "$test_index" -ne "$last_index" ]; then + more_to_run=yes + fi + if ! check_backend "$test_file" "$more_to_run"; then + skipped_tests=("${test_files[@]:$((test_index + 1))}") + break + fi + + # The retry comes after the liveness check so that a file which failed because the backend died + # underneath it is retried against the replacement rather than against nothing. One retry, + # reported either way: the harness this replaces retried silently, which made a file that failed + # once and passed once look exactly like a clean one. + if [ -n "$first_attempt_failed" ]; then + echo "$test_file failed; retrying once" + if CI=true npm test -- "$test_file"; then + retried_tests+=("$test_file") + else + failed_tests+=("$test_file") + fi + + # A retry can take the backend down exactly as a first attempt can, and checking here + # rather than leaving it to the next file keeps the crash on the file which caused it and + # keeps the next file from being run against a backend which is already gone. On the last + # file of a stage this is the only thing which reports the crash at all. + more_to_run='' + if [ "$test_index" -ne "$last_index" ]; then + more_to_run=yes + fi + if ! check_backend "$test_file" "$more_to_run"; then + skipped_tests=("${test_files[@]:$((test_index + 1))}") + break + fi + fi +done + +if [ ${#crash_sites[@]} -ne 0 ]; then + echo "carta_backend crashed after:" + printf ' %s\n' "${crash_sites[@]}" +fi + +if [ ${#retried_tests[@]} -ne 0 ]; then + echo "The following tests failed once and passed on a retry:" + printf ' %s\n' "${retried_tests[@]}" +fi + +# Filtered rather than printed directly: slicing a bash 3.2 array past its last element yields one +# empty string rather than nothing, which would otherwise be reported as a test that did not run. +remaining_tests=() +for skipped_test in "${skipped_tests[@]}"; do + if [ -n "$skipped_test" ]; then + remaining_tests+=("$skipped_test") + fi +done +if [ ${#remaining_tests[@]} -ne 0 ]; then + echo "The following tests did not run:" + printf ' %s\n' "${remaining_tests[@]}" +fi + +if [ ${#failed_tests[@]} -ne 0 ]; then + echo "The following tests failed:" + printf ' %s\n' "${failed_tests[@]}" +fi + +# A crash fails the stage even when every test which managed to run passed: the point of this +# harness is that a backend dying is itself the result worth reporting. +if [ ${#failed_tests[@]} -ne 0 ] || [ ${#crash_sites[@]} -ne 0 ]; then + exit 1 +fi +exit 0 diff --git a/scripts/start_backend.sh b/scripts/start_backend.sh new file mode 100755 index 00000000..7e2553df --- /dev/null +++ b/scripts/start_backend.sh @@ -0,0 +1,173 @@ +#!/bin/bash +# +# Start carta_backend and wait until it is accepting connections. +# +# Both CI actions used to carry their own copy of this command line, which made it easy for the +# two to drift and impossible to bring the backend back up from inside a test loop. With one +# script a stage can restart a crashed backend using exactly the invocation it was started with, +# so the files after the crash still run against a real backend instead of all reporting the same +# connection failure. +# +# Usage: start_backend.sh SRC_DIR BUILD_DIR PORT LOG_FILE [TIMEOUT_SECONDS] +# +# The backend's output is appended to LOG_FILE, so a restart leaves the previous crash report in +# place rather than overwriting the very thing we started the backend again to explain. Exits +# non-zero, with the tail of the log, if the backend does not start listening within +# TIMEOUT_SECONDS (default 120). +# +# The port is cleared before launching. A backend which has crashed can still hold the bind while +# it is being torn down, and a replacement which cannot bind would sit out the whole start-up +# timeout and then be reported as a backend that failed to start, hiding the real reason. + +set -u + +# How long to wait for a dying backend to let go of the port before giving up on it. +PORT_FREE_TIMEOUT=30 +# How long to wait after SIGTERM before resorting to SIGKILL. +PORT_FREE_GRACE=5 + +src_dir=${1:-} +build_dir=${2:-} +port=${3:-} +log_file=${4:-} +timeout=${5:-120} + +if [ -z "$src_dir" ] || [ -z "$build_dir" ] || [ -z "$port" ] || [ -z "$log_file" ]; then + echo "usage: $0 SRC_DIR BUILD_DIR PORT LOG_FILE [TIMEOUT_SECONDS]" >&2 + exit 2 +fi + +# Resolved before the cd below, so that wait_for_backend.sh is found next to this script whatever +# directory the caller happens to be in. +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) + +# Matched on the executable and its --port argument rather than on "carta_backend" and the port +# number appearing anywhere in a command line. Apptainer shares the host PID space, so this scan +# also sees `apptainer exec ... /bin/bash -c ""` -- and that script mentions +# carta_backend in one line and the port in another, which `carta_backend.*$port` matches, since a +# POSIX regex crosses newlines. The loose pattern therefore matched the very process running this +# script, and clear_port killed the stage it was restarting the backend for. The stage loop has no +# --port in it, and no shell is named carta_backend, so requiring both makes only a real backend +# match. (macOS never showed this: its pgrep does not match across the newlines.) +backend_pattern="(^|[/[:space:]])carta_backend[[:space:]].*--port[[:space:]=]+$port([[:space:]]|\$)" + +# pgrep is not guaranteed to be in a container image, so fall back to plain ps. +backend_pids() { + if command -v pgrep >/dev/null 2>&1; then + pgrep -f "$backend_pattern" + else + # By field rather than by a regex over the whole line: the pattern would otherwise appear + # in awk's own command line, and awk would report the process doing the matching. + ps -eo pid=,args= 2>/dev/null | awk -v port="$port" ' + { + argv0 = 0 + for (i = 2; i <= NF; i++) + if ($i ~ /(^|\/)carta_backend$/) { argv0 = i; break } + if (argv0) + for (i = argv0 + 1; i <= NF; i++) + if (($i == "--port" && $(i + 1) == port) || $i == "--port=" port) { + print $1 + break + } + }' + fi +} + +port_is_busy() { + (exec 3<>"/dev/tcp/127.0.0.1/$port") 2>/dev/null +} + +clear_port() { + local pids waited=0 signalled='' + + pids=$(backend_pids) + if [ -n "$pids" ]; then + echo "stopping carta_backend still holding port $port (PID $(echo $pids | tr '\n' ' '))" + kill $pids 2>/dev/null + signalled=yes + fi + + while [ "$waited" -lt "$PORT_FREE_TIMEOUT" ]; do + if ! port_is_busy; then + return 0 + fi + # Something is still listening. Give a signalled backend a few seconds to exit on its own + # terms -- it may be writing the sanitizer report we want -- and only then force it. + if [ -n "$signalled" ] && [ "$waited" -eq "$PORT_FREE_GRACE" ]; then + pids=$(backend_pids) + if [ -n "$pids" ]; then + kill -9 $pids 2>/dev/null + fi + fi + sleep 1 + waited=$((waited + 1)) + done + + return 1 +} + +if ! clear_port; then + echo "port $port is still in use after ${PORT_FREE_TIMEOUT}s; not starting carta_backend" >&2 + exit 1 +fi + +if ! cd "$build_dir"; then + echo "no such build directory: $build_dir" >&2 + exit 1 +fi + +launch_backend() { + ASAN_OPTIONS=suppressions=$src_dir/debug/asan/myasan.supp \ + LSAN_OPTIONS=suppressions=$src_dir/debug/asan/myasan-leaks.supp \ + ASAN_SYMBOLIZER_PATH=llvm-symbolizer \ + ./carta_backend /images \ + --top_level_folder /images \ + --port "$port" \ + --omp_threads=4 \ + --debug_no_auth \ + --no_frontend \ + --no_database \ + --no_log \ + --verbosity=5 >> "$log_file" 2>&1 & + + backend_pid=$! +} + +max_attempts=3 +attempt=1 +attempt_timeout=$((timeout / max_attempts)) +if [ "$attempt_timeout" -lt 10 ]; then + max_attempts=1 + attempt_timeout=$timeout +fi + +while :; do + launch_backend + + # A PID only says the backend forked; the tests need the port to be bound. The log is left out + # here so that a retried attempt does not tail it once per try. + if bash "$script_dir/wait_for_backend.sh" "$port" "$attempt_timeout"; then + echo "carta_backend is listening on port $port with PID $backend_pid" + exit 0 + fi + + # Still running, just not listening yet: it is slow or wedged, and launching a second copy + # would only fight the first one for the port. + if kill -0 "$backend_pid" 2>/dev/null; then + break + fi + + # Gone already, so it never got as far as listening. The usual reason after a crash is that it + # could not bind: connections from the previous backend can sit in TIME_WAIT for a while after + # it has stopped accepting, which is invisible to a connect-based check. That clears by itself, + # so try again instead of spending the rest of the budget waiting on a process which has exited. + if [ "$attempt" -ge "$max_attempts" ]; then + break + fi + attempt=$((attempt + 1)) + echo "carta_backend exited before it started listening; retrying ($attempt of $max_attempts)" +done + +# Report the failure the same way every other check does, with the tail of the log attached. +bash "$script_dir/wait_for_backend.sh" "$port" 0 "$log_file" +exit 1 diff --git a/scripts/wait_for_backend.sh b/scripts/wait_for_backend.sh new file mode 100755 index 00000000..26c72cd6 --- /dev/null +++ b/scripts/wait_for_backend.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# +# Wait until carta_backend is accepting connections on a port. +# +# The CI actions used to take `pgrep` as proof that the backend was up, but pgrep matches as soon +# as the process has forked, which is well before it has bound the port -- and `pgrep ... | head` +# reports the exit status of head, so a backend which never started at all still looked like a +# success. The tests then opened the stage against a port nobody was listening on and reported it +# as a connection failure in the first test file, or, if the backend died part way through a +# stage, as the same failure in every file after it. +# +# Connecting is the only check which answers the question the tests actually ask, so that is what +# this does, using bash's own /dev/tcp redirection rather than a tool which may be missing from a +# container. +# +# Usage: wait_for_backend.sh PORT [TIMEOUT_SECONDS] [LOG_FILE] +# +# A timeout of 0 (the default) makes a single immediate check, which is how a stage asks whether +# the backend it started is still alive. LOG_FILE, if given, is tailed to stderr on failure, so +# that a crash is reported where the stage failed instead of only in an uploaded artifact. + +set -u + +port=${1:-} +timeout=${2:-0} +log_file=${3:-} + +if [ -z "$port" ]; then + echo "usage: $0 PORT [TIMEOUT_SECONDS] [LOG_FILE]" >&2 + exit 2 +fi + +deadline=$((SECONDS + timeout)) + +while :; do + # The subshell keeps the descriptor from leaking into the caller and turns a refused + # connection into an ordinary non-zero status rather than a message on stderr. + if (exec 3<>"/dev/tcp/127.0.0.1/$port") 2>/dev/null; then + exit 0 + fi + if [ "$SECONDS" -ge "$deadline" ]; then + break + fi + sleep 1 +done + +if [ "$timeout" -eq 0 ]; then + echo "carta_backend is not listening on port $port" >&2 +else + echo "carta_backend did not start listening on port $port within ${timeout}s" >&2 +fi + +if [ -n "$log_file" ] && [ -f "$log_file" ]; then + echo "--- last 100 lines of $log_file ---" >&2 + tail -n 100 "$log_file" >&2 +fi + +exit 1