Disable fail-fast for CI unit and integration tests
#995
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: | |
| - "main" | |
| pull_request: | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| env: | |
| develop_repo_path_suffix: "-develop/" | |
| stable_repo_path_suffix: "-stable/" | |
| HEROKU_DISABLE_AUTOUPDATE: 1 | |
| HATCHET_RETRIES: 3 | |
| IS_RUNNING_ON_CI: true | |
| HATCHET_APP_LIMIT: 300 | |
| HATCHET_EXPENSIVE_MODE: 1 | |
| HATCHET_BUILDPACK_BASE: https://github.com/heroku/heroku-buildpack-php | |
| HATCHET_BUILDPACK_BRANCH: ${{ github.head_ref || github.ref_name }} | |
| HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }} | |
| HEROKU_API_USER: ${{ secrets.HEROKU_API_USER }} | |
| GIT_HTTP_LOW_SPEED_LIMIT: 1000 | |
| GIT_HTTP_LOW_SPEED_TIME: 300 | |
| # ensure these are in sync with other relevant workflow files | |
| setup_php_php_version: "8.4" | |
| setup_php_composer_version: "2.9" | |
| setup_ruby_ruby_version: "4.0" | |
| jobs: | |
| unit-test: | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: ["ubuntu-22.04", "ubuntu-24.04"] | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| - name: Install Ruby and Bundler | |
| uses: ruby/setup-ruby@v1 | |
| with: | |
| bundler-cache: true | |
| ruby-version: "${{ env.setup_ruby_ruby_version }}" | |
| - name: Execute tests | |
| run: bundle exec rspec test/spec/cgroup_spec.rb | |
| container-test: | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| - name: Run buildpack using default app fixture | |
| run: make run | |
| integration-test: | |
| runs-on: ubuntu-24.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| stack: ["heroku-22", "heroku-24"] | |
| env: | |
| STACK: ${{ matrix.stack }} | |
| BLACKFIRE_CLIENT_ID: ${{ secrets.BLACKFIRE_CLIENT_ID }} | |
| BLACKFIRE_CLIENT_TOKEN: ${{ secrets.BLACKFIRE_CLIENT_TOKEN }} | |
| BLACKFIRE_SERVER_ID: ${{ secrets.BLACKFIRE_SERVER_ID }} | |
| BLACKFIRE_SERVER_TOKEN: ${{ secrets.BLACKFIRE_SERVER_TOKEN }} | |
| PARALLEL_TEST_RUNTIME_LOG: test/var/log/parallel_runtime_rspec.${{ matrix.stack }}.log | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| - name: Calculate stack identifier and suffix for platform repo URL | |
| id: platform-repo-url-inputs | |
| run: | | |
| # needs to be done dynamically this way; comparisons in dynamic expressions for some reason do not work for with: in the step below | |
| echo "stack-identifier=${STACK}-amd64" >> "$GITHUB_OUTPUT" | |
| echo -n "repo-path-suffix=" >> "$GITHUB_OUTPUT" | |
| if [[ $GITHUB_REF_TYPE != "tag" && $GITHUB_REF_NAME != "main" && $GITHUB_EVENT_NAME != 'pull_request' ]]; then | |
| echo "$develop_repo_path_suffix" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "$stable_repo_path_suffix" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Perform platform repo snapshot checks | |
| id: platform-repo-snapshot-calc | |
| uses: ./.github/actions/platform-repo-snapshot-calc | |
| with: | |
| stacks-list-for-shell-expansion: ${{ steps.platform-repo-url-inputs.outputs.stack-identifier }} | |
| repo-path-suffix: ${{ steps.platform-repo-url-inputs.outputs.repo-path-suffix }} | |
| - name: Fail early if snapshot URL is not present | |
| if: ${{ steps.platform-repo-snapshot-calc.outputs.snapshot-urls-check-outcome != 'success' }} | |
| run: | | |
| echo "::error title=Platform repo snapshot missing::URL not found: ${{ steps.platform-repo-snapshot-calc.outputs.snapshot-urls }}" | |
| exit 1 | |
| - name: Install Ruby and Bundler | |
| uses: ruby/setup-ruby@v1 | |
| with: | |
| bundler-cache: true | |
| ruby-version: "${{ env.setup_ruby_ruby_version }}" | |
| - name: Install PHP and Composer | |
| uses: shivammathur/setup-php@accd6127cb78bee3e8082180cb391013d204ef9f # 2.37.0 | |
| with: | |
| php-version: "${{ env.setup_php_php_version }}" | |
| tools: "composer:${{ env.setup_php_composer_version }}" | |
| - name: Install packages from requirements.txt (for some tests in platform_spec.rb that run mkrepo.sh which needs natsort) | |
| run: | | |
| export VIRTUAL_ENV=$HOME/.venv | |
| python3 -m venv "$VIRTUAL_ENV" | |
| export PATH=$VIRTUAL_ENV/bin:$PATH | |
| pip install -r requirements.txt | |
| echo "$VIRTUAL_ENV/bin" >> "$GITHUB_PATH" | |
| - name: Hatchet setup | |
| run: bundle exec hatchet ci:setup | |
| - name: Export HEROKU_PHP_PLATFORM_REPOSITORIES (since we are not building main or a tag or for a PR) | |
| # we intentionally do not export this for tags or main or PR runs, since we want to use the default snapshot behavior in bin/compile | |
| if: github.ref_type != 'tag' && github.ref_name != 'main' && github.event_name != 'pull_request' | |
| run: | | |
| env_line="HEROKU_PHP_PLATFORM_REPOSITORIES=- ${{ steps.platform-repo-snapshot-calc.outputs.snapshot-urls }}" | |
| echo "::notice title=Platform repositories overridden for branch CI run::Running with '${env_line}'" | |
| echo "$env_line" >> "$GITHUB_ENV" | |
| - name: Calculate number of parallel_rspec processes (half of num of lines in runtime log) | |
| run: echo "PARALLEL_TEST_PROCESSORS=$(( ($(cat "$PARALLEL_TEST_RUNTIME_LOG" | wc -l)+2-1)/2 ))" >> "$GITHUB_ENV" | |
| - name: Execute tests | |
| run: | | |
| shopt -s extglob | |
| bundle exec parallel_rspec --group-by runtime --first-is-1 --unknown-runtime 1 --allowed-missing 100 --runtime-log "$PARALLEL_TEST_RUNTIME_LOG" --verbose-command --combine-stderr --prefix-output-with-test-env-number test/spec/!(cgroup)_spec.rb | |
| - name: Clean up and (locally) store updated runtime log | |
| run: grep -E '^test/spec/[a-z0-9_/\.-]+\.rb:[0-9]+\.[0-9]+$' "${PARALLEL_TEST_RUNTIME_LOG}.last" | sort > "$PARALLEL_TEST_RUNTIME_LOG" | |
| - name: Upload runtime log as artifact (for analysis across runs) | |
| if: github.ref_name == 'main' | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: parallel-test-runtime-log-${{ matrix.stack }} | |
| path: ${{ env.PARALLEL_TEST_RUNTIME_LOG }} | |
| retention-days: 1 | |
| - name: Print test run details to summary | |
| if: ${{ !cancelled() }} | |
| run: | | |
| # from all RSpec JSON "logs", produce the input file name (1.json, 2.json...) and run duration and (-n)atural (-r)everse sort into an array via (-z)ero byte (-d)elimited list | |
| readarray -d '' groups < <(jq --raw-output0 '"\(input_filename):\(.summary.duration)"' test/var/log/*.json | sort -znr -t ":" -k 2) | |
| echo '## Executed test groups, from slowest to fastest' >> "$GITHUB_STEP_SUMMARY" | |
| for entry in "${groups[@]}"; do | |
| # get file name from "$name:$duration" list entry | |
| json=$(cut -d ":" -f1 <<<"$entry") | |
| # build the other file names for the same group number | |
| group=$(basename "$json" .json) | |
| log="$(dirname "$json")/${group}.txt" | |
| profile="$(dirname "$json")/${group}.profile" | |
| # remember if any examples failed/errored or may have been filtered out | |
| failed=$(jq -r 'if(.summary.failure_count + .summary.errors_outside_of_examples_count > 0) then "1" else empty end' "$json") | |
| filtered=$(jq -r 'if(.messages // [] | any(test("(?m)^(Run options:)?\\s+include\\s+{"))) then "1" else empty end' "$json") | |
| echo "### TEST GROUP ${group}" >> "$GITHUB_STEP_SUMMARY" | |
| if [[ "$failed" ]]; then | |
| echo "> [!CAUTION]" >> "$GITHUB_STEP_SUMMARY" | |
| echo "> Tests failed in this group." >> "$GITHUB_STEP_SUMMARY" | |
| echo >> "$GITHUB_STEP_SUMMARY" # "alert" blockquotes are finicky with whitespace around them | |
| # write out GHA error annotations for all failures in this group | |
| jq -r '.examples[] | [select(.status == "failed")] | unique_by("\(.file_path):\(.line_number)")[] | "::error file=\(.file_path | sub("^\\./"; "")),line=\(.line_number)::\(.exception.class)"' "$json" | |
| fi | |
| # print summary and list of executed spec files | |
| # file_path is the actual file with the test, which we do not want since we have a lot of _shared.rb base tests | |
| # we want to show the spec that got executed instead, which we need to extract from the id field (looks like "foo_spec.rb[1:1:1]") | |
| jq -r '"\(.summary_line) in **\(.summary.duration) seconds** from the following files:", (.examples | map(. + {spec_path: .id | sub("\\[.+"; "")}) | unique_by(.spec_path)[] | "- `\(.spec_path)`")' "$json" >> "$GITHUB_STEP_SUMMARY" | |
| if [[ "$filtered" ]]; then | |
| echo "> [!WARNING]" >> "$GITHUB_STEP_SUMMARY" | |
| echo "> An inclusion filter was applied to this group. " >> "$GITHUB_STEP_SUMMARY" | |
| echo "> Any files in this group without the same inclusion filter will have had examples filtered out as a result." >> "$GITHUB_STEP_SUMMARY" | |
| echo >> "$GITHUB_STEP_SUMMARY" # "alert" blockquotes are finicky with whitespace around them | |
| # write out GHA warning annotation for all files in this group | |
| jq -r '.examples | unique_by(.file_path)[] | "::warning file=\(.file_path | sub("^\\./"; ""))::Some or all examples may have been skipped due to an exclusion filter in this group."' "$json" | |
| fi | |
| # printf because we need a bunch of blank lines, notably after the closing summary and details tags | |
| # details block gets auto-expanded for any failed groups via the "open" attribute | |
| printf >> "$GITHUB_STEP_SUMMARY" \ | |
| '<details%s><summary>Full documentation style output</summary>\n\n```\n%s\n```\n</details>\n\n' \ | |
| "${failed:+" open"}" \ | |
| "$(<"$log")" | |
| printf >> "$GITHUB_STEP_SUMMARY" \ | |
| '<details><summary>Profiler output of slowest examples/groups</summary>\n\n```\n%s\n```\n</details>\n\n' \ | |
| "$(<"$profile")" | |
| done | |
| - name: Print runtime log to summary | |
| run: | | |
| # printf because we need a bunch of blank lines, notably after the closing summary and details tags | |
| printf >> "$GITHUB_STEP_SUMMARY" \ | |
| '## Runtime log for `parallel_test` balancing\n<details><summary><code>%s</code></summary>\n\n```\n%s\n```\n</details>\n\n' \ | |
| "$PARALLEL_TEST_RUNTIME_LOG" \ | |
| "$(<"$PARALLEL_TEST_RUNTIME_LOG")" | |
| merge-runtime-logs: | |
| runs-on: ubuntu-24.04 | |
| if: github.ref_name == 'main' | |
| needs: integration-test | |
| steps: | |
| - name: Merge all runtime logs into a single artifact | |
| uses: actions/upload-artifact/merge@v7 | |
| with: | |
| name: parallel-test-runtime-logs | |
| pattern: parallel-test-runtime-log-* | |
| delete-merged: true | |
| retention-days: 90 |