Extract E2E Tests (every 4 hours) #712
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Extract E2E Tests (every 4 hours) | |
| on: | |
| schedule: | |
| - cron: "0 */4 * * *" | |
| workflow_dispatch: | |
| # Allows manual triggering | |
| inputs: | |
| environment: | |
| description: "Environment to run the tests in" | |
| required: false | |
| default: staging | |
| type: choice | |
| options: | |
| - staging | |
| - production | |
| notify_slack: | |
| description: "Notify Slack" | |
| required: false | |
| default: false | |
| type: boolean | |
| workflow_call: | |
| env: | |
| UV_VERSION: "0.7.20" | |
| PYTHON_VERSION: "3.12" | |
| SLACK_CHANNEL_ID: C078PHNTF44 # Extract channel ID | |
| API_E2E_LOG_PATH: ${{ github.workspace }}/extract-e2e.log | |
| jobs: | |
| extract-e2e: | |
| name: "Extract E2E Tests (${{ matrix.environment }})" | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.environment }} | |
| cancel-in-progress: true | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| environment: ${{ github.event_name == 'schedule' && fromJson('["staging", "production"]') || fromJson(format('["{0}"]', github.event.inputs.environment || 'staging')) }} | |
| steps: | |
| - name: Set runtime inputs | |
| id: runtime | |
| run: | | |
| environment=${{ matrix.environment }} | |
| notify_slack=${{ github.event.inputs.notify_slack || github.event_name == 'schedule' }} | |
| echo "environment=${environment}" >> $GITHUB_OUTPUT | |
| echo "notify_slack=${notify_slack}" >> $GITHUB_OUTPUT | |
| if [ "${environment}" = "production" ]; then | |
| echo "LLAMA_CLOUD_BASE_URL=https://api.cloud.llamaindex.ai" >> $GITHUB_ENV | |
| api_key_secret="${{ secrets.LLAMA_CLOUD_API_KEY }}" | |
| project_id_secret="${{ secrets.LLAMA_CLOUD_PROJECT_ID }}" | |
| else | |
| echo "LLAMA_CLOUD_BASE_URL=https://api.staging.llamaindex.ai" >> $GITHUB_ENV | |
| api_key_secret="${{ secrets.LLAMA_CLOUD_API_KEY_STAGING }}" | |
| project_id_secret="${{ secrets.LLAMA_CLOUD_PROJECT_ID_STAGING }}" | |
| fi | |
| if [ -n "$api_key_secret" ]; then | |
| echo "LLAMA_CLOUD_API_KEY=$api_key_secret" >> $GITHUB_ENV | |
| fi | |
| if [ -n "$project_id_secret" ]; then | |
| echo "LLAMA_CLOUD_PROJECT_ID=$project_id_secret" >> $GITHUB_ENV | |
| fi | |
| - uses: actions/checkout@v5 | |
| with: | |
| fetch-depth: 0 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v7 | |
| with: | |
| version: ${{ env.UV_VERSION }} | |
| - name: Set up Python | |
| run: uv python install ${{ env.PYTHON_VERSION }} && uv python pin ${{ env.PYTHON_VERSION }} | |
| - name: Run Extract E2E tests | |
| id: extract-tests | |
| continue-on-error: true | |
| working-directory: py | |
| run: | | |
| set -o pipefail | |
| rm -f "$API_E2E_LOG_PATH" | |
| uv run pytest -v -n 8 --timeout=300 --session-timeout=1740 tests/extract/ 2>&1 | tee "$API_E2E_LOG_PATH" | |
| - name: Extract pytest failure summary | |
| id: failed-tests | |
| if: steps.extract-tests.outcome == 'failure' || cancelled() | |
| run: | | |
| summary="$(python3 - <<'PY' | |
| import os | |
| import re | |
| from pathlib import Path | |
| log_path = Path(os.environ["API_E2E_LOG_PATH"]) | |
| if not log_path.exists(): | |
| print("Test log not found.") | |
| raise SystemExit(0) | |
| lines = log_path.read_text(errors="ignore").splitlines() | |
| # Find the "short test summary info" section | |
| start = None | |
| for i, line in enumerate(lines): | |
| if line.startswith("=") and "short test summary info" in line: | |
| start = i + 1 | |
| break | |
| if start is None: | |
| print("No test summary found.") | |
| raise SystemExit(0) | |
| # Extract just the FAILED/ERROR lines (test name + short reason) | |
| failed_tests = [] | |
| for line in lines[start:]: | |
| if line.startswith("="): | |
| break # End of section | |
| if line.startswith("FAILED ") or line.startswith("ERROR "): | |
| # Extract test name and truncate the error message | |
| match = re.match(r"(FAILED|ERROR) ([\w/:.\[\]_-]+)", line) | |
| if match: | |
| failed_tests.append(f"{match.group(1)}: {match.group(2)}") | |
| if failed_tests: | |
| print("\n".join(failed_tests[:20])) # Limit to 20 tests max | |
| else: | |
| print("No failed tests found in summary.") | |
| PY | |
| )" | |
| if [ -z "$summary" ]; then | |
| summary="Failed test summary not available. Review the full run logs." | |
| fi | |
| { | |
| printf 'summary<<EOF\n%s\nEOF\n' "$summary" | |
| } >> "$GITHUB_OUTPUT" | |
| - name: Check test results | |
| if: always() | |
| run: | | |
| if [ "${{ steps.extract-tests.outcome }}" == "failure" ]; then | |
| echo "Extract E2E tests failed" | |
| exit 1 | |
| fi | |
| - name: Post to Extract Slack channel | |
| id: slack | |
| if: (failure() || cancelled()) && steps.runtime.outputs.notify_slack == 'true' | |
| uses: slackapi/slack-github-action@v2.1.1 | |
| with: | |
| channel-id: ${{ env.SLACK_CHANNEL_ID }} | |
| slack-message: | | |
| :red_circle: *Extract E2E Failed* (${{ steps.runtime.outputs.environment }}) | |
| ``` | |
| ${{ steps.failed-tests.outputs.summary }} | |
| ``` | |
| <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Run> | |
| env: | |
| SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} |