From ccceaddeffc63d9a01aeebb4c9914fe4ce4ea3d6 Mon Sep 17 00:00:00 2001 From: JCantu248 Date: Wed, 9 Apr 2025 09:41:24 -0500 Subject: [PATCH 01/26] Fix incorrect escape characters in configuration strings. --- .github/workflows/regression.yml | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index fc84f0d93..bc9ec0164 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -50,11 +50,11 @@ jobs: --cluster crossfeed-playwright-staging-cd-ecs-cluster \ --task-definition crossfeed-staging-cd-playwright-worker \ --launch-type FARGATE \ - --network-configuration 'awsvpcConfiguration={ - "subnets": ["${{ secrets.AWS_SUBNET }}"], - "securityGroups": ["${{ secrets.AWS_SECURITY_GROUP }}"], - "assignPublicIp": "ENABLED" - }' \ + --network-configuration "awsvpcConfiguration={ + \"subnets\": [\"${{ secrets.AWS_SUBNET }}\"], + \"securityGroups\": [\"${{ secrets.AWS_SECURITY_GROUP }}\"], + \"assignPublicIp\": \"ENABLED\" + }" \ --region ${{ secrets.AWS_REGION }} \ --overrides '{ "containerOverrides": [ @@ -63,8 +63,9 @@ jobs: "command": [ "sh", "-c", - "echo \"Running Playwright Tests\""; \ - cd /app/xfd/playwright && npx playwright test" + "echo \"Running Playwright Tests\"; \ + cd /app/xfd/playwright && \ + npx playwright test" ] } ] @@ -134,11 +135,11 @@ jobs: --cluster crossfeed-playwright-staging-cd-ecs-cluster \ --task-definition crossfeed-staging-cd-playwright-worker \ --launch-type FARGATE \ - --network-configuration 'awsvpcConfiguration={ - "subnets": ["${{ secrets.AWS_SUBNET }}"], - "securityGroups": ["${{ secrets.AWS_SECURITY_GROUP }}"], - "assignPublicIp": "ENABLED" - }' \ + --network-configuration "awsvpcConfiguration={ + \"subnets\": [\"${{ secrets.AWS_SUBNET }}\"], + \"securityGroups\": [\"${{ secrets.AWS_SECURITY_GROUP }}\"], + \"assignPublicIp\": \"ENABLED\" + }" \ --region ${{ secrets.AWS_REGION }} \ --overrides '{ "containerOverrides": [ @@ -147,8 +148,9 @@ jobs: "command": [ "sh", "-c", - "echo \"Running Playwright Tests\""; \ - cd /app/xfd/playwright && npx playwright test" + "echo \"Running Playwright Tests\"; \ + cd /app/xfd/playwright && \ + npx playwright test" ] } ] From bca40347ddc907c0b54cb00c0410aa8f62e73d87 Mon Sep 17 00:00:00 2001 From: JCantu248 Date: Fri, 11 Apr 2025 10:51:48 -0500 Subject: [PATCH 02/26] Spun the ECS task to its own bash script for ease of testing. --- .github/workflows/regression.yml | 170 +++++++++++------------------ playwright/run_playwright_tests.sh | 70 ++++++++++++ 2 files changed, 131 insertions(+), 109 deletions(-) create mode 100755 playwright/run_playwright_tests.sh diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index bc9ec0164..431a499a2 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -44,68 +44,45 @@ jobs: PW_XFD_2FA_SECRET: ${{ secrets.PW_XFD_2FA_SECRET }} PW_XFD_PASSWORD: ${{ secrets.PW_XFD_PASSWORD }} PW_XFD_USERNAME: ${{ secrets.PW_XFD_USERNAME }} + AUTOMATED_TEST_REPORTS_BUCKET_NAME: + ${{ secrets.AUTOMATED_TEST_REPORT_BUCKET_NAME }} + AWS_REGION: ${{ secrets.AWS_REGION }} + AWS_SUBNET: ${{ secrets.AWS_SUBNET }} + AWS_SECURITY_GROUP: ${{ secrets.AWS_SECURITY_GROUP }} run: | - # Start the ECS task and capture the task ARN - TASK_ARN=$(aws ecs run-task \ - --cluster crossfeed-playwright-staging-cd-ecs-cluster \ - --task-definition crossfeed-staging-cd-playwright-worker \ - --launch-type FARGATE \ - --network-configuration "awsvpcConfiguration={ - \"subnets\": [\"${{ secrets.AWS_SUBNET }}\"], - \"securityGroups\": [\"${{ secrets.AWS_SECURITY_GROUP }}\"], - \"assignPublicIp\": \"ENABLED\" - }" \ - --region ${{ secrets.AWS_REGION }} \ - --overrides '{ - "containerOverrides": [ - { - "name": "main", - "command": [ - "sh", - "-c", - "echo \"Running Playwright Tests\"; \ - cd /app/xfd/playwright && \ - npx playwright test" - ] - } - ] - }' \ - --query 'tasks[0].taskArn' --output text) - - echo "Started ECS Task with ARN: $TASK_ARN" - - # Wait for the ECS task to complete - aws ecs wait tasks-stopped \ - --cluster crossfeed-playwright-staging-cd-ecs-cluster\ - --tasks $TASK_ARN --region ${{ secrets.AWS_REGION }} - - echo "ECS task completed. Continuing with the next steps." + chmod +x run_playwright_tests.sh + CLUSTER_NAME=crossfeed-playwright-staging-cd-ecs-cluster \ + TASK_DEFINITION=crossfeed-staging-cd-playwright-worker \ + AUTOMATED_TEST_RESULTS_BUCKET_NAME=$AUTOMATED_TEST_REPORT_BUCKET_NAME \ + AWS_SUBNET=$AWS_SUBNET \ + AWS_SECURITY_GROUP=$AWS_SECURITY_GROUP \ + ./run_playwright_tests.sh continue-on-error: false # Ensure it stops if the ECS task fails - - name: Upload test results as artifact + - name: Download test report from S3 + run: | + mkdir -p ./playwright-report/html + aws s3 cp \ + s3://$AUTOMATED_TEST_REPORTS_BUCKET_NAME/playwright-reports/html/ \ + ./playwright-report/html/ --recursive + + aws s3 cp \ + s3://$AUTOMATED_TEST_REPORTS_BUCKET_NAME/playwright-reports/results.json \ + ./playwright-report/results.json + + - name: Upload Playwright HTML report uses: actions/upload-artifact@v4 with: - name: playwright-test-results - path: ./test-results/*.json + name: playwright-html-report + path: ./playwright-report/html/ - - name: Upload HTML report to S3 - run: | - aws s3 cp ./playwright-report/html \ - s3://${{ vars.AUTOMATED_TEST_REPORT_BUCKET_NAME }}/\ - playwright-reports/html/ --recursive || \ - { echo "HTML report upload failed"; exit 1; } + - name: Upload Playwright results JSON + uses: actions/upload-artifact@v4 + with: + name: playwright-results-json + path: ./playwright-report/results.json - - name: Upload JSON report to S3 - run: | - if aws s3 cp ./playwright-report/results.json \ - s3://${{ vars.AUTOMATED_TEST_REPORT_BUCKET_NAME }}/\ - playwright-reports/results.json; then \ - echo "JSON report uploaded successfully."; \ - else \ - echo "Failed to upload JSON report to S3." >&2; \ - exit 1; \ - fi test_integration: runs-on: ubuntu-latest @@ -129,65 +106,40 @@ jobs: PW_XFD_2FA_SECRET: ${{ secrets.PW_XFD_2FA_SECRET }} PW_XFD_PASSWORD: ${{ secrets.PW_XFD_PASSWORD }} PW_XFD_USERNAME: ${{ secrets.PW_XFD_USERNAME }} + AUTOMATED_TEST_REPORTS_BUCKET_NAME: + ${{ secrets.AUTOMATED_TEST_REPORT_BUCKET_NAME }} + AWS_REGION: ${{ secrets.AWS_REGION }} run: | - # Start the ECS task and capture the task ARN - TASK_ARN=$(aws ecs run-task \ - --cluster crossfeed-playwright-staging-cd-ecs-cluster \ - --task-definition crossfeed-staging-cd-playwright-worker \ - --launch-type FARGATE \ - --network-configuration "awsvpcConfiguration={ - \"subnets\": [\"${{ secrets.AWS_SUBNET }}\"], - \"securityGroups\": [\"${{ secrets.AWS_SECURITY_GROUP }}\"], - \"assignPublicIp\": \"ENABLED\" - }" \ - --region ${{ secrets.AWS_REGION }} \ - --overrides '{ - "containerOverrides": [ - { - "name": "main", - "command": [ - "sh", - "-c", - "echo \"Running Playwright Tests\"; \ - cd /app/xfd/playwright && \ - npx playwright test" - ] - } - ] - }' \ - --query 'tasks[0].taskArn' --output text) - - echo "Started ECS Task with ARN: $TASK_ARN" - - # Wait for the ECS task to complete - aws ecs wait tasks-stopped \ - --cluster crossfeed-playwright-staging-cd-ecs-cluster\ - --tasks $TASK_ARN --region ${{ secrets.AWS_REGION }} - - echo "ECS task completed. Continuing with the next steps." + chmod +x run_playwright_tests.sh + CLUSTER_NAME=crossfeed-playwright-integration-ecs-cluster \ + TASK_DEFINITION=crossfeed-integration-playwright-worker \ + AUTOMATED_TEST_RESULTS_BUCKET_NAME=$AUTOMATED_TEST_REPORT_BUCKET_NAME \ + AWS_SUBNET=$AWS_SUBNET \ + AWS_SECURITY_GROUP=$AWS_SECURITY_GROUP \ + ./run_playwright_tests.sh + continue-on-error: false # Ensure it stops if the ECS task fails - - name: Upload test results as artifact + - name: Download test report from S3 + run: | + mkdir -p ./playwright-report/html + aws s3 cp \ + s3://$AUTOMATED_TEST_REPORTS_BUCKET_NAME/playwright-reports/html/ \ + ./playwright-report/html/ --recursive + + aws s3 cp \ + s3://$AUTOMATED_TEST_REPORTS_BUCKET_NAME/playwright-reports/results.json \ + ./playwright-report/results.json + + - name: Upload Playwright HTML report uses: actions/upload-artifact@v4 with: - name: playwright-test-results - path: ./test-results/*.json - - - name: Upload HTML report to S3 - run: | - aws s3 cp ./playwright-report/ \ - s3://${{ vars.AUTOMATED_TEST_REPORT_BUCKET_NAME }}/\ - playwright-reports/html/ --recursive || \ - { echo "HTML report upload failed"; exit 1; } + name: playwright-html-report + path: ./playwright-report/html/ - - name: Upload JSON report to S3 - run: | - if aws s3 cp ./playwright-report/results.json \ - s3://${{ vars.AUTOMATED_TEST_REPORT_BUCKET_NAME }}/\ - playwright-reports/results.json; then \ - echo "JSON report uploaded successfully."; \ - else \ - echo "Failed to upload JSON report to S3." >&2; \ - exit 1; \ - fi + - name: Upload Playwright results JSON + uses: actions/upload-artifact@v4 + with: + name: playwright-results-json + path: ./playwright-report/results.json diff --git a/playwright/run_playwright_tests.sh b/playwright/run_playwright_tests.sh new file mode 100755 index 000000000..836b1b8b5 --- /dev/null +++ b/playwright/run_playwright_tests.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# Get the current datetime (e.g., 2025-04-10T12:34:56) +DATETIME=$(date +%Y-%m-%dT%H:%M:%S) + +OVERRIDES=$( + jq -n \ + --arg datetime "$DATETIME" \ + --arg bucket "$AUTOMATED_TEST_REPORTS_BUCKET_NAME" \ + --arg region "$AWS_REGION" \ + '{ + "containerOverrides": [ + { + "name": "main", + "environment": [ + { "name": "DATETIME", "value": $datetime }, + { "name": "AUTOMATED_TEST_REPORTS_BUCKET_NAME", "value": $bucket }, + { "name": "AWS_REGION", "value": $region } + ], + "command": [ + "sh", + "-c", + "echo \"Running Playwright Tests\" && cd /app/xfd/playwright && npx playwright test && echo \"Uploading to S3\" && aws s3 cp ./playwright-report/html s3://$AUTOMATED_TEST_REPORTS_BUCKET_NAME/playwright-reports/$DATETIME/html/ --recursive --region $AWS_REGION && aws s3 cp ./playwright-report/results.json s3://$AUTOMATED_TEST_REPORTS_BUCKET_NAME/playwright-reports/$DATETIME/results.json --region $AWS_REGION && echo \"Test results uploaded successfully to S3 at $DATETIME\"" + ] + } + ] + }' +) + +echo "Starting ECS task to run Playwright tests..." + +TASK_ARN=$(aws ecs run-task \ + --cluster "$CLUSTER_NAME" \ + --task-definition "$TASK_DEFINITION" \ + --launch-type FARGATE \ + --network-configuration "{ + \"awsvpcConfiguration\":{ + \"subnets\": [\"${AWS_SUBNET}\"], + \"securityGroups\": [\"${AWS_SECURITY_GROUP}\"], + \"assignPublicIp\": \"ENABLED\" + } + }" \ + --region "${AWS_REGION}" \ + --overrides "$OVERRIDES" \ + --query 'tasks[0].taskArn' \ + --output text 2> /dev/null) || { + echo "❌ Failed to run ECS task (aws command error)." >&2 + exit 1 +} + +# Sanity check: ensure the result isn't empty +if [[ -z "$TASK_ARN" || "$TASK_ARN" == "None" ]]; then + echo "❌ Failed to run ECS task. No ARN returned." >&2 + exit 1 +fi + +echo "Started ECS Task with ARN: $TASK_ARN" + +echo "Waiting for ECS task to complete..." +aws ecs wait tasks-stopped \ + --cluster "$CLUSTER_NAME" \ + --tasks "$TASK_ARN" \ + --region "${AWS_REGION}" || { + echo "❌ Task did not complete successfully." >&2 + exit 1 +} + +echo "ECS task $TASK_ARN completed. Uploading test results to S3..." From 5d19e6b05346cdafc67eb5aa849934a02caa99a2 Mon Sep 17 00:00:00 2001 From: JCantu248 Date: Wed, 16 Apr 2025 14:42:23 -0500 Subject: [PATCH 03/26] Debugging bash script connecting to Fargate. --- .github/workflows/regression.yml | 4 ++++ playwright/global-setup.ts | 2 +- playwright/run_playwright_tests.sh | 37 ++++++++++++++++++++++++++---- 3 files changed, 38 insertions(+), 5 deletions(-) diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 431a499a2..87450b530 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -56,6 +56,10 @@ jobs: AUTOMATED_TEST_RESULTS_BUCKET_NAME=$AUTOMATED_TEST_REPORT_BUCKET_NAME \ AWS_SUBNET=$AWS_SUBNET \ AWS_SECURITY_GROUP=$AWS_SECURITY_GROUP \ + PW_XFD_2FA_ISSUER=$PW_XFD_2FA_ISSUER \ + PW_XFD_2FA_SECRET=$PW_XFD_2FA_SECRET \ + PW_XFD_PASSWORD=$PW_XFD_PASSWORD \ + PW_XFD_USERNAME=$PW_XFD_USERNAME \ ./run_playwright_tests.sh continue-on-error: false # Ensure it stops if the ECS task fails diff --git a/playwright/global-setup.ts b/playwright/global-setup.ts index 70995bbf2..fe5b1c09e 100644 --- a/playwright/global-setup.ts +++ b/playwright/global-setup.ts @@ -17,7 +17,7 @@ let totp = new OTPAuth.TOTP({ const axios = require('axios'); -const waitForFrontend = async (url, timeout = 900000, checkInterval = 5000) => { +const waitForFrontend = async (url, timeout = 120000, checkInterval = 5000) => { console.log(`Url: ${url}`); const startTime = Date.now(); while (Date.now() - startTime < timeout) { diff --git a/playwright/run_playwright_tests.sh b/playwright/run_playwright_tests.sh index 836b1b8b5..c6bce54c0 100755 --- a/playwright/run_playwright_tests.sh +++ b/playwright/run_playwright_tests.sh @@ -1,15 +1,26 @@ #!/usr/bin/env bash set -euo pipefail +set -x # Get the current datetime (e.g., 2025-04-10T12:34:56) DATETIME=$(date +%Y-%m-%dT%H:%M:%S) +echo "📅 Test timestamp: $DATETIME" +echo "📦 Bucket: $AUTOMATED_TEST_REPORTS_BUCKET_NAME" +echo "🌎 Region: $AWS_REGION" +echo "📤 Uploading to: s3://$AUTOMATED_TEST_REPORTS_BUCKET_NAME/playwright-reports/$DATETIME/" OVERRIDES=$( jq -n \ --arg datetime "$DATETIME" \ --arg bucket "$AUTOMATED_TEST_REPORTS_BUCKET_NAME" \ --arg region "$AWS_REGION" \ + --arg url "$PW_XFD_URL" \ + --arg username "$PW_XFD_USERNAME" \ + --arg password "$PW_XFD_PASSWORD" \ + --arg otpsecret "$PW_XFD_2FA_SECRET" \ + --arg s3HtmlPath "s3://$AUTOMATED_TEST_REPORTS_BUCKET_NAME/playwright-reports/$DATETIME/html/" \ + --arg s3JsonPath "s3://$AUTOMATED_TEST_REPORTS_BUCKET_NAME/playwright-reports/$DATETIME/results.json" \ '{ "containerOverrides": [ { @@ -17,12 +28,30 @@ OVERRIDES=$( "environment": [ { "name": "DATETIME", "value": $datetime }, { "name": "AUTOMATED_TEST_REPORTS_BUCKET_NAME", "value": $bucket }, - { "name": "AWS_REGION", "value": $region } + { "name": "AWS_REGION", "value": $region }, + { "name": "PW_XFD_URL", "value": $url }, + { "name": "PW_XFD_USERNAME", "value": $username }, + { "name": "PW_XFD_PASSWORD", "value": $password }, + { "name": "PW_XFD_2FA_SECRET", "value": $otpsecret } ], "command": [ "sh", "-c", - "echo \"Running Playwright Tests\" && cd /app/xfd/playwright && npx playwright test && echo \"Uploading to S3\" && aws s3 cp ./playwright-report/html s3://$AUTOMATED_TEST_REPORTS_BUCKET_NAME/playwright-reports/$DATETIME/html/ --recursive --region $AWS_REGION && aws s3 cp ./playwright-report/results.json s3://$AUTOMATED_TEST_REPORTS_BUCKET_NAME/playwright-reports/$DATETIME/results.json --region $AWS_REGION && echo \"Test results uploaded successfully to S3 at $DATETIME\"" + "echo \"Cloning Playwright tests from GitHub...\" && + git clone https://github.com/cisagov/xfd.git /app/xfd && + cd /app/xfd/playwright && + echo \"Installing Node dependencies...\" && + npm install && + echo \"Installing Playwright...\" && + npx playwright install --with-deps && + echo \"Running Playwright Tests\" && + npx playwright test && + echo \"📤 Uploading test results to S3...\" && + echo \"Uploading HTML report to: ${s3HtmlPath}\" && + aws s3 cp ./playwright-report/html ${s3HtmlPath} --recursive --region $region && + echo \"Uploading JSON report to: ${s3JsonPath}\" && + aws s3 cp ./playwright-report/results.json ${s3JsonPath} --region $region && + echo \"✅ Test results uploaded successfully to S3 at $datetime\"" ] } ] @@ -57,8 +86,8 @@ if [[ -z "$TASK_ARN" || "$TASK_ARN" == "None" ]]; then fi echo "Started ECS Task with ARN: $TASK_ARN" - echo "Waiting for ECS task to complete..." + aws ecs wait tasks-stopped \ --cluster "$CLUSTER_NAME" \ --tasks "$TASK_ARN" \ @@ -67,4 +96,4 @@ aws ecs wait tasks-stopped \ exit 1 } -echo "ECS task $TASK_ARN completed. Uploading test results to S3..." +echo "ECS task $TASK_ARN completed successfully." From 3ee619e33fb539d282ce462ce2c1a878f19571d2 Mon Sep 17 00:00:00 2001 From: JCantu248 Date: Wed, 16 Apr 2025 14:55:43 -0500 Subject: [PATCH 04/26] Testing for errors in frontend listener in test suite. --- playwright/global-setup.ts | 5 +++-- playwright/run_playwright_tests.sh | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/playwright/global-setup.ts b/playwright/global-setup.ts index fe5b1c09e..d0b266884 100644 --- a/playwright/global-setup.ts +++ b/playwright/global-setup.ts @@ -17,7 +17,7 @@ let totp = new OTPAuth.TOTP({ const axios = require('axios'); -const waitForFrontend = async (url, timeout = 120000, checkInterval = 5000) => { +const waitForFrontend = async (url, timeout = 60000, checkInterval = 5000) => { console.log(`Url: ${url}`); const startTime = Date.now(); while (Date.now() - startTime < timeout) { @@ -33,7 +33,8 @@ const waitForFrontend = async (url, timeout = 120000, checkInterval = 5000) => { `Frontend not ready yet. Status: ${error.response.status}. Retrying...` ); } else { - console.log('Error occurred while checking frontend:', error.message); + console.log('Error occurred while checking frontend:', error); + break; } await new Promise((resolve) => setTimeout(resolve, checkInterval)); // Wait before retrying diff --git a/playwright/run_playwright_tests.sh b/playwright/run_playwright_tests.sh index c6bce54c0..7c3021714 100755 --- a/playwright/run_playwright_tests.sh +++ b/playwright/run_playwright_tests.sh @@ -38,7 +38,7 @@ OVERRIDES=$( "sh", "-c", "echo \"Cloning Playwright tests from GitHub...\" && - git clone https://github.com/cisagov/xfd.git /app/xfd && + git clone -b CRASM-2286_Bug_fix_regression_yaml https://github.com/cisagov/xfd.git /app/xfd && cd /app/xfd/playwright && echo \"Installing Node dependencies...\" && npm install && From ab094937f5774b619d8d79176c3144dfc7e279a2 Mon Sep 17 00:00:00 2001 From: JCantu248 Date: Wed, 16 Apr 2025 14:56:29 -0500 Subject: [PATCH 05/26] Just omit listener check from playwright tests for now while testing. --- playwright/global-setup.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playwright/global-setup.ts b/playwright/global-setup.ts index d0b266884..410e1a937 100644 --- a/playwright/global-setup.ts +++ b/playwright/global-setup.ts @@ -50,7 +50,7 @@ async function globalSetup(config: FullConfig) { const page = await browser.newPage(); //Log in with credentials. - await waitForFrontend(process.env.PW_XFD_URL); + //await waitForFrontend(process.env.PW_XFD_URL); await page.goto(String(process.env.PW_XFD_URL)); await page.getByTestId('button').click(); await page From 3f2330aa21181b517136f209cb44d7593dea7f5b Mon Sep 17 00:00:00 2001 From: JCantu248 Date: Fri, 18 Apr 2025 14:51:26 -0500 Subject: [PATCH 06/26] Console logs for debugging environment variables. --- playwright/global-setup.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/playwright/global-setup.ts b/playwright/global-setup.ts index 410e1a937..1d97ef71d 100644 --- a/playwright/global-setup.ts +++ b/playwright/global-setup.ts @@ -51,6 +51,8 @@ async function globalSetup(config: FullConfig) { //Log in with credentials. //await waitForFrontend(process.env.PW_XFD_URL); + console.log(`URL: ${process.env.PW_XFD_URL}`); + console.log(`Username: ${process.env.PW_XFD_USERNAME}`); await page.goto(String(process.env.PW_XFD_URL)); await page.getByTestId('button').click(); await page From 279fe549a23d7dcaad59bb1a6d74ba08e634908b Mon Sep 17 00:00:00 2001 From: JCantu248 Date: Wed, 23 Apr 2025 19:12:08 -0500 Subject: [PATCH 07/26] Add login environment variable temporarily to test around the Okta login bug --- playwright/global-setup.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playwright/global-setup.ts b/playwright/global-setup.ts index d0b266884..957d0b515 100644 --- a/playwright/global-setup.ts +++ b/playwright/global-setup.ts @@ -55,7 +55,7 @@ async function globalSetup(config: FullConfig) { await page.getByTestId('button').click(); await page .getByLabel('Username (Email)') - .fill(String(process.env.PW_XFD_USERNAME)); + .fill(String(process.env.PW_XFD_LOGIN)); await page.getByRole('button', { name: 'Next' }).click(); await page .getByLabel('Email address') From f5f9881553eec6c2781bcacc97d2b1195e33382a Mon Sep 17 00:00:00 2001 From: JCantu248 Date: Tue, 29 Apr 2025 12:28:40 -0500 Subject: [PATCH 08/26] Added an autoconfiguration feature for different environment configurations. --- .github/workflows/regression.yml | 19 ++++++++++++++++--- docker-compose.yml | 1 + playwright/global-setup.ts | 13 ++++++++++--- playwright/playwright.config.ts | 5 +++-- playwright/run_playwright_tests.sh | 8 +++++++- 5 files changed, 37 insertions(+), 9 deletions(-) diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 87450b530..5267047ed 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -33,6 +33,9 @@ jobs: with: node-version: 18 + - name: Set environment variables for staging + run: echo "ENVIRONMENT=github" >> $GITHUB_ENV + - name: Install dependencies run: npm ci @@ -40,12 +43,15 @@ jobs: env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + PW_XFD_URL: ${{ vars.PW_XFD_URL }} PW_XFD_2FA_ISSUER: ${{ secrets.PW_XFD_2FA_ISSUER }} PW_XFD_2FA_SECRET: ${{ secrets.PW_XFD_2FA_SECRET }} PW_XFD_PASSWORD: ${{ secrets.PW_XFD_PASSWORD }} PW_XFD_USERNAME: ${{ secrets.PW_XFD_USERNAME }} + PW_ECS: true + GIT_BRANCH: integration AUTOMATED_TEST_REPORTS_BUCKET_NAME: - ${{ secrets.AUTOMATED_TEST_REPORT_BUCKET_NAME }} + ${{ vars.AUTOMATED_TEST_REPORT_BUCKET_NAME }} AWS_REGION: ${{ secrets.AWS_REGION }} AWS_SUBNET: ${{ secrets.AWS_SUBNET }} AWS_SECURITY_GROUP: ${{ secrets.AWS_SECURITY_GROUP }} @@ -59,7 +65,6 @@ jobs: PW_XFD_2FA_ISSUER=$PW_XFD_2FA_ISSUER \ PW_XFD_2FA_SECRET=$PW_XFD_2FA_SECRET \ PW_XFD_PASSWORD=$PW_XFD_PASSWORD \ - PW_XFD_USERNAME=$PW_XFD_USERNAME \ ./run_playwright_tests.sh continue-on-error: false # Ensure it stops if the ECS task fails @@ -99,6 +104,9 @@ jobs: with: node-version: 18 + - name: Set environment variables for staging + run: echo "ENVIRONMENT=github" >> $GITHUB_ENV + - name: Install dependencies run: npm ci @@ -106,13 +114,18 @@ jobs: env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + PW_XFD_URL: ${{ vars.PW_XFD_URL }} PW_XFD_2FA_ISSUER: ${{ secrets.PW_XFD_2FA_ISSUER }} PW_XFD_2FA_SECRET: ${{ secrets.PW_XFD_2FA_SECRET }} PW_XFD_PASSWORD: ${{ secrets.PW_XFD_PASSWORD }} PW_XFD_USERNAME: ${{ secrets.PW_XFD_USERNAME }} + PW_ECS: true + GIT_BRANCH: integration AUTOMATED_TEST_REPORTS_BUCKET_NAME: - ${{ secrets.AUTOMATED_TEST_REPORT_BUCKET_NAME }} + ${{ vars.AUTOMATED_TEST_REPORT_BUCKET_NAME }} AWS_REGION: ${{ secrets.AWS_REGION }} + AWS_SUBNET: ${{ secrets.AWS_SUBNET }} + AWS_SECURITY_GROUP: ${{ secrets.AWS_SECURITY_GROUP }} run: | chmod +x run_playwright_tests.sh CLUSTER_NAME=crossfeed-playwright-integration-ecs-cluster \ diff --git a/docker-compose.yml b/docker-compose.yml index 6c6b83737..ae53a7951 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -167,6 +167,7 @@ services: - PW_XFD_URL=${PW_XFD_URL} - PW_XFD_USER_ROLE=${PW_XFD_USER_ROLE} - PW_XFD_USERNAME=${PW_XFD_USERNAME} + - PW_DOCKER=true command: > /bin/bash -c "npm ci && npx playwright install --with-deps && npx playwright test" diff --git a/playwright/global-setup.ts b/playwright/global-setup.ts index 957d0b515..5012f83c3 100644 --- a/playwright/global-setup.ts +++ b/playwright/global-setup.ts @@ -1,6 +1,7 @@ import { chromium, FullConfig } from '@playwright/test'; import * as OTPAuth from 'otpauth'; import * as dotenv from 'dotenv'; +import { determineUrl } from './utils/env'; // ✅ Now this works dotenv.config(); @@ -18,7 +19,6 @@ let totp = new OTPAuth.TOTP({ const axios = require('axios'); const waitForFrontend = async (url, timeout = 60000, checkInterval = 5000) => { - console.log(`Url: ${url}`); const startTime = Date.now(); while (Date.now() - startTime < timeout) { try { @@ -46,12 +46,19 @@ const waitForFrontend = async (url, timeout = 60000, checkInterval = 5000) => { }; async function globalSetup(config: FullConfig) { + const baseUrl = determineUrl(); + console.log(`Base URL: ${baseUrl}`); + if (!baseUrl) { + throw new Error( + '❌ PW_XFD_URL is not defined. Make sure it is set as an environment variable.' + ); + } const browser = await chromium.launch(); const page = await browser.newPage(); //Log in with credentials. - await waitForFrontend(process.env.PW_XFD_URL); - await page.goto(String(process.env.PW_XFD_URL)); + await waitForFrontend(baseUrl); + await page.goto(baseUrl); await page.getByTestId('button').click(); await page .getByLabel('Username (Email)') diff --git a/playwright/playwright.config.ts b/playwright/playwright.config.ts index be02e5b23..7b4c548ab 100644 --- a/playwright/playwright.config.ts +++ b/playwright/playwright.config.ts @@ -1,5 +1,6 @@ import { defineConfig, devices } from '@playwright/test'; import dotenv from 'dotenv'; +import { determineUrl, determineHeadless } from './utils/env'; /** * Read environment variables from file. @@ -33,8 +34,8 @@ export default defineConfig({ /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ use: { /* Base URL to use in actions like `await page.goto('/')`. */ - baseURL: process.env.PW_XFD_URL, - headless: true, + baseURL: determineUrl(), + headless: determineHeadless(), storageState: 'storageState.json', /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ diff --git a/playwright/run_playwright_tests.sh b/playwright/run_playwright_tests.sh index 7c3021714..c64976f6b 100755 --- a/playwright/run_playwright_tests.sh +++ b/playwright/run_playwright_tests.sh @@ -19,6 +19,9 @@ OVERRIDES=$( --arg username "$PW_XFD_USERNAME" \ --arg password "$PW_XFD_PASSWORD" \ --arg otpsecret "$PW_XFD_2FA_SECRET" \ + --arg login "$PW_XFD_LOGIN" \ + --arg pw_ecs "$PW_ECS" \ + --arg git_branch "$GIT_BRANCH" \ --arg s3HtmlPath "s3://$AUTOMATED_TEST_REPORTS_BUCKET_NAME/playwright-reports/$DATETIME/html/" \ --arg s3JsonPath "s3://$AUTOMATED_TEST_REPORTS_BUCKET_NAME/playwright-reports/$DATETIME/results.json" \ '{ @@ -32,7 +35,10 @@ OVERRIDES=$( { "name": "PW_XFD_URL", "value": $url }, { "name": "PW_XFD_USERNAME", "value": $username }, { "name": "PW_XFD_PASSWORD", "value": $password }, - { "name": "PW_XFD_2FA_SECRET", "value": $otpsecret } + { "name": "PW_XFD_2FA_SECRET", "value": $otpsecret }, + { "name": "PW_XFD_LOGIN", "value": $login }, + { "name": "PW_ECS", "value": "true" }, + { "name": "GIT_BRANCH", "value": $git_branch } ], "command": [ "sh", From f93d65c89b28889cc958469c15b89e47c2bb04ab Mon Sep 17 00:00:00 2001 From: JCantu248 Date: Tue, 29 Apr 2025 12:36:29 -0500 Subject: [PATCH 09/26] Adding utils/env.ts --- playwright/utils/env.ts | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 playwright/utils/env.ts diff --git a/playwright/utils/env.ts b/playwright/utils/env.ts new file mode 100644 index 000000000..8094ad51d --- /dev/null +++ b/playwright/utils/env.ts @@ -0,0 +1,28 @@ +// utils/env.ts +export function determineUrl(): string { + const branch = process.env.GIT_BRANCH || ''; + const insideDocker = process.env.PW_DOCKER === 'true'; + const insideECS = process.env.PW_ECS; + + console.log( + `Inside Docker: ${insideDocker}, Inside ECS: ${insideECS}, Branch: ${branch}` + ); + if (insideECS) { + if (branch === 'integration') { + return 'https://integration.crossfeed.cyber.dhs.gov'; + } + return 'https://staging-cd.crossfeed.cyber.dhs.gov'; + } + + if (insideDocker) { + return 'http://xfd-frontend-1:3000'; + } + + return 'http://localhost'; +} + +export function determineHeadless(): boolean { + const insideDocker = process.env.PW_DOCKER === 'true'; + const insideECS = process.env.PW_ECS === 'true' || !!process.env.AWS_REGION; + return insideDocker || insideECS; +} From b353990573977a36643ee993fc4c937469f40a03 Mon Sep 17 00:00:00 2001 From: JCantu248 Date: Tue, 29 Apr 2025 12:43:48 -0500 Subject: [PATCH 10/26] Remove debug logging statements. --- playwright/utils/env.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/playwright/utils/env.ts b/playwright/utils/env.ts index 8094ad51d..9f211d5da 100644 --- a/playwright/utils/env.ts +++ b/playwright/utils/env.ts @@ -4,9 +4,6 @@ export function determineUrl(): string { const insideDocker = process.env.PW_DOCKER === 'true'; const insideECS = process.env.PW_ECS; - console.log( - `Inside Docker: ${insideDocker}, Inside ECS: ${insideECS}, Branch: ${branch}` - ); if (insideECS) { if (branch === 'integration') { return 'https://integration.crossfeed.cyber.dhs.gov'; From ba9bda9909c2629ccb7153361e824c83f9797a95 Mon Sep 17 00:00:00 2001 From: JCantu248 Date: Thu, 1 May 2025 12:23:54 -0500 Subject: [PATCH 11/26] Simplify logic for determining URL for respective environments. --- .github/workflows/regression.yml | 2 -- docker-compose.yml | 3 +-- playwright/run_playwright_tests.sh | 2 -- playwright/utils/env.ts | 22 +++------------------- 4 files changed, 4 insertions(+), 25 deletions(-) diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 5267047ed..4f0321534 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -48,7 +48,6 @@ jobs: PW_XFD_2FA_SECRET: ${{ secrets.PW_XFD_2FA_SECRET }} PW_XFD_PASSWORD: ${{ secrets.PW_XFD_PASSWORD }} PW_XFD_USERNAME: ${{ secrets.PW_XFD_USERNAME }} - PW_ECS: true GIT_BRANCH: integration AUTOMATED_TEST_REPORTS_BUCKET_NAME: ${{ vars.AUTOMATED_TEST_REPORT_BUCKET_NAME }} @@ -119,7 +118,6 @@ jobs: PW_XFD_2FA_SECRET: ${{ secrets.PW_XFD_2FA_SECRET }} PW_XFD_PASSWORD: ${{ secrets.PW_XFD_PASSWORD }} PW_XFD_USERNAME: ${{ secrets.PW_XFD_USERNAME }} - PW_ECS: true GIT_BRANCH: integration AUTOMATED_TEST_REPORTS_BUCKET_NAME: ${{ vars.AUTOMATED_TEST_REPORT_BUCKET_NAME }} diff --git a/docker-compose.yml b/docker-compose.yml index ae53a7951..016cc21b4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -164,10 +164,9 @@ services: - PW_XFD_2FA_ISSUER=${PW_XFD_2FA_ISSUER} - PW_XFD_2FA_SECRET=${PW_XFD_2FA_SECRET} - PW_XFD_PASSWORD=${PW_XFD_PASSWORD} - - PW_XFD_URL=${PW_XFD_URL} + - PW_XFD_URL=http://xfd-frontend-1:3000 - PW_XFD_USER_ROLE=${PW_XFD_USER_ROLE} - PW_XFD_USERNAME=${PW_XFD_USERNAME} - - PW_DOCKER=true command: > /bin/bash -c "npm ci && npx playwright install --with-deps && npx playwright test" diff --git a/playwright/run_playwright_tests.sh b/playwright/run_playwright_tests.sh index c64976f6b..8e75ee765 100755 --- a/playwright/run_playwright_tests.sh +++ b/playwright/run_playwright_tests.sh @@ -20,7 +20,6 @@ OVERRIDES=$( --arg password "$PW_XFD_PASSWORD" \ --arg otpsecret "$PW_XFD_2FA_SECRET" \ --arg login "$PW_XFD_LOGIN" \ - --arg pw_ecs "$PW_ECS" \ --arg git_branch "$GIT_BRANCH" \ --arg s3HtmlPath "s3://$AUTOMATED_TEST_REPORTS_BUCKET_NAME/playwright-reports/$DATETIME/html/" \ --arg s3JsonPath "s3://$AUTOMATED_TEST_REPORTS_BUCKET_NAME/playwright-reports/$DATETIME/results.json" \ @@ -37,7 +36,6 @@ OVERRIDES=$( { "name": "PW_XFD_PASSWORD", "value": $password }, { "name": "PW_XFD_2FA_SECRET", "value": $otpsecret }, { "name": "PW_XFD_LOGIN", "value": $login }, - { "name": "PW_ECS", "value": "true" }, { "name": "GIT_BRANCH", "value": $git_branch } ], "command": [ diff --git a/playwright/utils/env.ts b/playwright/utils/env.ts index 9f211d5da..e68c7ab1c 100644 --- a/playwright/utils/env.ts +++ b/playwright/utils/env.ts @@ -1,25 +1,9 @@ // utils/env.ts export function determineUrl(): string { - const branch = process.env.GIT_BRANCH || ''; - const insideDocker = process.env.PW_DOCKER === 'true'; - const insideECS = process.env.PW_ECS; - - if (insideECS) { - if (branch === 'integration') { - return 'https://integration.crossfeed.cyber.dhs.gov'; - } - return 'https://staging-cd.crossfeed.cyber.dhs.gov'; - } - - if (insideDocker) { - return 'http://xfd-frontend-1:3000'; - } - - return 'http://localhost'; + return process.env.PW_XFD_URL || 'http://localhost'; } export function determineHeadless(): boolean { - const insideDocker = process.env.PW_DOCKER === 'true'; - const insideECS = process.env.PW_ECS === 'true' || !!process.env.AWS_REGION; - return insideDocker || insideECS; + const url = determineUrl(); + return !url.includes('localhost'); } From cb6834808344b02e6fef9cb278fa43854e58e276 Mon Sep 17 00:00:00 2001 From: JCantu248 Date: Thu, 8 May 2025 15:09:27 -0500 Subject: [PATCH 12/26] Refactor to move logic to new Dockerfile. --- .github/workflows/regression.yml | 14 +------------- Dockerfile.playwright | 23 +++++++++++++++++++++++ docker-compose.yml | 2 +- entrypoint.playwright.sh | 12 ++++++++++++ infrastructure/stage-cd.tfvars | 2 +- playwright/run_playwright_tests.sh | 23 +++-------------------- 6 files changed, 41 insertions(+), 35 deletions(-) create mode 100644 Dockerfile.playwright create mode 100644 entrypoint.playwright.sh diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 4f0321534..d4a2a5390 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -29,16 +29,10 @@ jobs: if: github.event_name == 'push' && github.ref == 'refs/heads/develop' steps: - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: 18 - name: Set environment variables for staging run: echo "ENVIRONMENT=github" >> $GITHUB_ENV - - name: Install dependencies - run: npm ci - - name: Run Playwright tests env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} @@ -48,7 +42,7 @@ jobs: PW_XFD_2FA_SECRET: ${{ secrets.PW_XFD_2FA_SECRET }} PW_XFD_PASSWORD: ${{ secrets.PW_XFD_PASSWORD }} PW_XFD_USERNAME: ${{ secrets.PW_XFD_USERNAME }} - GIT_BRANCH: integration + GIT_BRANCH: develop AUTOMATED_TEST_REPORTS_BUCKET_NAME: ${{ vars.AUTOMATED_TEST_REPORT_BUCKET_NAME }} AWS_REGION: ${{ secrets.AWS_REGION }} @@ -99,16 +93,10 @@ jobs: if: github.event_name == 'push' && github.ref == 'refs/heads/integration' steps: - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: 18 - name: Set environment variables for staging run: echo "ENVIRONMENT=github" >> $GITHUB_ENV - - name: Install dependencies - run: npm ci - - name: Run Playwright tests env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} diff --git a/Dockerfile.playwright b/Dockerfile.playwright new file mode 100644 index 000000000..045bdd9e7 --- /dev/null +++ b/Dockerfile.playwright @@ -0,0 +1,23 @@ +FROM mcr.microsoft.com/playwright:v1.50.1-jammy + +# Set working directory +WORKDIR /app + +# Install needed tools +RUN apt-get update && apt-get install -y git awscli + +# Clone Playwright test repo and install dependencies +RUN git clone -b CRASM-2286_Bug_fix_regression_yaml https://github.com/cisagov/xfd.git /app/xfd \ + && cd /app/xfd/playwright \ + && npm ci \ + && npx playwright install --with-deps + +# Set working directory to where tests are located +WORKDIR /app/xfd/playwright + +# Copy the test runner script +COPY entrypoint.playwright.sh /entrypoint.playwright.sh +RUN chmod +x /entrypoint.playwright.sh + +# Define entrypoint +ENTRYPOINT ["/entrypoint.playwright.sh"] diff --git a/docker-compose.yml b/docker-compose.yml index a173f005b..1eca7af81 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -175,7 +175,7 @@ services: - PW_XFD_2FA_ISSUER=${PW_XFD_2FA_ISSUER} - PW_XFD_2FA_SECRET=${PW_XFD_2FA_SECRET} - PW_XFD_PASSWORD=${PW_XFD_PASSWORD} - - PW_XFD_URL=http://xfd-frontend-1:3000 + - PW_XFD_URL=http://frontend-1:3000 - PW_XFD_USER_ROLE=${PW_XFD_USER_ROLE} - PW_XFD_USERNAME=${PW_XFD_USERNAME} command: > diff --git a/entrypoint.playwright.sh b/entrypoint.playwright.sh new file mode 100644 index 000000000..145bfbf4a --- /dev/null +++ b/entrypoint.playwright.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -euo pipefail + +echo "🔁 Running Playwright tests..." +npx playwright test + +echo "📤 Uploading results to S3..." +aws s3 cp ./playwright-report/html "$S3_HTML_PATH" --recursive --region "$AWS_REGION" +aws s3 cp ./playwright-report/results.json "$S3_JSON_PATH" --region "$AWS_REGION" + +echo "✅ Tests completed and uploaded." diff --git a/infrastructure/stage-cd.tfvars b/infrastructure/stage-cd.tfvars index 7be04be5d..c3b8428d3 100644 --- a/infrastructure/stage-cd.tfvars +++ b/infrastructure/stage-cd.tfvars @@ -4,7 +4,7 @@ aws_partition = "aws" is_dmz = true project = "Crossfeed" stage = "staging-cd" -frontend_domain = "staging.crossfeed.cyber.dhs.gov" +frontend_domain = "staging-cd.crossfeed.cyber.dhs.gov" frontend_lambda_function = "crossfeed-security-headers-staging" frontend_bucket = "staging.crossfeed.cyber.dhs.gov" api_domain = "api.staging-cd.crossfeed.cyber.dhs.gov" diff --git a/playwright/run_playwright_tests.sh b/playwright/run_playwright_tests.sh index 8e75ee765..a13eef0cf 100755 --- a/playwright/run_playwright_tests.sh +++ b/playwright/run_playwright_tests.sh @@ -36,27 +36,10 @@ OVERRIDES=$( { "name": "PW_XFD_PASSWORD", "value": $password }, { "name": "PW_XFD_2FA_SECRET", "value": $otpsecret }, { "name": "PW_XFD_LOGIN", "value": $login }, - { "name": "GIT_BRANCH", "value": $git_branch } + { "name": "GIT_BRANCH", "value": $git_branch }, + { "name": "S3_HTML_PATH", "value": $s3HtmlPath }, + { "name": "S3_JSON_PATH", "value": $s3JsonPath } ], - "command": [ - "sh", - "-c", - "echo \"Cloning Playwright tests from GitHub...\" && - git clone -b CRASM-2286_Bug_fix_regression_yaml https://github.com/cisagov/xfd.git /app/xfd && - cd /app/xfd/playwright && - echo \"Installing Node dependencies...\" && - npm install && - echo \"Installing Playwright...\" && - npx playwright install --with-deps && - echo \"Running Playwright Tests\" && - npx playwright test && - echo \"📤 Uploading test results to S3...\" && - echo \"Uploading HTML report to: ${s3HtmlPath}\" && - aws s3 cp ./playwright-report/html ${s3HtmlPath} --recursive --region $region && - echo \"Uploading JSON report to: ${s3JsonPath}\" && - aws s3 cp ./playwright-report/results.json ${s3JsonPath} --region $region && - echo \"✅ Test results uploaded successfully to S3 at $datetime\"" - ] } ] }' From 2d7051226fba6f01d53f97867be2df604b027bc2 Mon Sep 17 00:00:00 2001 From: JCantu248 Date: Mon, 12 May 2025 13:07:59 -0500 Subject: [PATCH 13/26] Big refactor of Playwright ECS, with new entrypoints, terraform changes, and changes to deployment workers. --- backend/tools/build-worker.sh | 2 + backend/tools/deploy-worker.sh | 4 ++ entrypoint.playwright.sh | 2 + infrastructure/integration.tfvars | 1 + infrastructure/playwright.tf | 95 +++++++++++++++++-------------- infrastructure/stage-cd.tfvars | 1 + infrastructure/stage.tfvars | 1 + infrastructure/vars.tf | 6 ++ 8 files changed, 70 insertions(+), 42 deletions(-) diff --git a/backend/tools/build-worker.sh b/backend/tools/build-worker.sh index 91806d26a..4e58afac0 100755 --- a/backend/tools/build-worker.sh +++ b/backend/tools/build-worker.sh @@ -9,3 +9,5 @@ set -e docker build --platform linux/amd64 -t crossfeed-worker -f Dockerfile.worker . docker build --platform linux/amd64 -t pe-worker -f Dockerfile.pe . + +docker build --platform linux/amd64 -t playwright-worker -f Dockerfile.playwright . diff --git a/backend/tools/deploy-worker.sh b/backend/tools/deploy-worker.sh index 8ee25d7a0..8ace0c487 100755 --- a/backend/tools/deploy-worker.sh +++ b/backend/tools/deploy-worker.sh @@ -10,6 +10,7 @@ AWS_ECR_DOMAIN=957221700844.dkr.ecr.us-east-1.amazonaws.com WORKER_TAG=${1:-crossfeed-staging-worker} PE_WORKER_TAG=${1:-pe-staging-worker} +PW_WORKER_TAG=${1:-playwright-staging-worker} ./tools/build-worker.sh aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin $AWS_ECR_DOMAIN @@ -18,3 +19,6 @@ docker push $AWS_ECR_DOMAIN/"$WORKER_TAG":latest docker tag pe-worker:latest $AWS_ECR_DOMAIN/"$PE_WORKER_TAG":latest docker push $AWS_ECR_DOMAIN/"$PE_WORKER_TAG":latest + +docker tag crossfeed-worker:latest $AWS_ECR_DOMAIN/"$PW_WORKER_TAG":latest +docker push $AWS_ECR_DOMAIN/"$PW_WORKER_TAG":latest diff --git a/entrypoint.playwright.sh b/entrypoint.playwright.sh index 145bfbf4a..b40e0b35f 100644 --- a/entrypoint.playwright.sh +++ b/entrypoint.playwright.sh @@ -8,5 +8,7 @@ npx playwright test echo "📤 Uploading results to S3..." aws s3 cp ./playwright-report/html "$S3_HTML_PATH" --recursive --region "$AWS_REGION" aws s3 cp ./playwright-report/results.json "$S3_JSON_PATH" --region "$AWS_REGION" +aws s3 cp ./playwright-report/html "s3://$AUTOMATED_TEST_REPORTS_BUCKET_NAME/playwright-reports/latest/html/" --recursive --region "$AWS_REGION" +aws s3 cp ./playwright-report/results.json "s3://$AUTOMATED_TEST_REPORTS_BUCKET_NAME/playwright-reports/latest/results.json" --region "$AWS_REGION" echo "✅ Tests completed and uploaded." diff --git a/infrastructure/integration.tfvars b/infrastructure/integration.tfvars index 407b894a8..400599c03 100644 --- a/infrastructure/integration.tfvars +++ b/infrastructure/integration.tfvars @@ -118,3 +118,4 @@ ssm_whoisxml_thread_count = "/crossfeed/integration/WHOIS_XML_THREAD_ ssm_whoisxml_api_key = "/crossfeed/integration/WHOIS_XML_API_KEY" crossfeed_playwright = "crossfeed-playwright-integration" automated_test_reports_bucket_name = "cisa-crossfeed-integration-automated-test-reports" +playwright_worker_repository_name = "crossfeed-playwright-worker-integration" diff --git a/infrastructure/playwright.tf b/infrastructure/playwright.tf index 1f30c5475..c8b9015f9 100644 --- a/infrastructure/playwright.tf +++ b/infrastructure/playwright.tf @@ -55,51 +55,61 @@ resource "aws_iam_role_policy_attachment" "playwright_ecs_execution_policy" { role = aws_iam_role.playwright_worker_task_execution_role.id } +resource "aws_ecr_repository" "playwright_worker" { + name = var.playwright_worker_repository_name + + image_scanning_configuration { + scan_on_push = true + } + + image_tag_mutability = "MUTABLE" + + encryption_configuration { + encryption_type = "KMS" + kms_key = aws_kms_key.key.arn + } + + tags = { + Project = var.project + Stage = var.stage + Owner = "Crossfeed managed resource" + } +} + + resource "aws_ecs_task_definition" "playwright_worker" { family = var.playwright_worker_ecs_task_definition_family - container_definitions = < Date: Mon, 12 May 2025 15:09:07 -0500 Subject: [PATCH 14/26] Bug fixes to regression.yml. --- .github/workflows/regression.yml | 16 ++++++++-------- .../Dockerfile.playwright | 0 2 files changed, 8 insertions(+), 8 deletions(-) rename Dockerfile.playwright => backend/Dockerfile.playwright (100%) diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index d4a2a5390..a6a5a1e7b 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -44,7 +44,7 @@ jobs: PW_XFD_USERNAME: ${{ secrets.PW_XFD_USERNAME }} GIT_BRANCH: develop AUTOMATED_TEST_REPORTS_BUCKET_NAME: - ${{ vars.AUTOMATED_TEST_REPORT_BUCKET_NAME }} + ${{ vars.AUTOMATED_TEST_REPORTS_BUCKET_NAME }} AWS_REGION: ${{ secrets.AWS_REGION }} AWS_SUBNET: ${{ secrets.AWS_SUBNET }} AWS_SECURITY_GROUP: ${{ secrets.AWS_SECURITY_GROUP }} @@ -52,7 +52,7 @@ jobs: chmod +x run_playwright_tests.sh CLUSTER_NAME=crossfeed-playwright-staging-cd-ecs-cluster \ TASK_DEFINITION=crossfeed-staging-cd-playwright-worker \ - AUTOMATED_TEST_RESULTS_BUCKET_NAME=$AUTOMATED_TEST_REPORT_BUCKET_NAME \ + AUTOMATED_TEST_REPORTS_BUCKET_NAME=$AUTOMATED_TEST_REPORTS_BUCKET_NAME \ AWS_SUBNET=$AWS_SUBNET \ AWS_SECURITY_GROUP=$AWS_SECURITY_GROUP \ PW_XFD_2FA_ISSUER=$PW_XFD_2FA_ISSUER \ @@ -66,11 +66,11 @@ jobs: run: | mkdir -p ./playwright-report/html aws s3 cp \ - s3://$AUTOMATED_TEST_REPORTS_BUCKET_NAME/playwright-reports/html/ \ + "s3://$AUTOMATED_TEST_REPORTS_BUCKET_NAME/playwright-reports/html/" \ ./playwright-report/html/ --recursive aws s3 cp \ - s3://$AUTOMATED_TEST_REPORTS_BUCKET_NAME/playwright-reports/results.json \ + "s3://$AUTOMATED_TEST_REPORTS_BUCKET_NAME/playwright-reports/results.json" \ ./playwright-report/results.json - name: Upload Playwright HTML report @@ -108,7 +108,7 @@ jobs: PW_XFD_USERNAME: ${{ secrets.PW_XFD_USERNAME }} GIT_BRANCH: integration AUTOMATED_TEST_REPORTS_BUCKET_NAME: - ${{ vars.AUTOMATED_TEST_REPORT_BUCKET_NAME }} + ${{ vars.AUTOMATED_TEST_REPORTS_BUCKET_NAME }} AWS_REGION: ${{ secrets.AWS_REGION }} AWS_SUBNET: ${{ secrets.AWS_SUBNET }} AWS_SECURITY_GROUP: ${{ secrets.AWS_SECURITY_GROUP }} @@ -116,7 +116,7 @@ jobs: chmod +x run_playwright_tests.sh CLUSTER_NAME=crossfeed-playwright-integration-ecs-cluster \ TASK_DEFINITION=crossfeed-integration-playwright-worker \ - AUTOMATED_TEST_RESULTS_BUCKET_NAME=$AUTOMATED_TEST_REPORT_BUCKET_NAME \ + AUTOMATED_TEST_REPORTS_BUCKET_NAME=$AUTOMATED_TEST_REPORTS_BUCKET_NAME \ AWS_SUBNET=$AWS_SUBNET \ AWS_SECURITY_GROUP=$AWS_SECURITY_GROUP \ ./run_playwright_tests.sh @@ -128,11 +128,11 @@ jobs: run: | mkdir -p ./playwright-report/html aws s3 cp \ - s3://$AUTOMATED_TEST_REPORTS_BUCKET_NAME/playwright-reports/html/ \ + "s3://${AUTOMATED_TEST_REPORTS_BUCKET_NAME}/playwright-reports/html/" \ ./playwright-report/html/ --recursive aws s3 cp \ - s3://$AUTOMATED_TEST_REPORTS_BUCKET_NAME/playwright-reports/results.json \ + "s3://${AUTOMATED_TEST_REPORTS_BUCKET_NAME}/playwright-reports/results.json"\ ./playwright-report/results.json - name: Upload Playwright HTML report diff --git a/Dockerfile.playwright b/backend/Dockerfile.playwright similarity index 100% rename from Dockerfile.playwright rename to backend/Dockerfile.playwright From d18ca8b87677d58ddfd127818a2a2db85bfea227 Mon Sep 17 00:00:00 2001 From: JCantu248 Date: Mon, 12 May 2025 15:20:12 -0500 Subject: [PATCH 15/26] Bug fix to Dockerfile.playwright --- backend/Dockerfile.playwright | 4 ++-- .../worker/entrypoint.playwright.sh | 0 2 files changed, 2 insertions(+), 2 deletions(-) rename entrypoint.playwright.sh => backend/worker/entrypoint.playwright.sh (100%) diff --git a/backend/Dockerfile.playwright b/backend/Dockerfile.playwright index 045bdd9e7..46cc85815 100644 --- a/backend/Dockerfile.playwright +++ b/backend/Dockerfile.playwright @@ -16,8 +16,8 @@ RUN git clone -b CRASM-2286_Bug_fix_regression_yaml https://github.com/cisagov/x WORKDIR /app/xfd/playwright # Copy the test runner script -COPY entrypoint.playwright.sh /entrypoint.playwright.sh +COPY ../entrypoint.playwright.sh /entrypoint.playwright.sh RUN chmod +x /entrypoint.playwright.sh # Define entrypoint -ENTRYPOINT ["/entrypoint.playwright.sh"] +ENTRYPOINT ["worker/entrypoint.playwright.sh"] diff --git a/entrypoint.playwright.sh b/backend/worker/entrypoint.playwright.sh similarity index 100% rename from entrypoint.playwright.sh rename to backend/worker/entrypoint.playwright.sh From 356e1c61b91a39ee59afd6c34a1796a74ba72d48 Mon Sep 17 00:00:00 2001 From: JCantu248 Date: Mon, 12 May 2025 15:28:42 -0500 Subject: [PATCH 16/26] Fixing entrypoint for Playwright --- backend/Dockerfile.playwright | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/Dockerfile.playwright b/backend/Dockerfile.playwright index 46cc85815..c9b83f543 100644 --- a/backend/Dockerfile.playwright +++ b/backend/Dockerfile.playwright @@ -16,8 +16,8 @@ RUN git clone -b CRASM-2286_Bug_fix_regression_yaml https://github.com/cisagov/x WORKDIR /app/xfd/playwright # Copy the test runner script -COPY ../entrypoint.playwright.sh /entrypoint.playwright.sh +COPY worker/entrypoint.playwright.sh /entrypoint.playwright.sh RUN chmod +x /entrypoint.playwright.sh # Define entrypoint -ENTRYPOINT ["worker/entrypoint.playwright.sh"] +ENTRYPOINT ["entrypoint.playwright.sh"] From 464934c796aa6ee04285d6af5e7cd79fff581e7e Mon Sep 17 00:00:00 2001 From: JCantu248 Date: Mon, 12 May 2025 16:37:06 -0500 Subject: [PATCH 17/26] Debugging Dockerfile.playwright --- backend/Dockerfile.playwright | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/Dockerfile.playwright b/backend/Dockerfile.playwright index c9b83f543..f1a771777 100644 --- a/backend/Dockerfile.playwright +++ b/backend/Dockerfile.playwright @@ -16,8 +16,8 @@ RUN git clone -b CRASM-2286_Bug_fix_regression_yaml https://github.com/cisagov/x WORKDIR /app/xfd/playwright # Copy the test runner script -COPY worker/entrypoint.playwright.sh /entrypoint.playwright.sh -RUN chmod +x /entrypoint.playwright.sh +COPY worker/entrypoint.playwright.sh worker/entrypoint.playwright.sh +RUN chmod +x worker/entrypoint.playwright.sh # Define entrypoint -ENTRYPOINT ["entrypoint.playwright.sh"] +ENTRYPOINT ["worker/entrypoint.playwright.sh"] From bee15b37cf987a7e3d9dfc8a78ef39d8fc9f32c7 Mon Sep 17 00:00:00 2001 From: JCantu248 Date: Tue, 13 May 2025 08:53:08 -0500 Subject: [PATCH 18/26] Fix Dockerfile to have debian be set to noninteractive. --- backend/Dockerfile.playwright | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/Dockerfile.playwright b/backend/Dockerfile.playwright index f1a771777..dbb824ad6 100644 --- a/backend/Dockerfile.playwright +++ b/backend/Dockerfile.playwright @@ -4,6 +4,7 @@ FROM mcr.microsoft.com/playwright:v1.50.1-jammy WORKDIR /app # Install needed tools +ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && apt-get install -y git awscli # Clone Playwright test repo and install dependencies From a25260b7291c4f3fb16e902164fe02951bdf5257 Mon Sep 17 00:00:00 2001 From: JCantu248 Date: Tue, 13 May 2025 15:17:50 -0500 Subject: [PATCH 19/26] Fixing things based on review from Copilot --- .github/workflows/regression.yml | 14 ++++++++------ backend/Dockerfile.playwright | 4 ++-- playwright/run_playwright_tests.sh | 6 ++++-- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index a6a5a1e7b..822e30cf1 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -1,3 +1,5 @@ +# yamllint disable + --- name: Regression Testing @@ -31,7 +33,7 @@ jobs: - uses: actions/checkout@v3 - name: Set environment variables for staging - run: echo "ENVIRONMENT=github" >> $GITHUB_ENV + run: echo "ENVIRONMENT=staging" >> $ENVIRONMENT - name: Run Playwright tests env: @@ -66,11 +68,11 @@ jobs: run: | mkdir -p ./playwright-report/html aws s3 cp \ - "s3://$AUTOMATED_TEST_REPORTS_BUCKET_NAME/playwright-reports/html/" \ + "s3://$AUTOMATED_TEST_REPORTS_BUCKET_NAME/staging/playwright-reports/html/" \ ./playwright-report/html/ --recursive aws s3 cp \ - "s3://$AUTOMATED_TEST_REPORTS_BUCKET_NAME/playwright-reports/results.json" \ + "s3://$AUTOMATED_TEST_REPORTS_BUCKET_NAME/staging/playwright-reports/results.json \ ./playwright-report/results.json - name: Upload Playwright HTML report @@ -95,7 +97,7 @@ jobs: - uses: actions/checkout@v3 - name: Set environment variables for staging - run: echo "ENVIRONMENT=github" >> $GITHUB_ENV + run: echo "ENVIRONMENT=integration" >> $ENVIRONMENT - name: Run Playwright tests env: @@ -128,11 +130,11 @@ jobs: run: | mkdir -p ./playwright-report/html aws s3 cp \ - "s3://${AUTOMATED_TEST_REPORTS_BUCKET_NAME}/playwright-reports/html/" \ + "s3://${AUTOMATED_TEST_REPORTS_BUCKET_NAME}/integration/playwright-reports/html/" \ ./playwright-report/html/ --recursive aws s3 cp \ - "s3://${AUTOMATED_TEST_REPORTS_BUCKET_NAME}/playwright-reports/results.json"\ + "s3://${AUTOMATED_TEST_REPORTS_BUCKET_NAME}/integration/playwright-reports/results.json" \ ./playwright-report/results.json - name: Upload Playwright HTML report diff --git a/backend/Dockerfile.playwright b/backend/Dockerfile.playwright index dbb824ad6..66e245be2 100644 --- a/backend/Dockerfile.playwright +++ b/backend/Dockerfile.playwright @@ -17,8 +17,8 @@ RUN git clone -b CRASM-2286_Bug_fix_regression_yaml https://github.com/cisagov/x WORKDIR /app/xfd/playwright # Copy the test runner script -COPY worker/entrypoint.playwright.sh worker/entrypoint.playwright.sh +COPY ../worker/entrypoint.playwright.sh /entrypoint.playwright.sh RUN chmod +x worker/entrypoint.playwright.sh # Define entrypoint -ENTRYPOINT ["worker/entrypoint.playwright.sh"] +ENTRYPOINT ["/entrypoint.playwright.sh"] diff --git a/playwright/run_playwright_tests.sh b/playwright/run_playwright_tests.sh index a13eef0cf..af6a73c31 100755 --- a/playwright/run_playwright_tests.sh +++ b/playwright/run_playwright_tests.sh @@ -21,8 +21,9 @@ OVERRIDES=$( --arg otpsecret "$PW_XFD_2FA_SECRET" \ --arg login "$PW_XFD_LOGIN" \ --arg git_branch "$GIT_BRANCH" \ - --arg s3HtmlPath "s3://$AUTOMATED_TEST_REPORTS_BUCKET_NAME/playwright-reports/$DATETIME/html/" \ - --arg s3JsonPath "s3://$AUTOMATED_TEST_REPORTS_BUCKET_NAME/playwright-reports/$DATETIME/results.json" \ + --arg environment "$ENVIRONMENT" \ + --arg s3HtmlPath "s3://$AUTOMATED_TEST_REPORTS_BUCKET_NAME/$ENVIRONMENT/playwright-reports/$DATETIME/html/" \ + --arg s3JsonPath "s3://$AUTOMATED_TEST_REPORTS_BUCKET_NAME/$ENVIRONMENT/playwright-reports/$DATETIME/results.json" \ '{ "containerOverrides": [ { @@ -37,6 +38,7 @@ OVERRIDES=$( { "name": "PW_XFD_2FA_SECRET", "value": $otpsecret }, { "name": "PW_XFD_LOGIN", "value": $login }, { "name": "GIT_BRANCH", "value": $git_branch }, + { "name": "ENVIRONMENT", "value": $environment }, { "name": "S3_HTML_PATH", "value": $s3HtmlPath }, { "name": "S3_JSON_PATH", "value": $s3JsonPath } ], From 0eb755116eebcad5946e390928b853e5a5e06072 Mon Sep 17 00:00:00 2001 From: JCantu248 Date: Tue, 13 May 2025 15:55:32 -0500 Subject: [PATCH 20/26] Fix path for chmod --- backend/Dockerfile.playwright | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/Dockerfile.playwright b/backend/Dockerfile.playwright index 66e245be2..745483e8c 100644 --- a/backend/Dockerfile.playwright +++ b/backend/Dockerfile.playwright @@ -18,7 +18,7 @@ WORKDIR /app/xfd/playwright # Copy the test runner script COPY ../worker/entrypoint.playwright.sh /entrypoint.playwright.sh -RUN chmod +x worker/entrypoint.playwright.sh +RUN chmod +x /entrypoint.playwright.sh # Define entrypoint ENTRYPOINT ["/entrypoint.playwright.sh"] From b960caf975bcdb5d21e8799e2a5aadd6006bd862 Mon Sep 17 00:00:00 2001 From: JCantu248 Date: Wed, 14 May 2025 10:03:05 -0500 Subject: [PATCH 21/26] Add environment argument to dockerfile entrypoint. --- .github/workflows/regression.yml | 4 ++-- backend/worker/entrypoint.playwright.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 822e30cf1..b8d9e95a7 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -33,7 +33,7 @@ jobs: - uses: actions/checkout@v3 - name: Set environment variables for staging - run: echo "ENVIRONMENT=staging" >> $ENVIRONMENT + run: echo "ENVIRONMENT=staging" >> $GITHUB_ENV - name: Run Playwright tests env: @@ -97,7 +97,7 @@ jobs: - uses: actions/checkout@v3 - name: Set environment variables for staging - run: echo "ENVIRONMENT=integration" >> $ENVIRONMENT + run: echo "ENVIRONMENT=integration" >> $GITHUB_ENV - name: Run Playwright tests env: diff --git a/backend/worker/entrypoint.playwright.sh b/backend/worker/entrypoint.playwright.sh index b40e0b35f..398c79872 100644 --- a/backend/worker/entrypoint.playwright.sh +++ b/backend/worker/entrypoint.playwright.sh @@ -8,7 +8,7 @@ npx playwright test echo "📤 Uploading results to S3..." aws s3 cp ./playwright-report/html "$S3_HTML_PATH" --recursive --region "$AWS_REGION" aws s3 cp ./playwright-report/results.json "$S3_JSON_PATH" --region "$AWS_REGION" -aws s3 cp ./playwright-report/html "s3://$AUTOMATED_TEST_REPORTS_BUCKET_NAME/playwright-reports/latest/html/" --recursive --region "$AWS_REGION" -aws s3 cp ./playwright-report/results.json "s3://$AUTOMATED_TEST_REPORTS_BUCKET_NAME/playwright-reports/latest/results.json" --region "$AWS_REGION" +aws s3 cp ./playwright-report/html "s3://$AUTOMATED_TEST_REPORTS_BUCKET_NAME/$ENVIRONMENT/playwright-reports/latest/html/" --recursive --region "$AWS_REGION" +aws s3 cp ./playwright-report/results.json "s3://$AUTOMATED_TEST_REPORTS_BUCKET_NAME/$ENVIRONMENT/playwright-reports/latest/results.json" --region "$AWS_REGION" echo "✅ Tests completed and uploaded." From 9c5a66b946ee8c440058c77553f91c4b86648b62 Mon Sep 17 00:00:00 2001 From: JCantu248 Date: Mon, 19 May 2025 13:35:34 -0500 Subject: [PATCH 22/26] Adjust frontend URL for playwright and wait timer. --- docker-compose.yml | 2 +- playwright/global-setup.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 7b532ef98..ddcd7f6f2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -177,7 +177,7 @@ services: - PW_XFD_2FA_ISSUER=${PW_XFD_2FA_ISSUER} - PW_XFD_2FA_SECRET=${PW_XFD_2FA_SECRET} - PW_XFD_PASSWORD=${PW_XFD_PASSWORD} - - PW_XFD_URL=http://frontend-1:3000 + - PW_XFD_URL=http://frontend:3000 - PW_XFD_USER_ROLE=${PW_XFD_USER_ROLE} - PW_XFD_USERNAME=${PW_XFD_USERNAME} command: > diff --git a/playwright/global-setup.ts b/playwright/global-setup.ts index 5012f83c3..9528cb158 100644 --- a/playwright/global-setup.ts +++ b/playwright/global-setup.ts @@ -18,7 +18,7 @@ let totp = new OTPAuth.TOTP({ const axios = require('axios'); -const waitForFrontend = async (url, timeout = 60000, checkInterval = 5000) => { +const waitForFrontend = async (url, timeout = 600000, checkInterval = 5000) => { const startTime = Date.now(); while (Date.now() - startTime < timeout) { try { From c204cfc274ca852cfa18806a7f15269b03470b37 Mon Sep 17 00:00:00 2001 From: JCantu248 Date: Tue, 20 May 2025 10:02:47 -0500 Subject: [PATCH 23/26] Small changes to version issues in pre-commit, small change to regression workflow to download from s3 latest. --- .github/workflows/build.yml | 4 ++-- .github/workflows/regression.yml | 4 ++-- .pre-commit-config.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 411d76c7e..7da7bba22 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -51,12 +51,12 @@ jobs: # We need the Go version and Go cache location for the actions/cache step, # so the Go installation must happen before that. - id: setup-go - uses: actions/setup-go@v4 + uses: actions/setup-go@v5 with: # There is no expectation for actual Go code so we disable caching as # it relies on the existence of a go.sum file. cache: false - go-version: '1.20' + go-version: '1.23' - name: Lookup Go cache directory id: go-cache run: echo "dir=$(go env GOCACHE)" >> $GITHUB_OUTPUT diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index b8d9e95a7..4f9c9d2a8 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -68,11 +68,11 @@ jobs: run: | mkdir -p ./playwright-report/html aws s3 cp \ - "s3://$AUTOMATED_TEST_REPORTS_BUCKET_NAME/staging/playwright-reports/html/" \ + "s3://$AUTOMATED_TEST_REPORTS_BUCKET_NAME/staging/playwright-reports/latest/html/" \ ./playwright-report/html/ --recursive aws s3 cp \ - "s3://$AUTOMATED_TEST_REPORTS_BUCKET_NAME/staging/playwright-reports/results.json \ + "s3://$AUTOMATED_TEST_REPORTS_BUCKET_NAME/staging/playwright-reports/latest/results.json \ ./playwright-report/results.json - name: Upload Playwright HTML report diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9a817ee58..7e3f6e825 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -61,8 +61,8 @@ repos: - id: validate_manifest # Shell script hooks - - repo: https://github.com/cisagov/pre-commit-shfmt - rev: v0.0.2 + - repo: https://github.com/scop/pre-commit-shfmt + rev: v3.11.0-1 hooks: - id: shfmt args: From b7848928b1be59fa45be4de226b4859336efed2d Mon Sep 17 00:00:00 2001 From: JCantu248 Date: Fri, 6 Jun 2025 17:20:01 -0500 Subject: [PATCH 24/26] Fix a path issue with global-setup loading environment variables. --- dev.env.example | 1 + docker-compose.yml | 1 + playwright/global-setup.ts | 2 +- playwright/utils/env.ts | 3 +++ 4 files changed, 6 insertions(+), 1 deletion(-) diff --git a/dev.env.example b/dev.env.example index 749c6ac1a..04e88d8f7 100644 --- a/dev.env.example +++ b/dev.env.example @@ -151,6 +151,7 @@ PW_XFD_PASSWORD = PW_XFD_2FA_ISSUER = PW_XFD_2FA_SECRET = PW_XFD_USER_ROLE = +PW_HEADLESS=false SESSION_STORAGE = "" DMZ_SYNC_ENDPOINT='http://localhost:3000' diff --git a/docker-compose.yml b/docker-compose.yml index ddcd7f6f2..13d1f5691 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -180,6 +180,7 @@ services: - PW_XFD_URL=http://frontend:3000 - PW_XFD_USER_ROLE=${PW_XFD_USER_ROLE} - PW_XFD_USERNAME=${PW_XFD_USERNAME} + - PW_HEADLESS=false command: > /bin/bash -c "npm ci && npx playwright install --with-deps && npx playwright test" diff --git a/playwright/global-setup.ts b/playwright/global-setup.ts index 9528cb158..8b5febd9d 100644 --- a/playwright/global-setup.ts +++ b/playwright/global-setup.ts @@ -3,7 +3,7 @@ import * as OTPAuth from 'otpauth'; import * as dotenv from 'dotenv'; import { determineUrl } from './utils/env'; // ✅ Now this works -dotenv.config(); +dotenv.config({ path: '../.env' }); const authFile = './storageState.json'; diff --git a/playwright/utils/env.ts b/playwright/utils/env.ts index e68c7ab1c..a3b0ed83a 100644 --- a/playwright/utils/env.ts +++ b/playwright/utils/env.ts @@ -4,6 +4,9 @@ export function determineUrl(): string { } export function determineHeadless(): boolean { + if (process.env.PW_HEADLESS) { + return process.env.PW_HEADLESS !== 'false'; + } const url = determineUrl(); return !url.includes('localhost'); } From 14b843c280ccdb26096d012123c6bb84b9661172 Mon Sep 17 00:00:00 2001 From: JCantu248 Date: Fri, 6 Jun 2025 17:21:14 -0500 Subject: [PATCH 25/26] Change docker-compose.yml PW_Headless to be true for Docker instance of Playwright --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 13d1f5691..c2d88f88d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -180,7 +180,7 @@ services: - PW_XFD_URL=http://frontend:3000 - PW_XFD_USER_ROLE=${PW_XFD_USER_ROLE} - PW_XFD_USERNAME=${PW_XFD_USERNAME} - - PW_HEADLESS=false + - PW_HEADLESS=true command: > /bin/bash -c "npm ci && npx playwright install --with-deps && npx playwright test" From a986a7c46de2a2e642da09ef501120ba0dfddf44 Mon Sep 17 00:00:00 2001 From: JCantu248 Date: Tue, 10 Jun 2025 12:46:13 -0500 Subject: [PATCH 26/26] Revision to test case browser actions and test fixtures. --- playwright/axe-test.ts | 5 ++-- playwright/e2e/global-admin/home.spec.ts | 26 ++++++---------- playwright/e2e/global-admin/inventory.spec.ts | 30 ++++++++----------- .../e2e/global-admin/vulnerabilities.spec.ts | 24 +++++---------- playwright/global-setup.ts | 7 +++-- 5 files changed, 36 insertions(+), 56 deletions(-) diff --git a/playwright/axe-test.ts b/playwright/axe-test.ts index f464084d5..070a7456a 100644 --- a/playwright/axe-test.ts +++ b/playwright/axe-test.ts @@ -1,4 +1,4 @@ -import { test as base } from '@playwright/test'; +import { test as base, expect } from '@playwright/test'; import AxeBuilder from '@axe-core/playwright'; type AxeFixture = { @@ -18,5 +18,4 @@ export const test = base.extend({ await use(makeAxeBuilder); } }); -export { expect } from '@playwright/test'; -export { Page } from '@playwright/test'; +export { expect }; diff --git a/playwright/e2e/global-admin/home.spec.ts b/playwright/e2e/global-admin/home.spec.ts index c46ecc45f..721df2d09 100644 --- a/playwright/e2e/global-admin/home.spec.ts +++ b/playwright/e2e/global-admin/home.spec.ts @@ -1,27 +1,19 @@ -const { test, expect, Page } = require('../../axe-test'); - -test.describe.configure({ mode: 'parallel' }); -let page: InstanceType; +import { test, expect } from '../../axe-test'; +import type { TestInfo } from '@playwright/test'; test.describe('home', () => { - test.beforeEach(async ({ browser }) => { - const context = await browser.newContext(); - page = await context.newPage(); + test('Test homepage accessibility', async ({ + page, + makeAxeBuilder + }, testInfo: TestInfo) => { await page.goto('/'); - }); - - test.afterEach(async () => { - await page.close(); - }); - - test('Test homepage accessibility', async ({ makeAxeBuilder }, testInfo) => { - const accessibilityScanResults = await makeAxeBuilder().analyze(); + const results = await makeAxeBuilder().analyze(); await testInfo.attach('accessibility-scan-results', { - body: JSON.stringify(accessibilityScanResults, null, 2), + body: JSON.stringify(results, null, 2), contentType: 'application/json' }); - expect(accessibilityScanResults.violations).toHaveLength(0); + expect(results.violations).toHaveLength(0); }); }); diff --git a/playwright/e2e/global-admin/inventory.spec.ts b/playwright/e2e/global-admin/inventory.spec.ts index 7be9d0115..193318d38 100644 --- a/playwright/e2e/global-admin/inventory.spec.ts +++ b/playwright/e2e/global-admin/inventory.spec.ts @@ -1,19 +1,11 @@ -const { test, expect, Page } = require('../../axe-test'); - -test.describe.configure({ mode: 'parallel' }); -let page: InstanceType; +const { test, expect } = require('../../axe-test'); +import type { TestInfo } from '@playwright/test'; test.describe('Inventory', () => { - test.beforeEach(async ({ browser }) => { - const context = await browser.newContext(); - page = await context.newPage(); - await page.goto('/'); - }); - - test.afterEach(async () => { - await page.close(); - }); - test('Test inventory accessibility', async ({ makeAxeBuilder }, testInfo) => { + test('Test inventory accessibility', async ({ + page, + makeAxeBuilder + }, testInfo: TestInfo) => { await page.goto('/inventory'); const accessibilityScanResults = await makeAxeBuilder().analyze(); @@ -25,7 +17,10 @@ test.describe('Inventory', () => { expect(accessibilityScanResults.violations).toHaveLength(0); }); - test('Test domain accessibility', async ({ makeAxeBuilder }, testInfo) => { + test('Test domain accessibility', async ({ + page, + makeAxeBuilder + }, testInfo: TestInfo) => { await page.goto('/inventory/domains'); const accessibilityScanResults = await makeAxeBuilder().analyze(); @@ -39,8 +34,9 @@ test.describe('Inventory', () => { // TODO: Skip this test until the domain table data is loaded in localhost. test.skip('Test domain details accessibility', async ({ + page, makeAxeBuilder - }, testInfo) => { + }, testInfo: TestInfo) => { await page.goto('/inventory/domains'); await page .getByRole('row') @@ -64,7 +60,7 @@ test.describe('Inventory', () => { }); // TODO: Skip this test until the domain table data is loaded in localhost. - test.skip('Test domain table filter', async () => { + test.skip('Test domain table filter', async ({ page }) => { await page.goto('/inventory/domains'); await page.getByLabel('Show filters').click(); await page.getByPlaceholder('Filter value').click(); diff --git a/playwright/e2e/global-admin/vulnerabilities.spec.ts b/playwright/e2e/global-admin/vulnerabilities.spec.ts index bc7735035..b410bc81c 100644 --- a/playwright/e2e/global-admin/vulnerabilities.spec.ts +++ b/playwright/e2e/global-admin/vulnerabilities.spec.ts @@ -1,22 +1,11 @@ const { test, expect, Page } = require('../../axe-test'); - -test.describe.configure({ mode: 'parallel' }); -let page: InstanceType; +import type { TestInfo } from '@playwright/test'; test.describe('Vulnerabilities', () => { - test.beforeEach(async ({ browser }) => { - const context = await browser.newContext(); - page = await context.newPage(); - await page.goto('/'); - }); - - test.afterEach(async () => { - await page.close(); - }); - test('Test vulnerabilities accessibility', async ({ + page, makeAxeBuilder - }, testInfo) => { + }, testInfo: TestInfo) => { await page.goto('/inventory/vulnerabilities'); const accessibilityScanResults = await makeAxeBuilder().analyze(); @@ -30,7 +19,7 @@ test.describe('Vulnerabilities', () => { }); //TODO: Skip this test until the vulnerability table data is loaded in localhost. - test.skip('Test vulnerability details NIST link', async () => { + test.skip('Test vulnerability details NIST link', async ({ page }) => { await page.goto('/inventory/vulnerabilities'); const newTabPromise = page.waitForEvent('popup'); @@ -43,7 +32,7 @@ test.describe('Vulnerabilities', () => { }); //TODO: Skip this test until the vulnerability table data is loaded in localhost. - test.skip('Test domain details link', async () => { + test.skip('Test domain details link', async ({ page }) => { await page.goto('/inventory/vulnerabilities'); await page.getByRole('row').nth(1).getByRole('cell').nth(3).click(); await expect(page).toHaveURL(new RegExp('/inventory/domain/')); @@ -51,8 +40,9 @@ test.describe('Vulnerabilities', () => { //TODO: Skip this test until the vulnerability table data is loaded in localhost. test.skip('Test vulnerability details accessibility', async ({ + page, makeAxeBuilder - }, testInfo) => { + }, testInfo: TestInfo) => { await page.goto('/inventory/vulnerabilities'); await page.getByRole('row').nth(1).getByRole('cell').nth(7).click(); await expect(page).toHaveURL(new RegExp('/inventory/vulnerability/')); diff --git a/playwright/global-setup.ts b/playwright/global-setup.ts index 8b5febd9d..5802e1e55 100644 --- a/playwright/global-setup.ts +++ b/playwright/global-setup.ts @@ -1,7 +1,7 @@ import { chromium, FullConfig } from '@playwright/test'; import * as OTPAuth from 'otpauth'; import * as dotenv from 'dotenv'; -import { determineUrl } from './utils/env'; // ✅ Now this works +import { determineUrl } from './utils/env'; dotenv.config({ path: '../.env' }); @@ -58,7 +58,10 @@ async function globalSetup(config: FullConfig) { //Log in with credentials. await waitForFrontend(baseUrl); - await page.goto(baseUrl); + await page.goto(baseUrl, { + waitUntil: 'domcontentloaded', + timeout: 60000 + }); await page.getByTestId('button').click(); await page .getByLabel('Username (Email)')