diff --git a/.github/workflows/infrastructure-download-external.yml b/.github/workflows/infrastructure-download-external.yml index b167a056f..a90f5a6d2 100644 --- a/.github/workflows/infrastructure-download-external.yml +++ b/.github/workflows/infrastructure-download-external.yml @@ -42,6 +42,18 @@ on: required: false type: boolean default: false + CHUNK_INDEX: + # Which slice of the matrix this invocation processes (0..CHUNK_COUNT-1). + # Used to keep strategy.matrix under GitHub Actions' 256-entry cap when + # the number of packages × arches × releases grows past that. Defaults + # to 0/1 so the workflow still works when called un-chunked. + required: false + type: number + default: 0 + CHUNK_COUNT: + required: false + type: number + default: 1 secrets: GPG_KEY1: required: true @@ -184,6 +196,13 @@ jobs: name: "Mirror" outputs: matrix: ${{steps.lists.outputs.matrix}} + # 'true' when the matrix has real work; 'false' when start + # only emitted the {name:none,...} placeholder (empty slice + # for this chunk, or no actionable changes at all). The + # download job gates on this via a job-level `if:` so an + # empty chunk skips the entire matrix expansion instead of + # trying to source os/external/none.conf. + has_work: ${{steps.lists.outputs.has_work}} steps: - name: Fix workspace ownership @@ -209,7 +228,7 @@ jobs: - name: Upload Artifact uses: actions/upload-artifact@v7 with: - name: assets-for-download + name: assets-for-download-${{ inputs.CHUNK_INDEX }} path: downloads overwrite: true retention-days: 5 @@ -355,23 +374,87 @@ jobs: # Check if matrix is valid MATRIX_COUNT=$(echo "${MATRIX_JSON}" | jq '.include | length') + # has_work tracks whether the resulting matrix has any real + # entries vs the {name:none,...} placeholder. The download + # job's job-level `if:` reads this output and skips the + # whole matrix expansion when has_work='false', avoiding the + # source of os/external/none.conf that would otherwise hard- + # fail on the placeholder entry. + HAS_WORK="true" + # Handle empty matrix - add a dummy entry that will be skipped if [[ "${MATRIX_COUNT}" -eq 0 ]]; then echo "::warning::No matrix entries generated, adding placeholder" >&2 MATRIX_JSON_COMPACTED='{"include":[{"name":"none","arch":"amd64","release":"bookworm"}]}' + HAS_WORK="false" else - # Debug: Show raw matrix JSON - echo "::debug::Raw matrix JSON: ${MATRIX_JSON}" >&2 + # Chunk the matrix so each invocation keeps strategy.matrix + # below GitHub Actions' 256-entry cap. Callers that don't + # know about chunking get CHUNK_COUNT=1 and pass the whole + # thing through (legacy behavior). + # + # Validate the inputs explicitly. workflow_call input + # `type: number` is only enforced at the YAML boundary; + # direct API / curl invocations or templated callers can + # send anything that ends up as a string in the env. Bash's + # arithmetic context silently treats non-numeric as 0, + # which would let "abc" pass `-lt 1` and reset CHUNK_COUNT + # to 1 with no warning. Regex-check first, then range. + CHUNK_INDEX="${{ inputs.CHUNK_INDEX }}" + CHUNK_COUNT="${{ inputs.CHUNK_COUNT }}" + + if [[ ! "${CHUNK_INDEX}" =~ ^[0-9]+$ ]]; then + echo "::error::CHUNK_INDEX=${CHUNK_INDEX} is not a non-negative integer" >&2 + exit 1 + fi + if [[ ! "${CHUNK_COUNT}" =~ ^[0-9]+$ ]]; then + echo "::error::CHUNK_COUNT=${CHUNK_COUNT} is not a non-negative integer" >&2 + exit 1 + fi + if [[ "${CHUNK_COUNT}" -lt 1 ]]; then + echo "::error::CHUNK_COUNT=${CHUNK_COUNT} must be >= 1" >&2 + exit 1 + fi + if [[ "${CHUNK_INDEX}" -ge "${CHUNK_COUNT}" ]]; then + echo "::error::CHUNK_INDEX=${CHUNK_INDEX} must be < CHUNK_COUNT=${CHUNK_COUNT}" >&2 + exit 1 + fi - # Compact the JSON for GitHub Actions - MATRIX_JSON_COMPACTED=$(echo "${MATRIX_JSON}" | jq -c) + # Slice jq: split includes into CHUNK_COUNT roughly-equal + # groups by modular index (not contiguous ranges) so a + # slow package type doesn't cluster into one chunk. + SLICED=$(echo "${MATRIX_JSON}" | jq -c \ + --argjson idx "${CHUNK_INDEX}" \ + --argjson cnt "${CHUNK_COUNT}" \ + '{include: [.include | to_entries[] | select(.key % $cnt == $idx) | .value]}') + + SLICE_COUNT=$(echo "${SLICED}" | jq '.include | length') + echo "::notice::chunk ${CHUNK_INDEX}/${CHUNK_COUNT}: ${SLICE_COUNT} of ${MATRIX_COUNT} entries" >&2 + + # Empty slice still needs a placeholder so strategy.matrix + # is a valid (non-empty) value — but we set HAS_WORK=false + # so the download job's job-level `if:` skips it before + # the matrix even expands. + if [[ "${SLICE_COUNT}" -eq 0 ]]; then + MATRIX_JSON_COMPACTED='{"include":[{"name":"none","arch":"amd64","release":"bookworm"}]}' + HAS_WORK="false" + else + MATRIX_JSON_COMPACTED="${SLICED}" + fi echo "::debug::Compacted matrix JSON: ${MATRIX_JSON_COMPACTED}" >&2 fi echo "matrix=${MATRIX_JSON_COMPACTED}" >> $GITHUB_OUTPUT + echo "has_work=${HAS_WORK}" >> $GITHUB_OUTPUT download: needs: [start] + # Skip the whole job when start emitted a placeholder matrix + # (no work for this chunk, or no actionable changes at all). + # Job-level `if:` cannot reference `matrix.*` because matrix + # is expanded after if-evaluation; using a needs.* output works + # because those resolve at job-graph time. + if: needs.start.outputs.has_work == 'true' outputs: project: ${{steps.make.outputs.project}} strategy: @@ -897,5 +980,5 @@ jobs: - name: Clean artifacts uses: geekyeggo/delete-artifact@v6 with: - name: assets-for-download + name: assets-for-download-${{ inputs.CHUNK_INDEX }} failOnError: false diff --git a/.github/workflows/infrastructure-repository-update.yml b/.github/workflows/infrastructure-repository-update.yml index 11817470d..1ee3a2075 100644 --- a/.github/workflows/infrastructure-repository-update.yml +++ b/.github/workflows/infrastructure-repository-update.yml @@ -41,9 +41,19 @@ jobs: TEAM: "Release manager" external: - name: "Download external" + name: "Download external (chunk ${{ matrix.chunk_index }}/4)" needs: Check - uses: armbian/armbian.github.io/.github/workflows/infrastructure-download-external.yml@main + # Fan out across 4 chunks. The child workflow filters its matrix to + # only entries where `index % CHUNK_COUNT == CHUNK_INDEX`, so each + # invocation stays under GitHub Actions' 256-entry strategy.matrix + # cap. Effective concurrency is 4 × max-parallel (180) = 720. + # Bump `chunk_index` and `CHUNK_COUNT` together when total packages + # × arches × releases approaches 4 × 256 = 1024. + strategy: + fail-fast: false + matrix: + chunk_index: [0, 1, 2, 3] + uses: armbian/armbian.github.io/.github/workflows/infrastructure-download-external.yml@matrix-chunk-via-reusable-workflow with: ENABLED: ${{ inputs.download_external != false || github.event.client_payload.download_external != false }} SKIP_VERSION_CHECK: false @@ -51,6 +61,8 @@ jobs: BUILD_RUNNER: "docker" HOST_DEPLOY: "repo.armbian.com" PURGE: ${{ inputs.purge_external || false }} + CHUNK_INDEX: ${{ matrix.chunk_index }} + CHUNK_COUNT: 4 secrets: GPG_KEY1: ${{ secrets.GPG_KEY3 }} GPG_KEY2: ${{ secrets.GPG_KEY4 }}