Skip to content

Adjust GHA to build runtimes from fellows and sdk repos #75

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 11 commits into from
Oct 31, 2023
Merged

Adjust GHA to build runtimes from fellows and sdk repos #75

Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
cd8170b
adjust GHA to be able to set specific repo
EgorPopelyaev Oct 19, 2023
39cc562
chnage srtool-action version
EgorPopelyaev Oct 19, 2023
dd4fd79
add missing working-directory param
EgorPopelyaev Oct 19, 2023
75a6560
add the commit hash from the target repo as the key for cashe
EgorPopelyaev Oct 20, 2023
d1b03af
add working directory to get hash
EgorPopelyaev Oct 20, 2023
f378500
use get_values instead of srtool_build to retrieve wasms from cache
EgorPopelyaev Oct 23, 2023
4ba7da0
add repo choice and change caching for sdk
EgorPopelyaev Oct 23, 2023
e8b6bae
move TOML_CLI_VERSION to the env section
EgorPopelyaev Oct 24, 2023
164f15b
update srtool-action version
EgorPopelyaev Oct 30, 2023
72be07d
Switch to the latest srtool GHA
chevdor Oct 30, 2023
86da55b
Typo
chevdor Oct 30, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
162 changes: 107 additions & 55 deletions .github/workflows/manual-fellow-runtimes.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ name: Manual Build - Fellowship

env:
SUBWASM_VERSION: 0.20.0
TOML_CLI_VERSION: 0.2.4

on:
workflow_dispatch:
Expand All @@ -13,6 +14,10 @@ on:
description: The SRTOOL tag to use
default: 1.70.0
required: false
repo:
description: The repo to be used to build runtimes from
default: polkadot-fellows/runtimes
required: false
ref:
description: The ref to be used for the repo
default: main
Expand All @@ -24,64 +29,104 @@ on:
- cron: "00 04 * * 1"

jobs:
find-runtimes:
name: Scan repo ${{ inputs.repo }} ${{ inputs.ref }}
outputs:
runtime: ${{ steps.get_runtimes_list.outputs.runtime }}
commit_hash: ${{ steps.get_commit_hash.outputs.commit_hash }}
runs-on: ubuntu-latest
steps:
- name: Checkout the srtool repo
uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
with:
fetch-depth: 0
path: srtool

- name: Checkout repo ${{ inputs.repo }}
uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
with:
repository: ${{ inputs.repo }}
ref: ${{ inputs.ref }}
fetch-depth: 0
path: fellows-runtimes

- name: Get commit hash for ${{ inputs.repo }} ${{ inputs.ref }}
id: get_commit_hash
working-directory: fellows-runtimes
run: |
echo "commit_hash=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT

- name: Cache runtimes list for ${{ steps.get_commit_hash.outputs.commit_hash }}
id: cache_runtimes_list
if: inputs.cache == 'true'
uses: actions/cache@v3
with:
key: runtimes-list-${{ steps.get_commit_hash.outputs.commit_hash }}
path: |
fellows-runtimes/runtimes_list.json

- name: Install tooling
if: ${{ steps.cache_runtimes_list.outputs.cache-hit != 'true' }}
run: |
URL=https://github.com/chevdor/toml-cli/releases/download/v${{ env.TOML_CLI_VERSION }}/toml_linux_amd64_v${{ env.TOML_CLI_VERSION }}.deb
curl -L $URL --output toml.deb
sudo dpkg -i toml.deb
toml --version; jq --version

- name: Scan runtimes
if: ${{ steps.cache_runtimes_list.outputs.cache-hit != 'true' }}

run: |
. ./srtool/scripts/lib.sh

echo "Github workspace: ${{ github.workspace }}"
echo "Current folder: $(pwd)"; ls -al
cd fellows-runtimes; ls -al

MATRIX=$(find_runtimes | tee runtimes_list.json)
echo $MATRIX

- name: Get runtimes list
id: get_runtimes_list
run: |
cd fellows-runtimes; ls -al
MATRIX=$(cat runtimes_list.json)
echo $MATRIX
echo "runtime=$MATRIX" >> $GITHUB_OUTPUT

build:
name: Build ${{ matrix.chain }} ${{ github.event.inputs.ref }}
name: Build ${{ matrix.chain }} ${{ inputs.ref }}
needs:
- find-runtimes
strategy:
fail-fast: false
matrix:
include:
# Relay
- runtime_dir: relay/kusama
runtime: staging-kusama-runtime
chain: staging-kusama
- runtime_dir: relay/polkadot
runtime: polkadot-runtime
chain: polkadot

# System parachains / Asset Hub
- runtime_dir: system-parachains/asset-hubs/asset-hub-kusama
runtime: asset-hub-kusama-runtime
chain: asset-hub-kusama
- runtime_dir: system-parachains/asset-hubs/asset-hub-polkadot
runtime: asset-hub-polkadot-runtime
chain: asset-hub-polkadot

# System parachains / Bridge Hub
- runtime_dir: system-parachains/bridge-hubs/bridge-hub-kusama
runtime: bridge-hub-kusama-runtime
chain: bridge-hub-kusama
- runtime_dir: system-parachains/bridge-hubs/bridge-hub-polkadot
runtime: bridge-hub-polkadot-runtime
chain: bridge-hub-polkadot

# System parachains / Collectives
- runtime_dir: system-parachains/collectives/collectives-polkadot
runtime: collectives-polkadot-runtime
chain: collectives-polkadot

matrix: ${{ fromJSON(needs.find-runtimes.outputs.runtime) }}
runs-on: ubuntu-latest
steps:
- name: Cache ${{ matrix.chain }} runtime for ${{ github.sha }}
- name: Cache ${{ matrix.chain }} runtime for ${{ needs.find-runtimes.outputs.commit_hash }}
id: cache_runtime
if: github.event.inputs.cache == 'true'
uses: actions/cache@v3
if: inputs.cache == 'true'
uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 #v3.3.2
with:
key: ${{ matrix.chain }}-${{ github.sha }}
key: ${{ matrix.chain }}-${{ needs.find-runtimes.outputs.commit_hash }}
path: |
sdk/${{ matrix.runtime_dir }}/target/srtool/release/wbuild/${{ matrix.chain }}-runtime
fellows-runtimes/${{ matrix.runtime_dir }}/target/srtool/release/wbuild/${{ matrix.chain }}-runtime

- uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
- name: Checkout
uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
if: ${{ steps.cache_runtime.outputs.cache-hit != 'true' }}
with:
repository: polkadot-fellows/runtimes
ref: ${{ github.event.inputs.ref }}
repository: ${{ inputs.repo }}
ref: ${{ inputs.ref }}
fetch-depth: 0
path: fellows-runtimes

- name: Srtool build
id: srtool_build
if: ${{ steps.cache_runtime.outputs.cache-hit != 'true' }}
uses: chevdor/srtool-actions@v0.8.0
uses: chevdor/srtool-actions@wk-230920-add-debug
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The extra debug and clean is nice but it does not explain the need to go offroad.
I would suggest staying the a release tag.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Totally agree, then we need to wait till those changes are going to be released

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we really need the extra debug and fixes ?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

seems like the fix is needed, cause with the current released version 0.8.0 it was failing

with:
workdir: fellows-runtimes
chain: ${{ matrix.chain }}
runtime_dir: ${{ matrix.runtime_dir }}
image: ${{ github.event.inputs.image }}
Expand All @@ -91,6 +136,7 @@ jobs:
- name: Store build artifacts to disk
id: cache_digest
if: ${{ steps.cache_runtime.outputs.cache-hit != 'true' }}
working-directory: fellows-runtimes
run: |
cached_output=${{ matrix.runtime_dir }}/target/srtool/release/wbuild/${{ matrix.chain }}-runtime/
digest_file=${cached_output}/${{ matrix.chain }}-srtool-digest.json
Expand All @@ -113,6 +159,7 @@ jobs:
echo "wasm_compressed=$wasm_compressed" >> "$GITHUB_OUTPUT"

- name: Summary
working-directory: fellows-runtimes
run: |
cached_output=${{ matrix.runtime_dir }}/target/srtool/release/wbuild/${{ matrix.chain }}-runtime/
digest_file=${cached_output}/${{ matrix.chain }}-srtool-digest.json
Expand All @@ -127,9 +174,9 @@ jobs:
with:
name: ${{ matrix.chain }}-runtime
path: |
${{ steps.srtool_build.outputs.wasm }}
${{ steps.srtool_build.outputs.wasm_compressed }}
${{ matrix.chain }}-srtool-digest.json
fellows-runtimes/${{ steps.get_values.outputs.wasm }}
fellows-runtimes/${{ steps.get_values.outputs.wasm_compressed }}
fellows-runtimes/${{ matrix.chain }}-srtool-digest.json

# We now get extra information thanks to subwasm,
- name: Install subwasm ${{ env.SUBWASM_VERSION }}
Expand All @@ -139,25 +186,30 @@ jobs:
subwasm --version

- name: Show Runtime information
working-directory: fellows-runtimes
run: |
subwasm info ${{ steps.srtool_build.outputs.wasm }}
subwasm info ${{ steps.srtool_build.outputs.wasm_compressed }}
subwasm --json info ${{ steps.srtool_build.outputs.wasm }} > ${{ matrix.chain }}-info.json
subwasm --json info ${{ steps.srtool_build.outputs.wasm_compressed }} > ${{ matrix.chain }}-info_compressed.json
subwasm info ${{ steps.get_values.outputs.wasm }}
subwasm info ${{ steps.get_values.outputs.wasm_compressed }}
subwasm --json info ${{ steps.get_values.outputs.wasm }} > ${{ matrix.chain }}-info.json
subwasm --json info ${{ steps.get_values.outputs.wasm_compressed }} > ${{ matrix.chain }}-info_compressed.json

- name: Extract the metadata
working-directory: fellows-runtimes
run: |
subwasm meta ${{ steps.srtool_build.outputs.wasm }}
subwasm --json meta ${{ steps.srtool_build.outputs.wasm }} > ${{ matrix.chain }}-metadata.json
subwasm meta ${{ steps.get_values.outputs.wasm }}
subwasm --json meta ${{ steps.get_values.outputs.wasm }} > ${{ matrix.chain }}-metadata.json

- name: Check the metadata diff
working-directory: fellows-runtimes
run: |
subwasm diff ${{ steps.srtool_build.outputs.wasm }} --chain-b ${{ matrix.chain }} | tee ${{ matrix.chain }}-diff.txt
subwasm diff ${{ steps.get_values.outputs.wasm }} --chain-b ${{ matrix.chain }} | tee ${{ matrix.chain }}-diff.txt

- name: Archive Subwasm results
uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8 # v3.1.0
with:
name: ${{ matrix.chain }}-runtime
path: |
${{ matrix.chain }}-info.json
${{ matrix.chain }}-info_compressed.json
${{ matrix.chain }}-metadata.json
${{ matrix.chain }}-diff.txt
fellows-runtimes/${{ matrix.chain }}-info.json
fellows-runtimes/${{ matrix.chain }}-info_compressed.json
fellows-runtimes/${{ matrix.chain }}-metadata.json
fellows-runtimes/${{ matrix.chain }}-diff.txt
42 changes: 26 additions & 16 deletions .github/workflows/manual-polkadot-sdk.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ name: Manual Build - Polkadot SDK

env:
SUBWASM_VERSION: 0.20.0
TOML_CLI_VERSION: 0.2.4

on:
workflow_dispatch:
Expand All @@ -12,6 +13,10 @@ on:
srtool_tag:
description: The SRTOOL tag to use
default: 1.70.0
repo:
description: The repo to be used to build runtimes from
default: paritytech/polkadot-sdk
required: false
ref:
description: The ref to be used for the repo
default: master
Expand All @@ -23,9 +28,10 @@ on:

jobs:
find-runtimes:
name: Scan Polkadot SDK ${{ github.event.inputs.ref }}
name: Scan repo ${{ inputs.repo }} ${{ inputs.ref }}
outputs:
runtime: ${{ steps.get_runtimes_list.outputs.runtime }}
commit_hash: ${{ steps.get_commit_hash.outputs.commit_hash }}
runs-on: ubuntu-latest
steps:
- name: Checkout the srtool repo
Expand All @@ -34,29 +40,33 @@ jobs:
fetch-depth: 0
path: srtool

- name: Cache runtimes list for ${{ github.sha }}
id: cache_runtimes_list
if: github.event.inputs.cache == 'true'
uses: actions/cache@v3
with:
key: runtimes-list-${{ github.sha }}
path: |
sdk/runtimes_list.json

- name: Checkout the Polkadot SDK repo
if: ${{ steps.cache_runtimes_list.outputs.cache-hit != 'true' }}
- name: Checkout repo ${{ inputs.repo }}
uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
with:
repository: paritytech/polkadot-sdk
ref: ${{ github.event.inputs.ref }}
fetch-depth: 0
path: sdk

- name: Get commit hash for ${{ inputs.repo }} ${{ inputs.ref }}
id: get_commit_hash
working-directory: sdk
run: |
echo "commit_hash=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT

- name: Cache runtimes list for ${{ steps.get_commit_hash.outputs.commit_hash }}
id: cache_runtimes_list
if: github.event.inputs.cache == 'true'
uses: actions/cache@v3
with:
key: runtimes-list-${{ steps.get_commit_hash.outputs.commit_hash }}
path: |
sdk/runtimes_list.json

- name: Install tooling
if: ${{ steps.cache_runtimes_list.outputs.cache-hit != 'true' }}
run: |
TOML_CLI_VERSION=0.2.4
URL=https://github.com/chevdor/toml-cli/releases/download/v${TOML_CLI_VERSION}/toml_linux_amd64_v${TOML_CLI_VERSION}.deb
URL=https://github.com/chevdor/toml-cli/releases/download/v${{ env.TOML_CLI_VERSION }}/toml_linux_amd64_v${{ env.TOML_CLI_VERSION }}.deb
curl -L $URL --output toml.deb
sudo dpkg -i toml.deb
toml --version; jq --version
Expand Down Expand Up @@ -109,12 +119,12 @@ jobs:
ls -al
ls -al sdk

- name: Cache ${{ matrix.chain }} runtime for ${{ github.sha }}
- name: Cache ${{ matrix.chain }} runtime for ${{ needs.find-runtimes.outputs.commit_hash }}
id: cache_runtime
if: github.event.inputs.cache == 'true'
uses: actions/cache@v3
with:
key: ${{ matrix.chain }}-${{ github.sha }}
key: ${{ matrix.chain }}-${{ needs.find-runtimes.outputs.commit_hash }}
path: |
sdk/${{ matrix.runtime_dir }}/target/srtool/release/wbuild/${{ matrix.chain }}-runtime

Expand Down
1 change: 1 addition & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
.env
digests
.idea