Skip to content

Commit 6a0b015

Browse files
aykut-bozkurtaykut-bozkurt
committed
Improve makefile
1 parent aa91e14 commit 6a0b015

File tree

7 files changed

+273
-114
lines changed

7 files changed

+273
-114
lines changed

‎.devcontainer/.env‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,21 +10,21 @@ MINIO_ROOT_PASSWORD=minioadmin
1010

1111
# Azure Blob tests
1212
AZURE_STORAGE_ACCOUNT=devstoreaccount1
13-
AZURE_STORAGE_KEY="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
13+
AZURE_STORAGE_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
1414
AZURE_STORAGE_CONNECTION_STRING="DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1;"
1515
AZURE_STORAGE_ENDPOINT=http://localhost:10000/devstoreaccount1
1616
AZURE_ALLOW_HTTP=true
1717
AZURE_TEST_CONTAINER_NAME=testcontainer
18-
AZURE_TEST_READ_ONLY_SAS="se=2100-05-05&sp=r&sv=2022-11-02&sr=c&sig=YMPFnAHKe9y0o3hFegncbwQTXtAyvsJEgPB2Ne1b9CQ%3D"
19-
AZURE_TEST_READ_WRITE_SAS="se=2100-05-05&sp=rcw&sv=2022-11-02&sr=c&sig=TPz2jEz0t9L651t6rTCQr%2BOjmJHkM76tnCGdcyttnlA%3D"
18+
AZURE_TEST_READ_ONLY_SAS=se=2100-05-05&sp=r&sv=2022-11-02&sr=c&sig=YMPFnAHKe9y0o3hFegncbwQTXtAyvsJEgPB2Ne1b9CQ%3D
19+
AZURE_TEST_READ_WRITE_SAS=se=2100-05-05&sp=rcw&sv=2022-11-02&sr=c&sig=TPz2jEz0t9L651t6rTCQr%2BOjmJHkM76tnCGdcyttnlA%3D
2020

2121
# http(s) tests
2222
ALLOW_HTTP=true
2323
HTTP_ENDPOINT=http://localhost:8080
2424

2525
# GCS tests
2626
GOOGLE_TEST_BUCKET=testbucket
27-
GOOGLE_SERVICE_ACCOUNT_KEY='{"gcs_base_url": "http://localhost:4443","disable_oauth": true,"client_email": "","private_key_id": "","private_key": ""}'
27+
GOOGLE_SERVICE_ACCOUNT_KEY={"gcs_base_url": "http://localhost:4443","disable_oauth": true,"client_email": "","private_key_id": "","private_key": ""}
2828
GOOGLE_SERVICE_ENDPOINT=http://localhost:4443
2929

3030
# Others

‎.github/workflows/ci-arm64.yml‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
name: CI lints and tests on arm64
2+
on:
3+
push:
4+
branches: [ "main" ]
5+
pull_request:
6+
branches: [ "main" ]
7+
8+
concurrency:
9+
group: ${{ github.ref }}-arm64
10+
cancel-in-progress: true
11+
12+
env:
13+
RUST_BACKTRACE: 1
14+
CARGO_INCREMENTAL: 0
15+
RUSTC_WRAPPER: sccache
16+
SCCACHE_DIR: /home/runner/.cache/sccache
17+
18+
jobs:
19+
build-and-test-arm64:
20+
runs-on: [arm64, ubuntu-latest]
21+
strategy:
22+
matrix:
23+
postgres: [ 14, 15, 16, 17 ]
24+
env:
25+
PG_MAJOR: ${{ matrix.postgres }}
26+
PG_CONFIG: /usr/lib/postgresql/${{ matrix.postgres }}/bin/pg_config
27+
28+
steps:
29+
- uses: actions/checkout@v4
30+
31+
- name: Set up sccache
32+
run: |
33+
wget https://github.com/mozilla/sccache/releases/download/v$SCCACHE_VERSION/sccache-v$SCCACHE_VERSION-aarch64-unknown-linux-musl.tar.gz
34+
tar -xzf sccache-v$SCCACHE_VERSION-aarch64-unknown-linux-musl.tar.gz
35+
sudo mv sccache-v$SCCACHE_VERSION-aarch64-unknown-linux-musl/sccache /usr/local/bin
36+
chmod +x /usr/local/bin/sccache
37+
echo "$SCCACHE_SHA256 /usr/local/bin/sccache" | sha256sum --check
38+
env:
39+
SCCACHE_VERSION: 0.8.1
40+
SCCACHE_SHA256: "452cef732b24415493a7c6bca6e13536eb9464593fa87c753b6b7cb4733e9c50"
41+
42+
- name: Set up Rust
43+
uses: dtolnay/rust-toolchain@stable
44+
with:
45+
toolchain: 1.86.0
46+
target: aarch64-unknown-linux-gnu
47+
components: rustfmt, clippy, llvm-tools-preview
48+
49+
- name: Cache cargo registry
50+
uses: actions/cache@v4
51+
continue-on-error: false
52+
with:
53+
path: |
54+
~/.cargo/.crates.toml
55+
~/.cargo/.crates2.json
56+
~/.cargo/bin
57+
~/.cargo/registry/index
58+
~/.cargo/registry/cache
59+
~/.cargo/git/db
60+
key: pg_parquet-rust-cache-${{ runner.arch }}-${{ runner.os }}-${{ hashFiles('Cargo.lock', '.github/workflows/ci.yml') }}
61+
62+
- name: Cache sccache directory
63+
uses: actions/cache@v4
64+
continue-on-error: false
65+
with:
66+
path: ${{ env.SCCACHE_DIR }}
67+
key: pg_parquet-sccache-cache-${{ runner.arch }}-${{ runner.os }}-${{ hashFiles('Cargo.lock', '.github/workflows/ci.yml') }}
68+
69+
- name: Install PostgreSQL
70+
run: |
71+
sudo sh -c 'echo "deb https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
72+
wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
73+
sudo apt-get update
74+
sudo apt-get -y install build-essential libreadline-dev zlib1g-dev flex bison libxml2-dev \
75+
libxslt-dev libssl-dev libxml2-utils xsltproc ccache pkg-config \
76+
gnupg ca-certificates
77+
sudo apt-get -y install postgresql-${{ env.PG_MAJOR }}-postgis-3 \
78+
postgresql-server-dev-${{ env.PG_MAJOR }} \
79+
postgresql-client-${{ env.PG_MAJOR }} \
80+
libpq-dev
81+
echo "export PG_MAJOR=${{ env.PG_MAJOR }}" >> $GITHUB_ENV
82+
83+
- name: Install pgaudit extension
84+
run: |
85+
sudo apt-get install -y postgresql-${{ env.PG_MAJOR }}-pgaudit
86+
87+
- name: Install azure-cli
88+
run: |
89+
curl -sL https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor | sudo tee /etc/apt/keyrings/microsoft.gpg > /dev/null
90+
echo "deb [arch=`dpkg --print-architecture` signed-by=/etc/apt/keyrings/microsoft.gpg] https://packages.microsoft.com/repos/azure-cli/ `lsb_release -cs` main" | sudo tee /etc/apt/sources.list.d/azure-cli.list
91+
sudo apt-get update && sudo apt-get install -y azure-cli
92+
93+
- name: Install pgrx
94+
run: |
95+
cargo install --locked [email protected]
96+
97+
- name: Install cargo-llvm-cov for coverage report
98+
run: cargo install --locked [email protected]
99+
100+
- name: Set up permissions for PostgreSQL
101+
run: |
102+
sudo chmod a+rwx $(/usr/lib/postgresql/${{ env.PG_MAJOR }}/bin/pg_config --pkglibdir) \
103+
$(/usr/lib/postgresql/${{ env.PG_MAJOR }}/bin/pg_config --sharedir)/extension \
104+
/var/run/postgresql/
105+
106+
- name: Check format and lint
107+
run: |
108+
make check-format
109+
make check-lint
110+
111+
- name: Run tests
112+
run: |
113+
make check

.github/workflows/ci.yml renamed to .github/workflows/ci-x64.yml

Lines changed: 12 additions & 83 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
1-
name: CI lints and tests
1+
name: CI lints and tests on x86_64
22
on:
33
push:
44
branches: [ "main" ]
55
pull_request:
66
branches: [ "main" ]
77

88
concurrency:
9-
group: ${{ github.ref }}
9+
group: ${{ github.ref }}-x64
1010
cancel-in-progress: true
1111

1212
env:
@@ -16,13 +16,14 @@ env:
1616
SCCACHE_DIR: /home/runner/.cache/sccache
1717

1818
jobs:
19-
build-and-test:
20-
runs-on: ubuntu-latest
19+
build-and-test-x64:
20+
runs-on: [x64, ubuntu-latest]
2121
strategy:
2222
matrix:
2323
postgres: [ 14, 15, 16, 17 ]
2424
env:
2525
PG_MAJOR: ${{ matrix.postgres }}
26+
PG_CONFIG: /usr/lib/postgresql/${{ matrix.postgres }}/bin/pg_config
2627

2728
steps:
2829
- uses: actions/checkout@v4
@@ -56,21 +57,14 @@ jobs:
5657
~/.cargo/registry/index
5758
~/.cargo/registry/cache
5859
~/.cargo/git/db
59-
key: pg_parquet-rust-cache-${{ runner.os }}-${{ hashFiles('Cargo.lock', '.github/workflows/ci.yml') }}
60+
key: pg_parquet-rust-cache-${{ runner.arch }}-${{ runner.os }}-${{ hashFiles('Cargo.lock', '.github/workflows/ci.yml') }}
6061

6162
- name: Cache sccache directory
6263
uses: actions/cache@v4
6364
continue-on-error: false
6465
with:
6566
path: ${{ env.SCCACHE_DIR }}
66-
key: pg_parquet-sccache-cache-${{ runner.os }}-${{ hashFiles('Cargo.lock', '.github/workflows/ci.yml') }}
67-
68-
- name: Export environment variables from .env file
69-
uses: falti/dotenv-action@v1
70-
with:
71-
path: .devcontainer/.env
72-
export-variables: true
73-
keys-case: bypass
67+
key: pg_parquet-sccache-cache-${{ runner.arch }}-${{ runner.os }}-${{ hashFiles('Cargo.lock', '.github/workflows/ci.yml') }}
7468

7569
- name: Install PostgreSQL
7670
run: |
@@ -96,92 +90,27 @@ jobs:
9690
echo "deb [arch=`dpkg --print-architecture` signed-by=/etc/apt/keyrings/microsoft.gpg] https://packages.microsoft.com/repos/azure-cli/ `lsb_release -cs` main" | sudo tee /etc/apt/sources.list.d/azure-cli.list
9791
sudo apt-get update && sudo apt-get install -y azure-cli
9892
99-
- name: Install and configure pgrx
93+
- name: Install pgrx
10094
run: |
10195
cargo install --locked [email protected]
102-
cargo pgrx init --pg${{ env.PG_MAJOR }} /usr/lib/postgresql/${{ env.PG_MAJOR }}/bin/pg_config \
103-
--base-testing-port $PGRX_TEST_PG_BASE_PORT
10496
10597
- name: Install cargo-llvm-cov for coverage report
10698
run: cargo install --locked [email protected]
10799

108-
- name: Format and lint
109-
run: |
110-
cargo fmt --all -- --check
111-
cargo clippy --all-targets --features "pg${{ env.PG_MAJOR }}, pg_test" --no-default-features -- -D warnings
112-
113100
- name: Set up permissions for PostgreSQL
114101
run: |
115102
sudo chmod a+rwx $(/usr/lib/postgresql/${{ env.PG_MAJOR }}/bin/pg_config --pkglibdir) \
116103
$(/usr/lib/postgresql/${{ env.PG_MAJOR }}/bin/pg_config --sharedir)/extension \
117104
/var/run/postgresql/
118105
119-
- name: Start Minio for s3 emulator tests
120-
run: |
121-
docker run -d \
122-
--env-file .devcontainer/.env \
123-
-p 9000:9000 \
124-
--entrypoint "./entrypoint.sh" \
125-
--volume ./.devcontainer/minio-entrypoint.sh:/entrypoint.sh \
126-
minio/minio
127-
128-
while ! curl $AWS_ENDPOINT_URL; do
129-
echo "Waiting for $AWS_ENDPOINT_URL..."
130-
sleep 1
131-
done
132-
133-
- name: Start Azurite for Azure Blob Storage emulator tests
134-
run: |
135-
docker run -d \
136-
--env-file .devcontainer/.env \
137-
-p 10000:10000 \
138-
mcr.microsoft.com/azure-storage/azurite
139-
140-
while ! curl $AZURE_STORAGE_ENDPOINT; do
141-
echo "Waiting for $AZURE_STORAGE_ENDPOINT..."
142-
sleep 1
143-
done
144-
145-
# create container
146-
az storage container create -n $AZURE_TEST_CONTAINER_NAME --connection-string $AZURE_STORAGE_CONNECTION_STRING
147-
az storage container create -n ${AZURE_TEST_CONTAINER_NAME}2 --connection-string $AZURE_STORAGE_CONNECTION_STRING
148-
149-
- name: Start local web server for http(s) tests
106+
- name: Check format and lint
150107
run: |
151-
docker run -d \
152-
--env-file .devcontainer/.env \
153-
-p 8080:80 \
154-
rclone/rclone serve webdav /data --addr :80
155-
156-
while ! curl $HTTP_ENDPOINT; do
157-
echo "Waiting for $HTTP_ENDPOINT..."
158-
sleep 1
159-
done
160-
161-
- name: Start fake-gcs-server for Google Cloud Storage emulator tests
162-
run: |
163-
docker run -d \
164-
--env-file .devcontainer/.env \
165-
-p 4443:4443 \
166-
tustvold/fake-gcs-server -scheme http -public-host localhost:4443
167-
168-
while ! curl $GOOGLE_SERVICE_ENDPOINT; do
169-
echo "Waiting for $GOOGLE_SERVICE_ENDPOINT..."
170-
sleep 1
171-
done
172-
173-
# create bucket
174-
curl -v -X POST --data-binary "{\"name\":\"$GOOGLE_TEST_BUCKET\"}" -H "Content-Type: application/json" "$GOOGLE_SERVICE_ENDPOINT/storage/v1/b"
175-
curl -v -X POST --data-binary "{\"name\":\"${GOOGLE_TEST_BUCKET}2\"}" -H "Content-Type: application/json" "$GOOGLE_SERVICE_ENDPOINT/storage/v1/b"
108+
make check-format
109+
make check-lint
176110
177111
- name: Run tests
178112
run: |
179-
# Run tests with coverage tool
180-
source <(cargo llvm-cov show-env --export-prefix)
181-
cargo llvm-cov clean
182-
cargo build --features "pg${{ env.PG_MAJOR }}, pg_test" --no-default-features
183-
cargo pgrx test pg${{ env.PG_MAJOR }} --no-default-features
184-
cargo llvm-cov report --lcov > lcov.info
113+
make check-with-coverage
185114
186115
- name: Upload coverage report to Codecov
187116
if: ${{ env.PG_MAJOR }} == 17

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,3 +13,4 @@
1313
*.xml
1414
lcov.info
1515
playground.rs
16+
*.env

‎CONTRIBUTING.md‎

Lines changed: 6 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -91,24 +91,19 @@ testing via `cargo pgrx test pg16`.
9191

9292
# Testing
9393

94-
We run `RUST_TEST_THREADS=1 cargo pgrx test` to run all our unit tests. If you
95-
run a specific test, you can do it via regex patterns like below:
96-
`cargo pgrx run pg17 test_numeric_*`.
94+
Object storage tests are integration tests which require below containers running locally:
95+
- minio container for s3
96+
- azurite container for azure blob storage
97+
- fake_gcs container for google cloud storage
98+
- webdav container for http(s) stores
9799

98-
> [!WARNING]
99-
> Make sure to pass RUST_TEST_THREADS=1 as environment variable to `cargo pgrx test`.
100-
101-
Object storage tests are integration tests which require a storage emulator running
102-
locally. See [ci.yml](.github/workflows/ci.yml) to see how local storage emulators
103-
are started. You can also see the required environment variables from
104-
[.env file](.devcontainer/.env).
100+
Our [Makefile](Makefile) automates setting up all containers and running tests for pg_parquet.
105101

106102
# Format and Lint
107103

108104
We use `cargo-fmt` as formatter and `cargo-clippy` as linter. You can check
109105
how we run them from [ci.yml](.github/workflows/ci.yml).
110106

111-
112107
# Release
113108

114109
We apply semantic versioning for our releases. We do not support long term release branches (backporting) yet.

0 commit comments

Comments
 (0)