Improve makefile (#139)

- [x] Automate spinning up test containers before running tests,
- [x] Use makefile targets in ci workflow,
- [x] Add arm64 ci workflow. (tusvold/fake-gcs-server image is built from source since tusvold/fake-gcs-server container does not work on arm64. already tried qemu but got random crashes)

Author: aykut-bozkurt

.devcontainer/.env

@@ -10,21 +10,21 @@ MINIO_ROOT_PASSWORD=minioadmin
 
 # Azure Blob tests
 AZURE_STORAGE_ACCOUNT=devstoreaccount1
-AZURE_STORAGE_KEY="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
+AZURE_STORAGE_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
 AZURE_STORAGE_CONNECTION_STRING="DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1;"
 AZURE_STORAGE_ENDPOINT=http://localhost:10000/devstoreaccount1
 AZURE_ALLOW_HTTP=true
 AZURE_TEST_CONTAINER_NAME=testcontainer
-AZURE_TEST_READ_ONLY_SAS="se=2100-05-05&sp=r&sv=2022-11-02&sr=c&sig=YMPFnAHKe9y0o3hFegncbwQTXtAyvsJEgPB2Ne1b9CQ%3D"
-AZURE_TEST_READ_WRITE_SAS="se=2100-05-05&sp=rcw&sv=2022-11-02&sr=c&sig=TPz2jEz0t9L651t6rTCQr%2BOjmJHkM76tnCGdcyttnlA%3D"
+AZURE_TEST_READ_ONLY_SAS=se=2100-05-05&sp=r&sv=2022-11-02&sr=c&sig=YMPFnAHKe9y0o3hFegncbwQTXtAyvsJEgPB2Ne1b9CQ%3D
+AZURE_TEST_READ_WRITE_SAS=se=2100-05-05&sp=rcw&sv=2022-11-02&sr=c&sig=TPz2jEz0t9L651t6rTCQr%2BOjmJHkM76tnCGdcyttnlA%3D
 
 # http(s) tests
 ALLOW_HTTP=true
 HTTP_ENDPOINT=http://localhost:8080
 
 # GCS tests
 GOOGLE_TEST_BUCKET=testbucket
-GOOGLE_SERVICE_ACCOUNT_KEY='{"gcs_base_url": "http://localhost:4443","disable_oauth": true,"client_email": "","private_key_id": "","private_key": ""}'
+GOOGLE_SERVICE_ACCOUNT_KEY={"gcs_base_url":"http://localhost:4443","disable_oauth":true,"client_email":"","private_key_id":"","private_key":""}
 GOOGLE_SERVICE_ENDPOINT=http://localhost:4443
 
 # Others

.github/workflows/ci.yml

@@ -1,4 +1,4 @@
-name: CI lints and tests
+name: Build and tests
 on:
   push:
     branches: [ "main" ]
@@ -17,17 +17,27 @@ env:
 
 jobs:
   build-and-test:
-    runs-on: ubuntu-latest
     strategy:
       matrix:
         postgres: [ 14, 15, 16, 17 ]
+        runs_on: [ 'ubuntu-22.04', 'ubuntu-22.04-arm' ]
+        include:
+          - runs_on: ubuntu-22.04
+            arch: x64
+          - runs_on: ubuntu-22.04-arm
+            arch: arm64
+
+    runs-on: ${{ matrix.runs_on }}
+
     env:
       PG_MAJOR: ${{ matrix.postgres }}
+      PG_CONFIG: /usr/lib/postgresql/${{ matrix.postgres }}/bin/pg_config
 
     steps:
       - uses: actions/checkout@v4
 
-      - name: Set up sccache
+      - name: Set up sccache for x86_64
+        if: ${{ matrix.arch == 'x64' }}
         run: |
           wget https://github.com/mozilla/sccache/releases/download/v$SCCACHE_VERSION/sccache-v$SCCACHE_VERSION-x86_64-unknown-linux-musl.tar.gz
           tar -xzf sccache-v$SCCACHE_VERSION-x86_64-unknown-linux-musl.tar.gz
@@ -38,13 +48,34 @@ jobs:
           SCCACHE_VERSION: 0.8.1
           SCCACHE_SHA256: "7203a4dcb3a67f3a0272366d50ede22e5faa3e2a798deaa4d1ea377b51c0ab0c"
 
-      - name: Set up Rust
+      - name: Set up sccache for arm64
+        if: ${{ matrix.arch == 'arm64' }}
+        run: |
+          wget https://github.com/mozilla/sccache/releases/download/v$SCCACHE_VERSION/sccache-v$SCCACHE_VERSION-aarch64-unknown-linux-musl.tar.gz
+          tar -xzf sccache-v$SCCACHE_VERSION-aarch64-unknown-linux-musl.tar.gz
+          sudo mv sccache-v$SCCACHE_VERSION-aarch64-unknown-linux-musl/sccache /usr/local/bin
+          chmod +x /usr/local/bin/sccache
+          echo "$SCCACHE_SHA256  /usr/local/bin/sccache" | sha256sum --check
+        env:
+          SCCACHE_VERSION: 0.8.1
+          SCCACHE_SHA256: "36b2fd1c6c3a104ec1d526edb0533a3827c266054bf4552fb97f524beff6a612"
+
+      - name: Set up Rust for x86_64
+        if: ${{ matrix.arch == 'x64' }}
         uses: dtolnay/rust-toolchain@stable
         with:
           toolchain: 1.86.0
           target: x86_64-unknown-linux-gnu
           components: rustfmt, clippy, llvm-tools-preview
 
+      - name: Set up Rust for arm64
+        if: ${{ matrix.arch == 'arm64' }}
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          toolchain: 1.86.0
+          target: aarch64-unknown-linux-gnu
+          components: rustfmt, clippy, llvm-tools-preview
+
       - name: Cache cargo registry
         uses: actions/cache@v4
         continue-on-error: false
@@ -56,21 +87,14 @@ jobs:
             ~/.cargo/registry/index
             ~/.cargo/registry/cache
             ~/.cargo/git/db
-          key: pg_parquet-rust-cache-${{ runner.os }}-${{ hashFiles('Cargo.lock', '.github/workflows/ci.yml') }}
+          key: pg_parquet-rust-cache-${{ runner.arch }}-${{ runner.os }}-${{ hashFiles('Cargo.lock', '.github/workflows/ci.yml') }}
 
       - name: Cache sccache directory
         uses: actions/cache@v4
         continue-on-error: false
         with:
           path: ${{ env.SCCACHE_DIR }}
-          key: pg_parquet-sccache-cache-${{ runner.os }}-${{ hashFiles('Cargo.lock', '.github/workflows/ci.yml') }}
-
-      - name: Export environment variables from .env file
-        uses: falti/dotenv-action@v1
-        with:
-          path: .devcontainer/.env
-          export-variables: true
-          keys-case: bypass
+          key: pg_parquet-sccache-cache-${{ runner.arch }}-${{ runner.os }}-${{ hashFiles('Cargo.lock', '.github/workflows/ci.yml') }}
 
       - name: Install PostgreSQL
         run: |
@@ -96,95 +120,36 @@ jobs:
           echo "deb [arch=`dpkg --print-architecture` signed-by=/etc/apt/keyrings/microsoft.gpg] https://packages.microsoft.com/repos/azure-cli/ `lsb_release -cs` main" | sudo tee /etc/apt/sources.list.d/azure-cli.list
           sudo apt-get update && sudo apt-get install -y azure-cli
 
-      - name: Install and configure pgrx
+      - name: Install pgrx
         run: |
           cargo install --locked [email protected]
-          cargo pgrx init --pg${{ env.PG_MAJOR }} /usr/lib/postgresql/${{ env.PG_MAJOR }}/bin/pg_config \
-                          --base-testing-port $PGRX_TEST_PG_BASE_PORT
 
       - name: Install cargo-llvm-cov for coverage report
         run: cargo install --locked [email protected]
 
-      - name: Format and lint
-        run: |
-          cargo fmt --all -- --check
-          cargo clippy --all-targets --features "pg${{ env.PG_MAJOR }}, pg_test" --no-default-features -- -D warnings
-
       - name: Set up permissions for PostgreSQL
         run: |
           sudo chmod a+rwx $(/usr/lib/postgresql/${{ env.PG_MAJOR }}/bin/pg_config --pkglibdir)          \
                            $(/usr/lib/postgresql/${{ env.PG_MAJOR }}/bin/pg_config --sharedir)/extension \
                            /var/run/postgresql/
 
-      - name: Start Minio for s3 emulator tests
-        run: |
-          docker run -d \
-            --env-file .devcontainer/.env \
-            -p 9000:9000 \
-            --entrypoint "./entrypoint.sh" \
-            --volume ./.devcontainer/minio-entrypoint.sh:/entrypoint.sh \
-            minio/minio
-
-          while ! curl $AWS_ENDPOINT_URL; do
-            echo "Waiting for $AWS_ENDPOINT_URL..."
-            sleep 1
-          done
-
-      - name: Start Azurite for Azure Blob Storage emulator tests
-        run: |
-          docker run -d \
-            --env-file .devcontainer/.env \
-            -p 10000:10000 \
-            mcr.microsoft.com/azure-storage/azurite
-
-          while ! curl $AZURE_STORAGE_ENDPOINT; do
-            echo "Waiting for $AZURE_STORAGE_ENDPOINT..."
-            sleep 1
-          done
-
-          # create container
-          az storage container create -n $AZURE_TEST_CONTAINER_NAME --connection-string $AZURE_STORAGE_CONNECTION_STRING
-          az storage container create -n ${AZURE_TEST_CONTAINER_NAME}2 --connection-string $AZURE_STORAGE_CONNECTION_STRING
-
-      - name: Start local web server for http(s) tests
+      - name: Check format and lint
         run: |
-          docker run -d \
-            --env-file .devcontainer/.env \
-            -p 8080:80 \
-            rclone/rclone serve webdav /data --addr :80
+          make check-format
+          make check-lint
 
-          while ! curl $HTTP_ENDPOINT; do
-            echo "Waiting for $HTTP_ENDPOINT..."
-            sleep 1
-          done
-
-      - name: Start fake-gcs-server for Google Cloud Storage emulator tests
+      - name: Run tests without coverage
+        if: ${{ env.PG_MAJOR != '17' || matrix.arch != 'x64' }}
         run: |
-          docker run -d \
-            --env-file .devcontainer/.env \
-            -p 4443:4443 \
-            tustvold/fake-gcs-server -scheme http -public-host localhost:4443
-
-          while ! curl $GOOGLE_SERVICE_ENDPOINT; do   
-            echo "Waiting for $GOOGLE_SERVICE_ENDPOINT..."
-            sleep 1
-          done
-
-          # create bucket
-          curl -v -X POST --data-binary "{\"name\":\"$GOOGLE_TEST_BUCKET\"}" -H "Content-Type: application/json" "$GOOGLE_SERVICE_ENDPOINT/storage/v1/b"
-          curl -v -X POST --data-binary "{\"name\":\"${GOOGLE_TEST_BUCKET}2\"}" -H "Content-Type: application/json" "$GOOGLE_SERVICE_ENDPOINT/storage/v1/b"
+          make check
 
-      - name: Run tests
+      - name: Run tests with coverage
+        if: ${{ env.PG_MAJOR == '17' && matrix.arch == 'x64' }}
         run: |
-          # Run tests with coverage tool
-          source <(cargo llvm-cov show-env --export-prefix)
-          cargo llvm-cov clean
-          cargo build --features "pg${{ env.PG_MAJOR }}, pg_test" --no-default-features
-          cargo pgrx test pg${{ env.PG_MAJOR }} --no-default-features
-          cargo llvm-cov report --lcov > lcov.info
+          make check-with-coverage
 
       - name: Upload coverage report to Codecov
-        if: ${{ env.PG_MAJOR }} == 17
+        if: ${{ env.PG_MAJOR == '17' && matrix.arch == 'x64' }}
         uses: codecov/codecov-action@v4
         with:
           fail_ci_if_error: true

.gitignore

@@ -13,3 +13,4 @@
 *.xml
 lcov.info
 playground.rs
+*.env

CONTRIBUTING.md

@@ -91,24 +91,29 @@ testing via `cargo pgrx test pg16`.
 
 # Testing
 
-We run `RUST_TEST_THREADS=1 cargo pgrx test` to run all our unit tests. If you
-run a specific test, you can do it via regex patterns like below:
-`cargo pgrx run pg17 test_numeric_*`.
+Object storage tests are integration tests which require below containers running locally:
+- minio container for s3
+- azurite container for azure blob storage
+- fake_gcs container for google cloud storage
+- webdav container for http(s) stores
 
-> [!WARNING]
-> Make sure to pass RUST_TEST_THREADS=1 as environment variable to `cargo pgrx test`.
+Our [Makefile](Makefile) automates setting up all containers and running tests for pg_parquet.
 
-Object storage tests are integration tests which require a storage emulator running
-locally. See [ci.yml](.github/workflows/ci.yml) to see how local storage emulators
-are started. You can also see the required environment variables from
-[.env file](.devcontainer/.env).
+#### Build Dependencies
+- rustup
+- pgrx
+- postgresql development deps
+
+#### Test Dependencies
+- docker
+- azure cli
+- pgaudit extension
 
 # Format and Lint
 
 We use `cargo-fmt` as formatter and `cargo-clippy` as linter. You can check
 how we run them from [ci.yml](.github/workflows/ci.yml).
 
-
 # Release
 
 We apply semantic versioning for our releases. We do not support long term release branches (backporting) yet.