debian and almalinux images

Author: aykut-bozkurt

.devcontainer/.env

@@ -2,7 +2,7 @@
 AWS_ACCESS_KEY_ID=minioadmin
 AWS_SECRET_ACCESS_KEY=minioadmin
 AWS_REGION=us-east-1
-AWS_ENDPOINT_URL=http://localhost:9000
+AWS_ENDPOINT_URL=http://minio:9000
 AWS_ENDPOINT_PROXY_PORT=9999
 AWS_ALLOW_HTTP=true
 AWS_S3_TEST_BUCKET=testbucket
@@ -12,24 +12,44 @@ MINIO_ROOT_PASSWORD=minioadmin
 # Azure Blob tests
 AZURE_STORAGE_ACCOUNT=devstoreaccount1
 AZURE_STORAGE_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
-AZURE_STORAGE_CONNECTION_STRING="DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1;"
-AZURE_STORAGE_ENDPOINT=http://localhost:10000/devstoreaccount1
+AZURE_STORAGE_CONNECTION_STRING="DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1;"
+AZURE_STORAGE_ENDPOINT=http://azurite:10000/devstoreaccount1
 AZURE_ALLOW_HTTP=true
 AZURE_TEST_CONTAINER_NAME=testcontainer
 AZURE_TEST_READ_ONLY_SAS=se=2100-05-05&sp=r&sv=2022-11-02&sr=c&sig=YMPFnAHKe9y0o3hFegncbwQTXtAyvsJEgPB2Ne1b9CQ%3D
 AZURE_TEST_READ_WRITE_SAS=se=2100-05-05&sp=rcw&sv=2022-11-02&sr=c&sig=TPz2jEz0t9L651t6rTCQr%2BOjmJHkM76tnCGdcyttnlA%3D
 
 # http(s) tests
 ALLOW_HTTP=true
-HTTP_ENDPOINT=http://localhost:8080
+HTTP_ENDPOINT=http://webdav:8080
 
 # GCS tests
 GOOGLE_TEST_BUCKET=testbucket
-GOOGLE_SERVICE_ACCOUNT_KEY={"gcs_base_url":"http://localhost:4443","disable_oauth":true,"client_email":"","private_key_id":"","private_key":""}
-GOOGLE_SERVICE_ENDPOINT=http://localhost:4443
+GOOGLE_SERVICE_ACCOUNT_KEY={"gcs_base_url":"http://fake-gcs-server:4443","disable_oauth":true,"client_email":"","private_key_id":"","private_key":""}
+GOOGLE_SERVICE_ENDPOINT=http://fake-gcs-server:4443
 
 # Others
+
 # run pgrx tests with a single thread to avoid race conditions
 RUST_TEST_THREADS=1
+
 # pgrx runs test on the port base_port + pg_version
 PGRX_TEST_PG_BASE_PORT=5454
+
+# default PostgreSQL version
+DEFAULT_PG_MAJOR=18
+
+# Rust version
+RUSTC_VERSION=1.90.0
+
+# rust build flags
+RUST_BACKTRACE=1
+CARGO_INCREMENTAL=0
+
+# Docker images
+MINIO_IMAGE=docker.io/minio/minio
+AZURITE_IMAGE=mcr.microsoft.com/azure-storage/azurite
+FAKE_GCS_IMAGE=docker.io/tustvold/fake-gcs-server
+WEBDAV_IMAGE=docker.io/rclone/rclone
+BASE_IMAGE_OS=almalinux
+BASE_IMAGE_TAG=9

.devcontainer/Dockerfile

@@ -1,48 +1,140 @@
-FROM ubuntu:22.04
+ARG BASE_IMAGE_OS=almalinux
+ARG BASE_IMAGE_OS_VERSION=9
+FROM ${BASE_IMAGE_OS}:${BASE_IMAGE_OS_VERSION}
+ARG BASE_IMAGE_OS
+ARG BASE_IMAGE_OS_VERSION
 
 ENV DEBIAN_FRONTEND="noninteractive"
 ENV TZ="Europe/Istanbul"
 
-ARG PG_MAJOR=18
+ARG DEFAULT_PG_MAJOR=18
+
+SHELL ["/bin/bash", "-c"]
 
 # install deps
-RUN apt-get update && apt-get -y install build-essential libreadline-dev zlib1g-dev      \
-                                         flex bison libxml2-dev libxslt-dev libssl-dev   \
-                                         libxml2-utils xsltproc ccache pkg-config wget   \
-                                         curl lsb-release ca-certificates gnupg sudo git \
-                                         nano net-tools awscli libkrb5-dev
+RUN if [ "${BASE_IMAGE_OS}" = "debian" ]; then \
+      apt-get update && apt-get -y install build-essential libreadline-dev zlib1g-dev \
+                                    flex bison libxml2-dev libxslt-dev libssl-dev     \
+                                    libxml2-utils xsltproc ccache pkg-config wget     \
+                                    curl lsb-release ca-certificates gnupg sudo git   \
+                                    nano net-tools awscli libkrb5-dev python3.11 pip; \
+    fi
+RUN if [ "${BASE_IMAGE_OS}" = "almalinux" ]; then \
+      dnf -y update && \
+      dnf -y install epel-release && \
+      dnf config-manager --enable crb && \
+      dnf -y install gcc gcc-c++ make readline-devel zlib-devel which     \
+                     flex bison libxml2-devel libxslt-devel openssl-devel \
+                     ccache pkgconfig wget lsb_release ca-certificates    \
+                     gnupg sudo git nano net-tools awscli krb5-devel      \
+                     python3.11 pip;                                      \
+    fi
+
+# Upgrade pip and uv
+RUN if [ "$BASE_IMAGE_OS" = "almalinux" ]; then \
+        python3.11 -m ensurepip --upgrade && \
+        python3.11 -m pip install --upgrade pip && \
+        python3.11 -m pip install uv; \
+    fi
+RUN if [ "$BASE_IMAGE_OS" = "debian" ]; then \
+        python3.11 -m pip install --upgrade pip --break-system-packages && \
+        python3.11 -m pip install uv --break-system-packages; \
+    fi
+
+ENV PATH="~/.local/bin:$PATH"
+
+# Install uv (we run mitmproxy with it)
+RUN curl -LsSf https://astral.sh/uv/install.sh | sh
 
 # install azure-cli
-RUN curl -sL https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor | tee /etc/apt/keyrings/microsoft.gpg > /dev/null
-RUN echo "deb [arch=`dpkg --print-architecture` signed-by=/etc/apt/keyrings/microsoft.gpg] https://packages.microsoft.com/repos/azure-cli/ `lsb_release -cs` main" | tee /etc/apt/sources.list.d/azure-cli.list
-RUN apt-get update && apt-get install -y azure-cli
+RUN if [ "${BASE_IMAGE_OS}" = "debian" ]; then \
+      curl -sL https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor | tee /etc/apt/keyrings/microsoft.gpg > /dev/null && \
+      echo "deb [arch=`dpkg --print-architecture` signed-by=/etc/apt/keyrings/microsoft.gpg] https://packages.microsoft.com/repos/azure-cli/ `lsb_release -cs` main" | tee /etc/apt/sources.list.d/azure-cli.list && \
+      apt-get update && apt-get install -y azure-cli; \
+    fi
+RUN if [ "${BASE_IMAGE_OS}" = "almalinux" ]; then \
+      rpm --import https://packages.microsoft.com/keys/microsoft.asc && \
+      dnf install -y https://packages.microsoft.com/config/rhel/${BASE_IMAGE_OS_VERSION}/packages-microsoft-prod.rpm && \
+      dnf install -y azure-cli; \
+    fi
 
 # install Postgres
-RUN sh -c 'echo "deb https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
-RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
-RUN apt-get update && apt-get -y install postgresql-${PG_MAJOR}-postgis-3  \
-                                         postgresql-server-dev-${PG_MAJOR} \
-                                         postgresql-client-${PG_MAJOR}     \
-                                         libpq-dev
+RUN if [ "${BASE_IMAGE_OS}" = "debian" ]; then \
+      sh -c 'echo "deb https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' && \
+      wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - && \
+      apt-get update && apt-get -y install postgresql-{14,15,16,17,18}-postgis-3  \
+                                           postgresql-{14,15,16,17,18}-pgaudit    \
+                                           postgresql-server-dev-{14,15,16,17,18} \
+                                           postgresql-client-{14,15,16,17,18}     \
+                                           libpq-dev;                             \
+    fi
+RUN if [ "${BASE_IMAGE_OS}" = "almalinux" ]; then \
+      ARCH=$(uname -m) && \
+      dnf install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-$(rpm -E %{rhel})-${ARCH}/pgdg-redhat-repo-latest.noarch.rpm && \
+      dnf -qy module disable postgresql && \
+      dnf install -y --nogpgcheck \
+                     postgis36_{14,15,16,17,18}        \
+                     pgaudit{16_14,17_15,_16,_17,_18}  \
+                     postgresql{14,15,16,17,18}-libs   \
+                     postgresql{14,15,16,17,18}-devel; \
+    fi
+
+# default pg version for debian
+ENV PATH="/usr/lib/postgresql/${DEFAULT_PG_MAJOR}/bin:${PATH}"
+
+# default pg version for almalinux
+ENV PATH="/usr/pgsql-${DEFAULT_PG_MAJOR}/bin/:${PATH}"
 
 # set up permissions so that rust user can create extensions
-RUN chmod a+rwx `pg_config --pkglibdir`          \
-                `pg_config --pkglibdir`/bitcode  \
-                `pg_config --sharedir`/extension \
-                /var/run/postgresql/
-
-# install pgaudit extension
-RUN if [ ${PG_MAJOR} != "18" ]; then \
-      apt-get -y install postgresql-${PG_MAJOR}-pgaudit; \
-    else \
-      git clone --branch integration https://github.com/pgaudit/pgaudit.git /tmp/pgaudit && \
-      cd /tmp/pgaudit && make install USE_PGXS=1 PG_CONFIG=pg_config; \
+RUN if [ "${BASE_IMAGE_OS}" = "debian" ]; then \
+        chmod a+rwx `/usr/lib/postgresql/14/bin/pg_config --pkglibdir`          \
+                    `/usr/lib/postgresql/14/bin/pg_config --pkglibdir`/bitcode  \
+                    `/usr/lib/postgresql/14/bin/pg_config --sharedir`/extension \
+                    /var/run/postgresql/ && \
+        chmod a+rwx `/usr/lib/postgresql/15/bin/pg_config --pkglibdir`          \
+                    `/usr/lib/postgresql/15/bin/pg_config --pkglibdir`/bitcode  \
+                    `/usr/lib/postgresql/15/bin/pg_config --sharedir`/extension \
+                    /var/run/postgresql/ && \
+        chmod a+rwx `/usr/lib/postgresql/16/bin/pg_config --pkglibdir`          \
+                    `/usr/lib/postgresql/16/bin/pg_config --pkglibdir`/bitcode  \
+                    `/usr/lib/postgresql/16/bin/pg_config --sharedir`/extension \
+                    /var/run/postgresql/ && \
+        chmod a+rwx `/usr/lib/postgresql/17/bin/pg_config --pkglibdir`          \
+                    `/usr/lib/postgresql/17/bin/pg_config --pkglibdir`/bitcode  \
+                    `/usr/lib/postgresql/17/bin/pg_config --sharedir`/extension \
+                    /var/run/postgresql/ && \
+        chmod a+rwx `/usr/lib/postgresql/18/bin/pg_config --pkglibdir`          \
+                    `/usr/lib/postgresql/18/bin/pg_config --pkglibdir`/bitcode  \
+                    `/usr/lib/postgresql/18/bin/pg_config --sharedir`/extension \
+                    /var/run/postgresql/; \
+    fi
+RUN if [ "${BASE_IMAGE_OS}" = "almalinux" ]; then \
+        chmod a+rwx `/usr/pgsql-14/bin/pg_config --pkglibdir`          \
+                    `/usr/pgsql-14/bin/pg_config --pkglibdir`/bitcode  \
+                    `/usr/pgsql-14/bin/pg_config --sharedir`/extension \
+                    /var/run/postgresql/ && \
+        chmod a+rwx `/usr/pgsql-15/bin/pg_config --pkglibdir`          \
+                    `/usr/pgsql-15/bin/pg_config --pkglibdir`/bitcode  \
+                    `/usr/pgsql-15/bin/pg_config --sharedir`/extension \
+                    /var/run/postgresql/ && \
+        chmod a+rwx `/usr/pgsql-16/bin/pg_config --pkglibdir`          \
+                    `/usr/pgsql-16/bin/pg_config --pkglibdir`/bitcode  \
+                    `/usr/pgsql-16/bin/pg_config --sharedir`/extension \
+                    /var/run/postgresql/ && \
+        chmod a+rwx `/usr/pgsql-17/bin/pg_config --pkglibdir`          \
+                    `/usr/pgsql-17/bin/pg_config --pkglibdir`/bitcode  \
+                    `/usr/pgsql-17/bin/pg_config --sharedir`/extension \
+                    /var/run/postgresql/ && \
+        chmod a+rwx `/usr/pgsql-18/bin/pg_config --pkglibdir`          \
+                    `/usr/pgsql-18/bin/pg_config --pkglibdir`/bitcode  \
+                    `/usr/pgsql-18/bin/pg_config --sharedir`/extension \
+                    /var/run/postgresql/; \
     fi
 
 # initdb requires non-root user. This will also be the user that runs the container.
 ARG USERNAME=rust
-ARG USER_UID=1000
-ARG USER_GID=1000
+ARG USER_UID=1001
+ARG USER_GID=1001
 RUN groupadd --gid $USER_GID $USERNAME
 RUN useradd --uid $USER_UID --gid $USER_GID -s /bin/bash -m $USERNAME
 
@@ -51,14 +143,45 @@ RUN echo "$USERNAME ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/$USERNAME
 USER $USERNAME
 
 # install Rust environment
+ARG RUSTC_VERSION=1.90.0
 RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | bash -s -- -y
+
 ENV PATH="/home/rust/.cargo/bin:${PATH}"
 
+RUN rustup install ${RUSTC_VERSION} && \
+    rustup default ${RUSTC_VERSION} && \
+    rustup component add rustfmt clippy llvm-tools-preview
+
+# install llvm-cov for coverage reports
+RUN cargo install --locked [email protected]
+
 # install and configure pgrx
 ARG PGRX_VERSION=0.16.1
 RUN cargo install --locked cargo-pgrx@${PGRX_VERSION}
-RUN cargo pgrx init --pg${PG_MAJOR} $(which pg_config)
-RUN echo "shared_preload_libraries = 'pgaudit,pg_parquet'" >> $HOME/.pgrx/data-${PG_MAJOR}/postgresql.conf
+
+ENV PGRX_TEST_PG_BASE_PORT=5454
+RUN if [ "${BASE_IMAGE_OS}" = "debian" ]; then \
+        cargo pgrx init --pg14 /usr/lib/postgresql/14/bin/pg_config \
+                        --pg15 /usr/lib/postgresql/15/bin/pg_config \
+                        --pg16 /usr/lib/postgresql/16/bin/pg_config \
+                        --pg17 /usr/lib/postgresql/17/bin/pg_config \
+                        --pg18 /usr/lib/postgresql/18/bin/pg_config \
+                        --base-testing-port ${PGRX_TEST_PG_BASE_PORT}; \
+    fi
+RUN if [ "${BASE_IMAGE_OS}" = "almalinux" ]; then \
+        cargo pgrx init --pg14 /usr/pgsql-14/bin/pg_config \
+                        --pg15 /usr/pgsql-15/bin/pg_config \
+                        --pg16 /usr/pgsql-16/bin/pg_config \
+                        --pg17 /usr/pgsql-17/bin/pg_config \
+                        --pg18 /usr/pgsql-18/bin/pg_config \
+                        --base-testing-port ${PGRX_TEST_PG_BASE_PORT}; \
+    fi
+
+RUN echo "shared_preload_libraries = 'pgaudit,pg_parquet'" >> $HOME/.pgrx/data-14/postgresql.conf && \
+    echo "shared_preload_libraries = 'pgaudit,pg_parquet'" >> $HOME/.pgrx/data-15/postgresql.conf && \
+    echo "shared_preload_libraries = 'pgaudit,pg_parquet'" >> $HOME/.pgrx/data-16/postgresql.conf && \
+    echo "shared_preload_libraries = 'pgaudit,pg_parquet'" >> $HOME/.pgrx/data-17/postgresql.conf && \
+    echo "shared_preload_libraries = 'pgaudit,pg_parquet'" >> $HOME/.pgrx/data-18/postgresql.conf
 
 # required for pgrx to work
 ENV USER=$USERNAME

.devcontainer/README.md

@@ -0,0 +1,23 @@
+## Devcontainer
+If you install devcontainer extension at vscode, you can spin up your development environment via pg_parquet devcontainer. All necessary components and dependencies would be installed in your environment after the spin-up. You can work in the container as if it is your development environment. (git, make, aws cli and postgres commands should work seamlessly) The container offers reproducible environment as we use the same dockerfile for CI and devcontainer.
+
+## Prerequisites
+1. Make sure to allocate >= ~10GB memory resource to docker.
+2. Your `~/.aws` folder is mounted as a read only volume to pg_parquet container so that it can read/write buckets with your aws credentials. You are expected to set up your aws credentials before. (possible to read/write from/to **production** s3 buckets)
+
+After opening the project inside the devcontainer, you can do the following steps:
+
+1. Connect to the postgres server (build takes place at the first run)
+```bash
+cargo pgrx run
+```
+2. Create pg_parquet extension
+```sql
+pg_parquet=# create extension pg_parquet;
+CREATE EXTENSION
+```
+3. Copy test data to parquet file
+```sql
+pg_parquet=# copy (select i from generate_series(1, 10) i) to '/tmp/test.parquet';
+COPY 10
+```

.devcontainer/devcontainer.json

@@ -1,9 +1,10 @@
 {
     "name": "pg_parquet Dev Environment",
     "dockerComposeFile": "docker-compose.yml",
-    "service": "app",
+    "service": "pg_parquet",
     "workspaceFolder": "/workspace",
-    "postAttachCommand": "sudo chown -R rust /workspace",
+    "remoteUser": "rust",
+    "containerUser": "rust",
     "customizations": {
         "vscode": {
             "extensions": [