diff --git a/.github/workflows/postgresql.yml b/.github/workflows/postgresql.yml index f66e7ae..835c101 100644 --- a/.github/workflows/postgresql.yml +++ b/.github/workflows/postgresql.yml @@ -92,16 +92,34 @@ jobs: username: ${{ secrets._TEMP_DOCKERHUB_USER }} password: ${{ secrets._TEMP_DOCKERHUB_PASSWORD }} - - name: build and push amd64 image + - name: Build amd64 image locally run: | docker buildx build \ --build-arg GIT_COMMIT=${{ github.sha }} \ - --push \ + --load \ --platform linux/amd64 \ --no-cache-filter trimmed \ --no-cache-filter trimmed-all \ $TAG . + - name: Install slim toolkit + run: | + curl -sL https://raw.githubusercontent.com/slimtoolkit/slim/master/scripts/install-slim.sh | sudo -E bash - + + - name: Slim the image + run: | + # Extract image name from TAG (remove -t prefix) + IMAGE_NAME=$(echo "$TAG" | sed 's/-t //') + chmod +x ./slim-image.sh + ./slim-image.sh "$IMAGE_NAME" "${IMAGE_NAME}-slim" amd64 + # Replace original with slim version + docker tag ${IMAGE_NAME}-slim $IMAGE_NAME + + - name: Push amd64 image + run: | + IMAGE_NAME=$(echo "$TAG" | sed 's/-t //') + docker push $IMAGE_NAME + image_postgresql_arm64: needs: image_postgresql_amd64 runs-on: ubuntu-latest @@ -171,16 +189,34 @@ jobs: username: ${{ secrets._TEMP_DOCKERHUB_USER }} password: ${{ secrets._TEMP_DOCKERHUB_PASSWORD }} - - name: build and push arm64 image + - name: Build arm64 image locally run: | docker buildx build \ --build-arg GIT_COMMIT=${{ github.sha }} \ - --push \ - --platform linux/aarch64 \ + --load \ + --platform linux/arm64 \ --no-cache-filter trimmed \ --no-cache-filter trimmed-all \ $TAG . + - name: Install slim toolkit + run: | + curl -sL https://raw.githubusercontent.com/slimtoolkit/slim/master/scripts/install-slim.sh | sudo -E bash - + + - name: Slim the image + run: | + # Extract image name from TAG (remove -t prefix) + IMAGE_NAME=$(echo "$TAG" | sed 's/-t //') + chmod +x ./slim-image.sh + ./slim-image.sh "$IMAGE_NAME" "${IMAGE_NAME}-slim" arm64 + # Replace original with slim version + docker tag ${IMAGE_NAME}-slim $IMAGE_NAME + + - name: Push arm64 image + run: | + IMAGE_NAME=$(echo "$TAG" | sed 's/-t //') + docker push $IMAGE_NAME + create_manifest: needs: [image_postgresql_amd64, image_postgresql_arm64] runs-on: ubuntu-latest diff --git a/Dockerfile b/Dockerfile index fd95e43..1d185bc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,51 +1,78 @@ - ARG PG_MAJOR=17 +ARG PREV_PG_MAJOR=15 ARG TIMESCALE_VERSION=2.22 -FROM timescale/timescaledb-ha:pg17-ts${TIMESCALE_VERSION} AS trimmed -LABEL maintainer="support@openremote.io" +# Stage 1: Get PostgreSQL ${PREV_PG_MAJOR} binaries for upgrade support +FROM timescale/timescaledb-ha:pg${PG_MAJOR}-ts${TIMESCALE_VERSION}-all AS pg-all USER root -# install fd to find files to speed up chown and chgrp -RUN apt-get update && apt-get install -y fd-find && rm -rf /var/lib/apt/lists/* - -# Give postgres user the same UID and GID as the old alpine postgres image to simplify migration of existing DB -RUN usermod -u 70 postgres \ - && groupmod -g 70 postgres \ - && (fd / -group 1000 -exec chgrp -h postgres {} \; || true) \ - && (fd / -user 1000 -exec chown -h postgres {} \; || true) +ARG PREV_PG_MAJOR -# Set PGDATA to the same location as our old alpine image -RUN mkdir -p /var/lib/postgresql && mv /home/postgres/pgdata/* /var/lib/postgresql/ && chown -R postgres:postgres /var/lib/postgresql +# Strip debug symbols and remove unnecessary files from PG ${PREV_PG_MAJOR} in this stage +# For pg_upgrade we need bin/, lib/, and extension/ (for TimescaleDB upgrade scripts) +RUN find /usr/lib/postgresql/${PREV_PG_MAJOR} -type f -name '*.so*' -exec strip --strip-unneeded {} \; 2>/dev/null || true \ + && find /usr/lib/postgresql/${PREV_PG_MAJOR} -type f -executable -exec strip --strip-unneeded {} \; 2>/dev/null || true \ + && rm -rf /usr/share/postgresql/${PREV_PG_MAJOR}/man \ + /usr/share/postgresql/${PREV_PG_MAJOR}/doc \ + /usr/share/postgresql/${PREV_PG_MAJOR}/contrib -# Add custom entry point (see file header for details) -COPY or-entrypoint.sh / -RUN chmod +x /or-entrypoint.sh +# Stage 2: Prepare the main image with UID/GID changes and cleanup +FROM timescale/timescaledb-ha:pg${PG_MAJOR}-ts${TIMESCALE_VERSION} AS final +LABEL maintainer="support@openremote.io" -# Add custom initdb script(s) -COPY docker-entrypoint-initdb.d/ /docker-entrypoint-initdb.d/ -RUN chmod +x /docker-entrypoint-initdb.d/* +USER root +ARG PREV_PG_MAJOR -# Below is mostly copied from https://github.com/timescale/timescaledb-docker-ha/blob/master/Dockerfile (with OR specific entrypoint, -# workdir and OR env defaults) +# Copy PG ${PREV_PG_MAJOR} bin and lib directories for pg_upgrade +COPY --from=pg-all /usr/lib/postgresql/${PREV_PG_MAJOR}/bin /usr/lib/postgresql/${PREV_PG_MAJOR}/bin +COPY --from=pg-all /usr/lib/postgresql/${PREV_PG_MAJOR}/lib /usr/lib/postgresql/${PREV_PG_MAJOR}/lib +# Copy share files including extensions (needed for TimescaleDB upgrade on old PG before pg_upgrade) +COPY --from=pg-all /usr/share/postgresql/${PREV_PG_MAJOR} /usr/share/postgresql/${PREV_PG_MAJOR} -# Get the -all variant which contains multiple PostgreSQL versions -# According to TimescaleDB docs: "timescale/timescaledb-ha images have the files necessary to run previous versions" -FROM timescale/timescaledb-ha:pg17-ts${TIMESCALE_VERSION}-all AS trimmed-all +# Copy entrypoint scripts +COPY or-entrypoint.sh / +COPY docker-entrypoint-initdb.d/ /docker-entrypoint-initdb.d/ -## Create a smaller Docker image from the builder image -FROM scratch -COPY --from=trimmed / / +# Install fd-find, fix UID/GID, setup directories, strip binaries, and cleanup - all in one layer +RUN apt-get update && apt-get install -y --no-install-recommends fd-find \ + # Give postgres user the same UID and GID as the old alpine postgres image + && usermod -u 70 postgres \ + && groupmod -g 70 postgres \ + && (fdfind . / -group 1000 -exec chgrp -h postgres {} \; 2>/dev/null || true) \ + && (fdfind . / -user 1000 -exec chown -h postgres {} \; 2>/dev/null || true) \ + # Set PGDATA to the same location as our old alpine image + && mkdir -p /var/lib/postgresql \ + && mv /home/postgres/pgdata/* /var/lib/postgresql/ \ + && chown -R postgres:postgres /var/lib/postgresql \ + # Make scripts executable + && chmod +x /or-entrypoint.sh /docker-entrypoint-initdb.d/* \ + # Strip debug symbols from PostgreSQL binaries to reduce size + && find /usr/lib/postgresql -type f -name '*.so*' -exec strip --strip-unneeded {} \; 2>/dev/null || true \ + && find /usr/lib/postgresql -type f -executable -exec strip --strip-unneeded {} \; 2>/dev/null || true \ + # Remove fd-find and clean up + && apt-get purge -y fd-find \ + && apt-get autoremove -y --purge \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* \ + /var/cache/apt/* \ + /var/log/* \ + /usr/share/doc/* \ + /usr/share/man/* \ + /usr/share/info/* \ + /usr/share/lintian/* \ + /usr/share/locale/* \ + /tmp/* \ + /var/tmp/* \ + /root/.cache \ + /home/postgres/.cache \ + /usr/local/lib/pgai \ + /usr/share/postgresql/*/man \ + /usr/share/postgresql/*/doc ARG PG_MAJOR - -## Copy only PostgreSQL 14 and 15 for upgrade support -COPY --from=trimmed-all /usr/lib/postgresql/14 /usr/lib/postgresql/14 -COPY --from=trimmed-all /usr/lib/postgresql/15 /usr/lib/postgresql/15 -COPY --from=trimmed-all /usr/share/postgresql/14 /usr/share/postgresql/14 -COPY --from=trimmed-all /usr/share/postgresql/15 /usr/share/postgresql/15 +ARG PREV_PG_MAJOR # Increment this to indicate that a re-index should be carried out on first startup with existing data; REINDEX can still be overidden # with OR_DISABLE_REINDEX=true @@ -80,6 +107,7 @@ ENV PGROOT=/var/lib/postgresql \ POSTGRES_USER=${POSTGRES_USER:-postgres} \ POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres} \ PG_MAJOR=$PG_MAJOR \ + PREV_PG_MAJOR=$PREV_PG_MAJOR \ OR_REINDEX_COUNTER=${OR_REINDEX_COUNTER} \ OR_DISABLE_REINDEX=${OR_DISABLE_REINDEX:-false} \ POSTGRES_MAX_CONNECTIONS=${POSTGRES_MAX_CONNECTIONS:-50} \ diff --git a/README.md b/README.md index cb77985..0ee4ec2 100644 --- a/README.md +++ b/README.md @@ -1,33 +1,99 @@ # Postgresql docker image -[![build multirach postgresql Docker image and push to it dockerhub](https://github.com/openremote/postgresql/actions/workflows/postgresql.yml/badge.svg)](https://github.com/openremote/postgresql/actions/workflows/postgresql.yml) +[![build multiarch postgresql Docker image and push to dockerhub](https://github.com/openremote/postgresql/actions/workflows/postgresql.yml/badge.svg)](https://github.com/openremote/postgresql/actions/workflows/postgresql.yml) - POSTGIS and TimescaleDB (inc. toolkit for hyperfunctions) image built for aarch64 support using `timescaledev/timescaledb-ha` base image with: +POSTGIS and TimescaleDB (inc. toolkit for hyperfunctions) image built for amd64 and arm64 using `timescale/timescaledb-ha` base image with: - OR specific ENV variables and a healthcheck added - Easy configuration of `max_connections` using `POSTGRES_MAX_CONNECTIONS` environment variable (set to `-1` to disable this setting) - PGDATA path set to match old Alpine image (for ease of DB migration) - POSTGRES user UID and GID changed to match old Alpine image (for ease of DB migration) - Auto upgrade of database with PG major version changes from previous PG major version; can be disabled using - OR_DISABLE_AUTO_UPGRADE=true. + `OR_DISABLE_AUTO_UPGRADE=true` - Auto upgrade of timescaleDB extension when a new version is available in the container; can be disabled using - OR_DISABLE_AUTO_UPGRADE=true. + `OR_DISABLE_AUTO_UPGRADE=true` - OR_DISABLE_REINDEX env variable with associated scripts to determine if a REINDEX of the entire DB should be carried - out at first startup with existing DB (checks whether or not $PGDATA/OR_REINDEX_COUNTER.$OR_REINDEX_COUNTER exists). + out at first startup with existing DB (checks whether or not `$PGDATA/OR_REINDEX_COUNTER.$OR_REINDEX_COUNTER` exists). This is used when a collation change has occurred (glibc version change, muslc <-> glibc) which can break the indexes; migration can either be manually handled or auto handled depending on OR_DISABLE_REINDEX env variable value. - NOTE THAT A REINDEX CAN TAKE A LONG TIME DEPENDING ON THE SIZE OF THE DB! And startup will be delayed until completed + NOTE THAT A REINDEX CAN TAKE A LONG TIME DEPENDING ON THE SIZE OF THE DB! And startup will be delayed until completed. This functionality is intended to simplify migration for basic users; advanced users with large DBs should take care of this themselves. +- **Slimmed images** using [slim toolkit](https://github.com/slimtoolkit/slim) to reduce image size by ~60% -`timescale/timescaledb-ha` image is ubuntu based and only currently supports amd64; they are working on ARM64 support in timescaledev/timescaledb-ha see: +## Local Development -https://github.com/timescale/timescaledb-docker-ha/pull/355 +### Prerequisites -See this issue for POSTGIS base image aarch64 support discussion: +- **Docker** must be installed and running +- **slim toolkit** must be installed for image optimization -https://github.com/postgis/docker-postgis/issues/216 +Install slim toolkit via the install script: +```bash +curl -sL https://raw.githubusercontent.com/slimtoolkit/slim/master/scripts/install-slim.sh | sudo -E bash - +``` -TODO: Switch over to timescale/timescaledb-ha once arm64 supported +Or via Homebrew (macOS): +```bash +brew install docker-slim +``` + +For more installation options, see the [slim toolkit documentation](https://github.com/slimtoolkit/slim#installation). + +### Building the Image + +1. Build the Docker image (replace `17` with desired PostgreSQL major version): +```bash +docker build --build-arg PG_MAJOR=17 -t openremote/postgresql:pg17 . +``` + +2. Slim the image using the provided script: +```bash +# Usage: ./slim-image.sh +# Architecture: amd64 or arm64 (auto-detected if omitted) + +./slim-image.sh openremote/postgresql:pg17 openremote/postgresql:pg17-slim +``` + +3. Optionally replace the original with the slimmed version: +```bash +docker tag openremote/postgresql:pg17-slim openremote/postgresql:pg17 +``` ## Upgrading + ***NOTE: If you change the version of container you use then make sure you have backed up your DB first as this container will try to auto upgrade your DB and/or TimescaleDB extension; this auto upgrade functionality can be disabled using `OR_DISABLE_AUTO_UPGRADE=true`*** + +### Automatic Upgrade + +This image supports automatic upgrades from the previous PostgreSQL major version. When the container starts with an existing database from a supported older version, it will: + +1. Upgrade TimescaleDB extensions on the old PostgreSQL version +2. Run `pg_upgrade` to migrate the database to the new PostgreSQL version +3. Upgrade TimescaleDB extensions on the new PostgreSQL version + +### Manual Upgrade + +If automatic upgrade is not supported for your database version (e.g., skipping multiple major versions), you will need to perform a manual upgrade. Follow these steps: + +1. **Backup your database** using `pg_dump` or `pg_dumpall` +2. **Upgrade TimescaleDB first** (if installed) - this must be done before PostgreSQL upgrade +3. **Use pg_upgrade** to migrate between PostgreSQL versions, or restore from backup to a fresh database + +#### Useful Resources + +- [PostgreSQL pg_upgrade documentation](https://www.postgresql.org/docs/current/pgupgrade.html) +- [TimescaleDB upgrade guide](https://docs.timescale.com/self-hosted/latest/upgrades/) +- [TimescaleDB major upgrade guide](https://docs.timescale.com/self-hosted/latest/upgrades/major-upgrade/) + +#### Example: Manual pg_dump/restore + +```bash +# On the old container, dump the database +docker exec -it pg_dumpall -U postgres > backup.sql + +# Start the new container with a fresh data directory +docker run -d --name new_postgres -v /path/to/new/data:/var/lib/postgresql/data openremote/postgresql:latest + +# Restore the backup +docker exec -i new_postgres psql -U postgres < backup.sql +``` diff --git a/or-entrypoint.sh b/or-entrypoint.sh index de041c5..7a57299 100644 --- a/or-entrypoint.sh +++ b/or-entrypoint.sh @@ -77,6 +77,27 @@ if [ -n "$DATABASE_ALREADY_EXISTS" ]; then echo "---------------------------------------------------------------------------------" fi + # Check if the old DB version is supported for upgrade + if [ "$DB_VERSION" != "$PG_MAJOR" ] && [ "$OR_DISABLE_AUTO_UPGRADE" != "true" ]; then + # Only PREV_PG_MAJOR and PG_MAJOR are supported + if [ "$DB_VERSION" != "$PREV_PG_MAJOR" ] && [ "$DB_VERSION" != "$PG_MAJOR" ]; then + echo "********************************************************************************" + echo "ERROR: Database version ${DB_VERSION} is not supported for automatic upgrade!" + echo "This image only supports upgrading from PostgreSQL ${PREV_PG_MAJOR} to ${PG_MAJOR}." + echo "" + echo "Options:" + echo " 1. Use an intermediate image version that supports upgrading from ${DB_VERSION}" + echo " 2. Manually upgrade the database (see documentation below)" + echo "" + echo "Documentation:" + echo " - OpenRemote PostgreSQL upgrade guide: https://github.com/openremote/postgresql#upgrading" + echo " - PostgreSQL pg_upgrade: https://www.postgresql.org/docs/current/pgupgrade.html" + echo " - TimescaleDB upgrade guide: https://docs.timescale.com/self-hosted/latest/upgrades/" + echo "********************************************************************************" + exit 12 + fi + fi + # STEP 1: Upgrade TimescaleDB on OLD PostgreSQL version (if needed) # This must happen BEFORE pg_upgrade so both old and new PG have the same TS version if [ "$DB_VERSION" != "$PG_MAJOR" ] && [ "$OR_DISABLE_AUTO_UPGRADE" != "true" ]; then diff --git a/slim-image.sh b/slim-image.sh new file mode 100755 index 0000000..1d6cf46 --- /dev/null +++ b/slim-image.sh @@ -0,0 +1,85 @@ +#!/bin/bash +# +# Shared script to slim a PostgreSQL Docker image using slimtoolkit. +# Used by both local development (build_and_slim.sh) and GitHub Actions workflow. +# +# Usage: ./slim-image.sh +# architecture: amd64 or arm64 +# +# Prerequisites: +# - slim toolkit must be installed (https://github.com/slimtoolkit/slim) +# - Install via: curl -sL https://raw.githubusercontent.com/slimtoolkit/slim/master/scripts/install-slim.sh | sudo -E bash - +# - Or via Homebrew: brew install docker-slim + +set -e + +SOURCE_IMAGE=$1 +TARGET_IMAGE=$2 +ARCH=${3:-$(uname -m)} + +if [ -z "$SOURCE_IMAGE" ] || [ -z "$TARGET_IMAGE" ]; then + echo "Usage: $0 [architecture]" + echo " architecture: amd64, arm64, x86_64, aarch64 (default: auto-detect)" + exit 1 +fi + +# Normalize architecture names +case "$ARCH" in + amd64|x86_64) + LIB_ARCH="x86_64-linux-gnu" + ;; + arm64|aarch64) + LIB_ARCH="aarch64-linux-gnu" + ;; + *) + echo "Unsupported architecture: $ARCH" + echo "Supported: amd64, arm64, x86_64, aarch64" + exit 1 + ;; +esac + +echo "Slimming image: $SOURCE_IMAGE -> $TARGET_IMAGE (arch: $ARCH, lib: $LIB_ARCH)" + +slim build --target "$SOURCE_IMAGE" \ + --tag "$TARGET_IMAGE" \ + --http-probe=false \ + --continue-after=15 \ + --expose=5432 \ + --expose=8008 \ + --expose=8081 \ + --include-path=/usr/lib/postgresql \ + --include-path=/usr/lib/${LIB_ARCH} \ + --include-path=/usr/share/postgresql \ + --include-path=/usr/share/proj \ + --include-path=/usr/share/gdal \ + --include-path=/etc/alternatives \ + --preserve-path=/var/lib/postgresql \ + --preserve-path=/docker-entrypoint-initdb.d \ + --preserve-path=/or-entrypoint.sh \ + --preserve-path=/etc/postgresql \ + --preserve-path=/etc/ssl \ + --include-shell \ + --include-bin=/usr/bin/sort \ + --include-bin=/usr/bin/find \ + --include-bin=/usr/bin/xargs \ + --include-bin=/usr/bin/dirname \ + --include-bin=/usr/bin/basename \ + --include-bin=/usr/bin/head \ + --include-bin=/usr/bin/tail \ + --include-bin=/usr/bin/wc \ + --include-bin=/usr/bin/cut \ + --include-bin=/usr/bin/tr \ + --include-bin=/usr/bin/sed \ + --include-bin=/usr/bin/awk \ + --include-bin=/usr/bin/grep \ + --include-bin=/bin/cat \ + --include-bin=/bin/mv \ + --include-bin=/bin/mkdir \ + --include-bin=/bin/chmod \ + --include-bin=/bin/rm \ + --include-bin=/bin/cp \ + --include-bin=/bin/touch \ + --include-bin=/usr/bin/id \ + --include-bin=/usr/bin/env + +echo "Successfully created slimmed image: $TARGET_IMAGE"