diff --git a/.github/workflows/extensions.yaml b/.github/workflows/extensions.yaml index 9a5df538..a3739902 100644 --- a/.github/workflows/extensions.yaml +++ b/.github/workflows/extensions.yaml @@ -57,14 +57,22 @@ jobs: - dind - xlarge-16x16 container: - image: quay.io/tembo/trunk-test-tembo:bdb0905-pg15 + image: quay.io/tembo/trunk-test-tembo:1d073a4-pg15 options: --user root needs: - find_directories strategy: fail-fast: false matrix: - ext: ${{ fromJson(needs.find_directories.outputs.directories).include }} + # ext: ${{ fromJson(needs.find_directories.outputs.directories).include }} + ext: + - { name: pgfincore, path: "./contrib/pgfincore" } + - { name: pgtelemetry, path: "./contrib/pgtelemetry" } + - { name: pljava, path: "./contrib/pljava" } + - { name: postgresql_anonymizer, path: "./contrib/postgresql_anonymizer" } + - { name: emaj, path: "./contrib/emaj" } + - { name: pg_semver, path: "./contrib/pg_semver" } + - { name: rdkit, path: "./contrib/rdkit" } pg: [15] exclude: # pg_hint_plan has separate extensions for each major version @@ -92,14 +100,22 @@ jobs: - dind - xlarge-16x16 container: - image: quay.io/tembo/trunk-test-tembo:bdb0905-pg15 + image: quay.io/tembo/trunk-test-tembo:1d073a4-pg15 options: --user root needs: - find_directories strategy: fail-fast: false matrix: - ext: ${{ fromJson(needs.find_directories.outputs.directories).include }} + # ext: ${{ fromJson(needs.find_directories.outputs.directories).include }} + ext: + - { name: pgfincore, path: "./contrib/pgfincore" } + - { name: pgtelemetry, path: "./contrib/pgtelemetry" } + - { name: pljava, path: "./contrib/pljava" } + - { name: postgresql_anonymizer, path: "./contrib/postgresql_anonymizer" } + - { name: emaj, path: "./contrib/emaj" } + - { name: pg_semver, path: "./contrib/pg_semver" } + - { name: rdkit, path: "./contrib/rdkit" } pg: [17, 16, 15, 14] exclude: - { pg: 14, ext: { name: pg_walinspect } } # Added in Postgres 17 diff --git a/.github/workflows/trunk-install-test.yml b/.github/workflows/trunk-install-test.yml index 7eb64e09..ce8b8e81 100644 --- a/.github/workflows/trunk-install-test.yml +++ b/.github/workflows/trunk-install-test.yml @@ -18,7 +18,7 @@ jobs: - dind - large-8x8 container: - image: quay.io/tembo/trunk-test-tembo:bdb0905-pg15 + image: quay.io/tembo/trunk-test-tembo:1d073a4-pg15 options: --user root env: PGHOST: "localhost" diff --git a/README.md b/README.md index 1ce33b35..ec3b9339 100644 --- a/README.md +++ b/README.md @@ -32,35 +32,46 @@ complexities of extension development and management with the following commands Compile extensions of all kinds. ```shell -❯ trunk build -Building from path . -Detected that we are building a pgrx extension -Detected pgrx version range 0.7.4 -Using pgrx version 0.7.4 -Building pgrx extension at path . +❯ trunk build --pg-version 17 +info: Building from path . +info: Using Dockerfile at ./Dockerfile +info: Using install command /usr/bin/bash -c make -C pg_cron install +Building with name pg_cron +Building with version 1.6.4 +Building for PostgreSQL 17 . . . -Creating package at: ./.trunk/pgmq-0.5.0.tar.gz +Creating package at: ./.trunk/pg_cron-1.6.4-pg17.tar.gz +Fetching extension_name from control file: extension/pg_cron.control +Using extension_name: pg_cron Create Trunk bundle: - pgmq.so - extension/pgmq--0.5.0.sql - extension/pgmq.control + pg_cron.so + licenses/LICENSE + extension/pg_cron--1.0--1.1.sql + extension/pg_cron--1.0.sql + extension/pg_cron--1.1--1.2.sql + extension/pg_cron--1.2--1.3.sql + extension/pg_cron--1.3--1.4.sql + extension/pg_cron--1.4--1.4-1.sql + extension/pg_cron--1.4-1--1.5.sql + extension/pg_cron--1.5--1.6.sql + extension/pg_cron.control manifest.json -Packaged to ./.trunk/pgmq-0.5.0.tar.gz +Packaged to ./.trunk/pg_cron-1.6.4-pg17.tar.gz ``` ### `trunk publish` Publish extensions to the registry, making them available to the Postgres community for discovery and installation. ```shell -❯ trunk publish pgmq \ ---version 0.5.0 \ ---description "Message Queue for postgres" \ ---documentation "https://github.com/tembo-io/pgmq" \ ---repository "https://github.com/tembo-io/pgmq" \ ---license "Apache-2.0" \ ---homepage "https://www.tembo.io" +❯ trunk publish pg_cron \ +--version 1.6.4 \ +--description "Job scheduler for PostgreSQL" \ +--documentation "https://github.com/citusdata/pg_cron" \ +--repository "https://github.com/citusdata/pg_cron" \ +--license "PostgreSQL" \ +--homepage "https://www.citusdata.com/" ``` ### `trunk install` @@ -69,16 +80,39 @@ Downloads Postgres extensions from the Trunk registry and installs into your env Supports nested dependencies, e.g. installing `extension_a` will automatically install `extension_b` if required. ```shell -❯ trunk install pgmq -Using pg_config: /usr/bin/pg_config -Using pkglibdir: "/usr/lib/postgresql/15/lib" -Using sharedir: "/usr/share/postgresql/15" -Downloading from: https://cdb-plat-use1-prod-pgtrunkio.s3.amazonaws.com/extensions/pgmq/pgmq-0.5.0.tar.gz -Dependencies: ["pg_partman"] -Installing pgmq 0.5.0 -[+] pgmq.so => /usr/lib/postgresql/15/lib -[+] extension/pgmq--0.5.0.sql => /usr/share/postgresql/15 -[+] extension/pgmq.control => /usr/share/postgresql/15 +❯ trunk install pg_cron +Using pkglibdir: "/var/lib/postgresql/data/tembo/17/lib" +Using sharedir: "/var/lib/postgresql/data/tembo" +Using Postgres version: 17 +info: Downloading from: https://cdb-plat-use1-prod-pgtrunkio.s3.amazonaws.com/extensions/pg_cron/pg_cron-pg17-1.6.4.tar.gz +info: Dependent extensions to be installed: [] +info: Installing pg_cron 1.6.4 +[+] pg_cron.so => /var/lib/postgresql/data/tembo/17/lib +info: Skipping license file licenses/LICENSE +[+] extension/pg_cron--1.0--1.1.sql => /var/lib/postgresql/data/tembo +[+] extension/pg_cron--1.0.sql => /var/lib/postgresql/data/tembo +[+] extension/pg_cron--1.1--1.2.sql => /var/lib/postgresql/data/tembo +[+] extension/pg_cron--1.2--1.3.sql => /var/lib/postgresql/data/tembo +[+] extension/pg_cron--1.3--1.4.sql => /var/lib/postgresql/data/tembo +[+] extension/pg_cron--1.4--1.4-1.sql => /var/lib/postgresql/data/tembo +[+] extension/pg_cron--1.4-1--1.5.sql => /var/lib/postgresql/data/tembo +[+] extension/pg_cron--1.5--1.6.sql => /var/lib/postgresql/data/tembo +[+] extension/pg_cron.control => /var/lib/postgresql/data/tembo + +*************************** +* POST INSTALLATION STEPS * +*************************** + +Install the following system-level dependencies: + On systems using apt: + libpq5 + libc6 + +Add the following to your postgresql.conf: + shared_preload_libraries = 'pg_cron' + +Enable the extension with: + CREATE EXTENSION IF NOT EXISTS pg_cron CASCADE; ``` ## Trunk Registry - https://pgt.dev @@ -102,7 +136,6 @@ If you're interested in contributing, please open a pull request, issue, or reac - [Twitter](https://twitter.com/tembo_io) - Email us at [hello@tembo.io](mailto:hello@tembo.io) - Thanks goes to these incredible people: diff --git a/cli/README.md b/cli/README.md index 2a315b6c..6ad024e8 100644 --- a/cli/README.md +++ b/cli/README.md @@ -24,42 +24,72 @@ The `build` command allows for compiling and packaging Postgres extensions from ```shell ❯ trunk build --help +Build a PGRX or C based Postgres extension + Usage: trunk build [OPTIONS] Options: - -p, --path [default: .] - -o, --output-path [default: ./.trunk] - --version - --name - --platform - --dockerfile - --install-command - -h, --help Print help + -p, --path + The file path of the extension to build [default: .] + -o, --output-path + + -v, --version + + -n, --name + + -e, --extension_name + + -x, --extension_dependencies + + -s, --preload-libraries + + -P, --platform + + -d, --dockerfile + + -i, --install-command + + -t, --test + Run this extension's integration tests after building, if any are found + --pg-version + The PostgreSQL version to build this extension against. Experimental for versions other than Postgres 15 [default: 15] + -h, --help + Print help ``` ### PGRX Based Extensions + Extensions can be built in many ways, and [PGRX](https://github.com/tcdi/pgrx) allows for us to do so with Rust. Trunk makes building and packaging PGRX based extensions easier than ever. Example `trunk build` with PGRX based extension -[pgmq](https://github.com/tembo-io/coredb/tree/main/pgmq/extension): +[jsonschema](https://pgxn.org/dist/jsonschema/): + ```shell -❯ trunk build -Building from path . -Detected that we are building a pgrx extension -Detected pgrx version range 0.7.4 -Using pgrx version 0.7.4 -Building pgrx extension at path . +❯ trunk build --pg-version 17 +info: Building from path . +info: Using Dockerfile at ./Dockerfile +info: Using install command /usr/bin/bash -c make install +Building with name jsonschema +Building with version 0.1.4 +Building for PostgreSQL 17 . . . -Creating package at: ./.trunk/pgmq-0.5.0.tar.gz +Creating package at: ./.trunk/jsonschema-0.1.4-pg17.tar.gz +Fetching extension_name from control file: extension/jsonschema.control +Using extension_name: jsonschema Create Trunk bundle: - pgmq.so - extension/pgmq--0.5.0.sql - extension/pgmq.control + jsonschema.so + licenses/LICENSE.md + extension/jsonschema--0.1.0--0.1.1.sql + extension/jsonschema--0.1.1--0.1.2.sql + extension/jsonschema--0.1.2--0.1.3.sql + extension/jsonschema--0.1.3--0.1.4.sql + extension/jsonschema--0.1.4.sql + extension/jsonschema.control manifest.json -Packaged to ./.trunk/pgmq-0.5.0.tar.gz +Packaged to ./.trunk/jsonschema-0.1.4-pg17.tar.gz ``` ### C & SQL Based Extensions @@ -68,24 +98,22 @@ Extensions can also be written in C & SQL. Let's take a look at how we can build #### Example `trunk build` with C & SQL based extension [pg_cron](https://github.com/citusdata/pg_cron) ```shell -❯ trunk build --name pg_cron --version 1.5.2 -Building from path . -Detected a Makefile, guessing that we are building an extension with 'make', 'make install...' -Using install command make install +❯ trunk build --pg-version 17 +info: Building from path . +info: Using Dockerfile at ./Dockerfile +info: Using install command /usr/bin/bash -c make -C pg_cron install Building with name pg_cron -Building with version 1.5.2 +Building with version 1.6.4 +Building for PostgreSQL 17 . . . -Creating package at: ./.trunk/pg_cron-1.5.2.tar.gz +Creating package at: ./.trunk/pg_cron-1.6.4-pg17.tar.gz +Fetching extension_name from control file: extension/pg_cron.control +Using extension_name: pg_cron Create Trunk bundle: - bitcode/pg_cron/src/entry.bc - bitcode/pg_cron/src/job_metadata.bc - bitcode/pg_cron/src/misc.bc - bitcode/pg_cron/src/pg_cron.bc - bitcode/pg_cron/src/task_states.bc - bitcode/pg_cron.index.bc pg_cron.so + licenses/LICENSE extension/pg_cron--1.0--1.1.sql extension/pg_cron--1.0.sql extension/pg_cron--1.1--1.2.sql @@ -93,9 +121,10 @@ Create Trunk bundle: extension/pg_cron--1.3--1.4.sql extension/pg_cron--1.4--1.4-1.sql extension/pg_cron--1.4-1--1.5.sql + extension/pg_cron--1.5--1.6.sql extension/pg_cron.control manifest.json -Packaged to ./.trunk/pg_cron-1.5.2.tar.gz +Packaged to ./.trunk/pg_cron-1.6.4-pg17.tar.gz ``` Some extensions are part of larger projects and include Makefiles with references to parent directories. @@ -138,35 +167,40 @@ Run `trunk build` with `--dockerfile` and `--install-command` flags: ```shell ❯ trunk build \ --name pg_stat_statements \ ---version 1.10.0 \ +--version 1.11.0 \ --dockerfile Dockerfile.pg_stat_statements \ --install-command make -C contrib/pg_stat_statements USE_PGXS=1 install -Building from path . -Detected a Makefile, guessing that we are building an extension with 'make', 'make install...' -Using Dockerfile at Dockerfile.pg_stat_statements -Using install command /bin/sh -c make -C contrib/pg_stat_statements USE_PGXS=1 install Building with name pg_stat_statements -Building with version 1.10.0 +Building with version 1.11.0 +info: Building from path . +info: Using Dockerfile.pg_stat_statements +info: Using install command /bin/sh -c make -C postgres/contrib/pg_stat_statements install USE_PGXS=1 +Building for PostgreSQL 17 . . . -Creating package at: ./.trunk/pg_stat_statements-1.10.0.tar.gz +Creating package at: ./.trunk/pg_stat_statements-1.11.0-pg17.tar.gz +Fetching extension_name from control file: extension/pg_stat_statements.control +Using extension_name: pg_stat_statements Create Trunk bundle: -pg_stat_statements.so -extension/pg_stat_statements--1.0--1.1.sql -extension/pg_stat_statements--1.1--1.2.sql -extension/pg_stat_statements--1.2--1.3.sql -extension/pg_stat_statements--1.3--1.4.sql -extension/pg_stat_statements--1.4--1.5.sql -extension/pg_stat_statements--1.4.sql -extension/pg_stat_statements--1.5--1.6.sql -extension/pg_stat_statements--1.6--1.7.sql -extension/pg_stat_statements--1.7--1.8.sql -extension/pg_stat_statements--1.8--1.9.sql -extension/pg_stat_statements--1.9--1.10.sql -extension/pg_stat_statements.control -manifest.json -Packaged to ./.trunk/pg_stat_statements-1.10.0.tar.gz + pg_stat_statements.so + licenses/COPYRIGHT + licenses/COPYRIGHT.~1~ + extension/pg_stat_statements--1.0--1.1.sql + extension/pg_stat_statements--1.1--1.2.sql + extension/pg_stat_statements--1.10--1.11.sql + extension/pg_stat_statements--1.2--1.3.sql + extension/pg_stat_statements--1.3--1.4.sql + extension/pg_stat_statements--1.4--1.5.sql + extension/pg_stat_statements--1.4.sql + extension/pg_stat_statements--1.5--1.6.sql + extension/pg_stat_statements--1.6--1.7.sql + extension/pg_stat_statements--1.7--1.8.sql + extension/pg_stat_statements--1.8--1.9.sql + extension/pg_stat_statements--1.9--1.10.sql + extension/pg_stat_statements.control + manifest.json +Packaged to ./.trunk/pg_stat_statements-1.11.0-pg17.tar.gz ``` ## `trunk publish` @@ -174,21 +208,27 @@ Packaged to ./.trunk/pg_stat_statements-1.10.0.tar.gz The `publish` command allows you to publish your newly-minted Postgres extension to the Trunk registry. ```shell ❯ trunk publish --help -Usage: trunk publish [OPTIONS] --version +Publish a Postgres extension to the Trunk registry + +Usage: trunk publish [OPTIONS] [NAME] Arguments: - + [NAME] Options: + -e, --extension_name + -x, --extension_dependencies + -s, --preload_libraries -v, --version -f, --file -d, --description -D, --documentation -H, --homepage -l, --license - -r, --registry [default: https://registry.pgtrunk.io] + -r, --registry -R, --repository - -h, --help Print help + -c, --category + -h, --help Print help ``` 1. Sign in at the [Trunk registry](https://pgtrunk.io) and click `Profile`. @@ -199,13 +239,13 @@ Options: 4. Run the following from the same directory your extension is in: ```shell -❯ trunk publish pgmq \ ---version 0.5.0 \ ---description "Message Queue for postgres" \ ---documentation "https://tembo-io.github.io/coredb/extensions/pgmq" \ ---repository "https://github.com/tembo-io/coredb" \ ---license "Apache-2.0" \ ---homepage "https://www.coredb.io" +❯ trunk publish pg_cron \ +--version 1.6.4 \ +--description "Job scheduler for PostgreSQL" \ +--documentation "https://github.com/citusdata/pg_cron" \ +--repository "https://github.com/citusdata/pg_cron" \ +--license "PostgreSQL" \ +--homepage "https://www.citusdata.com/" ``` ## `trunk install` @@ -215,6 +255,8 @@ additional extension dependencies that exist in the registry. ```shell ❯ trunk install --help +Install a Postgres extension from the Trunk registry + Usage: trunk install [OPTIONS] Arguments: @@ -223,20 +265,45 @@ Arguments: Options: -p, --pg-config -f, --file - -v, --version [default: latest] - -r, --registry [default: https://registry.pgtrunk.io] - -h, --help Print help + -v, --version [default: latest] + -r, --registry [default: https://registry.pgtrunk.io] + --pg-version The PostgreSQL version for which this extension should be installed. Experimental for versions other than Postgres 15 + -s, --skip-dependencies Skip dependency resolution + -h, --help Print help ``` ```shell -❯ trunk install pgmq -Using pg_config: /usr/bin/pg_config -Using pkglibdir: "/usr/lib/postgresql/15/lib" -Using sharedir: "/usr/share/postgresql/15" -Downloading from: https://cdb-plat-use1-prod-pgtrunkio.s3.amazonaws.com/extensions/pgmq/pgmq-0.5.0.tar.gz -Dependencies: ["pg_partman"] -Installing pgmq 0.5.0 -[+] pgmq.so => /usr/lib/postgresql/15/lib -[+] extension/pgmq--0.5.0.sql => /usr/share/postgresql/15 -[+] extension/pgmq.control => /usr/share/postgresql/15 +❯ trunk install pg_cron +Using pkglibdir: "/var/lib/postgresql/data/tembo/17/lib" +Using sharedir: "/var/lib/postgresql/data/tembo" +Using Postgres version: 17 +info: Downloading from: https://cdb-plat-use1-prod-pgtrunkio.s3.amazonaws.com/extensions/pg_cron/pg_cron-pg17-1.6.4.tar.gz +info: Dependent extensions to be installed: [] +info: Installing pg_cron 1.6.4 +[+] pg_cron.so => /var/lib/postgresql/data/tembo/17/lib +info: Skipping license file licenses/LICENSE +[+] extension/pg_cron--1.0--1.1.sql => /var/lib/postgresql/data/tembo +[+] extension/pg_cron--1.0.sql => /var/lib/postgresql/data/tembo +[+] extension/pg_cron--1.1--1.2.sql => /var/lib/postgresql/data/tembo +[+] extension/pg_cron--1.2--1.3.sql => /var/lib/postgresql/data/tembo +[+] extension/pg_cron--1.3--1.4.sql => /var/lib/postgresql/data/tembo +[+] extension/pg_cron--1.4--1.4-1.sql => /var/lib/postgresql/data/tembo +[+] extension/pg_cron--1.4-1--1.5.sql => /var/lib/postgresql/data/tembo +[+] extension/pg_cron--1.5--1.6.sql => /var/lib/postgresql/data/tembo +[+] extension/pg_cron.control => /var/lib/postgresql/data/tembo + +*************************** +* POST INSTALLATION STEPS * +*************************** + +Install the following system-level dependencies: + On systems using apt: + libpq5 + libc6 + +Add the following to your postgresql.conf: + shared_preload_libraries = 'pg_cron' + +Enable the extension with: + CREATE EXTENSION IF NOT EXISTS pg_cron CASCADE; ``` diff --git a/contrib/mobilitydb/Dockerfile b/contrib/mobilitydb/Dockerfile index 5c7af12e..adb67d6b 100644 --- a/contrib/mobilitydb/Dockerfile +++ b/contrib/mobilitydb/Dockerfile @@ -1,6 +1,6 @@ ARG PG_VERSION # Set up iamge to copy trunk from. -FROM quay.io/tembo/trunk-test-tembo:bdb0905-pg15 AS trunk +FROM quay.io/tembo/trunk-test-tembo:1d073a4-pg15 AS trunk FROM quay.io/coredb/c-builder:pg${PG_VERSION} USER root