diff --git a/.github/workflows/extensions.yaml b/.github/workflows/extensions.yaml index 4e95df11..9a5df538 100644 --- a/.github/workflows/extensions.yaml +++ b/.github/workflows/extensions.yaml @@ -36,7 +36,7 @@ jobs: changed_relative_to_ref="origin/${{ github.base_ref || 'not-a-branch' }}" fi echo "changed_relative_to_ref=${changed_relative_to_ref}" >> $GITHUB_OUTPUT - - name: Check out the coredb repo to reuse some actions + - name: Check out the repo to reuse some actions uses: actions/checkout@v4 with: repository: tembo-io/tembo diff --git a/cli/Cargo.toml b/cli/Cargo.toml index e895d018..039fe21b 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pg-trunk" -version = "0.15.7" +version = "0.15.8" edition = "2021" authors = ["Ian Stanton", "Vinícius Miguel", "David E. Wheeler"] description = "A package manager for PostgreSQL extensions" diff --git a/cli/README-DEV.md b/cli/README-DEV.md index bf3361c6..15973589 100644 --- a/cli/README-DEV.md +++ b/cli/README-DEV.md @@ -20,9 +20,65 @@ The `--nocapture` option ensures all output prints to the terminal. The `--test-threads=4` limits the number of threads so as not to overwhelm Docker (seems especially prone with macOS Docker's VM). +## Docker Desktop + +Start a pgxn-tools privileged container with the repository directory mounted: + +```sh +cd trunk +docker run -it --rm -v "$PWD:/repo" --privileged --platform linux/amd64 -w /repo pgxn/pgxn-tools bash +``` + +Inside the container, follow the [installation instructions] to set up the Apt repo: + +``` sh +# Add Docker's official GPG key: +sudo apt-get update +sudo apt-get install ca-certificates curl +sudo install -m 0755 -d /etc/apt/keyrings +sudo curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc +sudo chmod a+r /etc/apt/keyrings/docker.asc + +# Add the repository to Apt sources: +echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian \ + $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \ + sudo tee /etc/apt/sources.list.d/docker.list > /dev/null +sudo apt-get update +``` + +Install Docker: + +``` sh +sudo apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin +``` + +Fix [bug] and start Docker: + +``` sh +perl -i -pe 's/ulimit -Hn/ulimit -n/' /etc/init.d/docker +systemctl start docker +``` + +Install and start Postgres and upgrade Rust: + +``` sh +pg-start 15 +rustup update +``` + +Run the tests: + +``` sh +cd cli +cargo test -- --nocapture --test-threads=4 +``` + [Postgres]: https://www.postgresql.org "PostgreSQL: The World's Most Advanced Open Source Relational Database" [`pg_config`]: https://www.postgresql.org/docs/current/app-pgconfig.html "PostgreSQL Docs: pg_config" [Docker]: https://www.docker.com "Docker: Accelerated Container Application Development" + [installation instructions]: https://docs.docker.com/engine/install/debian/#install-using-the-repository + [bug]: https://forums.docker.com/t/etc-init-d-docker-62-ulimit-error-setting-limit-invalid-argument-problem/139424/2 diff --git a/cli/README.md b/cli/README.md index 550eeff4..2a315b6c 100644 --- a/cli/README.md +++ b/cli/README.md @@ -106,21 +106,31 @@ Trunk can help us build and package these types of extensions as well. Create a custom Dockerfile named `Dockerfile.pg_stat_statements` at the root of the [postgres/contrib](https://github.com/postgres/postgres/tree/master/contrib) repository: + ```dockerfile -ARG PG_VERSION=15 +# Set up variables for build. +ARG PG_VERSION FROM quay.io/coredb/c-builder:pg${PG_VERSION} -USER root -# Postgres build dependencies. Additional system dependencies for the extension can be added here. -# https://wiki.postgresql.org/wiki/Compile_and_Install_from_source_code -RUN apt-get update && apt-get install -y build-essential libreadline-dev zlib1g-dev flex bison libxml2-dev libxslt-dev libssl-dev libxml2-utils xsltproc ccache - -# Copy working directory into container -COPY --chown=postgres:postgres . . -# Necessary step for building extensions in postgres/contrib -RUN ./configure -# Run make in the pg_stat_statements directory -RUN cd contrib/pg_stat_statements && make +# Extension build dependencies +USER root +RUN apt-get update && apt-get install -y \ + build-essential \ + libreadline-dev \ + zlib1g-dev \ + flex bison \ + libxml2-dev \ + libxslt-dev \ + libssl-dev \ + libxml2-utils \ + xsltproc \ + ccache + +# Clone repository and build extension. +ARG EXTENSION_NAME +ARG PG_RELEASE +RUN git clone --depth 1 --branch "${PG_RELEASE}" https://github.com/postgres/postgres.git \ + && make -C postgres/contrib/${EXTENSION_NAME} USE_PGXS=1 ``` Run `trunk build` with `--dockerfile` and `--install-command` flags: @@ -130,16 +140,11 @@ Run `trunk build` with `--dockerfile` and `--install-command` flags: --name pg_stat_statements \ --version 1.10.0 \ --dockerfile Dockerfile.pg_stat_statements \ ---install-command \ -"cd contrib/pg_stat_statements \ -&& make install \ -&& set -x \ -&& mv /usr/local/pgsql/share/extension/* /usr/share/postgresql/15/extension \ -&& mv /usr/local/pgsql/lib/* /usr/lib/postgresql/15/lib" +--install-command make -C contrib/pg_stat_statements USE_PGXS=1 install Building from path . Detected a Makefile, guessing that we are building an extension with 'make', 'make install...' Using Dockerfile at Dockerfile.pg_stat_statements -Using install command /bin/sh -c cd contrib/pg_stat_statements && make install && set -x && mv /usr/local/pgsql/share/extension/* /usr/share/postgresql/15/extension && mv /usr/local/pgsql/lib/* /usr/lib/postgresql/15/lib +Using install command /bin/sh -c make -C contrib/pg_stat_statements USE_PGXS=1 install Building with name pg_stat_statements Building with version 1.10.0 . diff --git a/cli/tests/integration_tests.rs b/cli/tests/integration_tests.rs index a75ac417..d8532623 100644 --- a/cli/tests/integration_tests.rs +++ b/cli/tests/integration_tests.rs @@ -6,11 +6,11 @@ use std::path::{Path, PathBuf}; use std::process::Command; // Run programs use tempfile::TempDir; -const CARGO_BIN: &str = "trunk"; +const TRUNK_BIN: &str = "trunk"; #[test] fn help() -> Result<(), Box> { - let mut cmd = Command::cargo_bin(CARGO_BIN)?; + let mut cmd = Command::cargo_bin(TRUNK_BIN)?; cmd.arg("--help"); cmd.assert().stdout(predicate::str::contains("Usage: ")); @@ -25,7 +25,7 @@ fn install_manifest_v1_extension() -> Result<(), Box> { extension_path.pop(); // Remove the file name from the path extension_path.push("artifact-v1/my_extension-0.0.0.tar.gz"); - let mut cmd = Command::cargo_bin(CARGO_BIN)?; + let mut cmd = Command::cargo_bin(TRUNK_BIN)?; cmd.arg("install"); cmd.arg("--pg-version"); cmd.arg("15"); @@ -53,14 +53,14 @@ fn build_and_install_extension_with_directory_field() -> Result<(), Box Result<(), Box Result<(), Box { + for entry in entries { + match entry { + Ok(entry) => println!(" {:?}", entry.path()), + Err(e) => eprintln!("Error: {}", e), + } + } + } + Err(e) => eprintln!("Error: {}", e), + } + // Make sure files were installed. assert!(sharedir.join("pljava/pljava.control").exists()); - assert!(sharedir.join("pljava/pljava-1.6.5.jar").exists()); - assert!(sharedir.join("pljava/pljava-api-1.6.5.jar").exists()); - assert!(sharedir.join("pljava/pljava--1.6.5.sql").exists()); - assert!(pkglibdir.join("libpljava-so-1.6.5.so").exists()); + assert!(sharedir.join("pljava/pljava-1.6.8.jar").exists()); + assert!(sharedir.join("pljava/pljava-api-1.6.8.jar").exists()); + assert!(sharedir.join("pljava/pljava--1.6.8.sql").exists()); + println!("pkglibdir: {}", pkglibdir.display()); + assert!(pkglibdir.join("libpljava-so-1.6.8.so").exists()); Ok(()) } @@ -110,7 +137,7 @@ fn build_pgrx_extension() -> Result<(), Box> { extension_path.pop(); // Remove the file name from the path extension_path.push("test_pgrx_extension"); - let mut cmd = Command::cargo_bin(CARGO_BIN)?; + let mut cmd = Command::cargo_bin(TRUNK_BIN)?; cmd.arg("build"); cmd.arg("--path"); cmd.arg(extension_path.as_os_str()); @@ -148,7 +175,7 @@ fn build_pgrx_extension() -> Result<(), Box> { } // assert post installation steps contain correct CREATE EXTENSION command - let mut cmd = Command::cargo_bin(CARGO_BIN)?; + let mut cmd = Command::cargo_bin(TRUNK_BIN)?; cmd.arg("install"); cmd.arg("--pg-version"); cmd.arg("15"); @@ -180,7 +207,7 @@ fn build_pgrx_extension_bad_name() -> Result<(), Box> { extension_path.pop(); // Remove the file name from the path extension_path.push("test_pgrx_extension"); - let mut cmd = Command::cargo_bin(CARGO_BIN)?; + let mut cmd = Command::cargo_bin(TRUNK_BIN)?; cmd.arg("build"); cmd.arg("--path"); cmd.arg(extension_path.as_os_str()); @@ -204,7 +231,7 @@ fn build_pgrx_extension_bad_version() -> Result<(), Box> extension_path.pop(); // Remove the file name from the path extension_path.push("test_pgrx_extension"); - let mut cmd = Command::cargo_bin(CARGO_BIN)?; + let mut cmd = Command::cargo_bin(TRUNK_BIN)?; cmd.arg("build"); cmd.arg("--path"); cmd.arg(extension_path.as_os_str()); @@ -242,7 +269,7 @@ fn build_c_extension() -> Result<(), Box> { } .expect("Failed to set HEAD"); - let mut cmd = Command::cargo_bin(CARGO_BIN)?; + let mut cmd = Command::cargo_bin(TRUNK_BIN)?; cmd.arg("build"); cmd.arg("--path"); cmd.arg(repo_dir); @@ -289,35 +316,21 @@ fn build_extension_custom_dockerfile() -> Result<(), Box> let manifest_file = &output_dir.join("manifest.json"); // Example of a C extension requires another build-time requirement - let repo_url = "https://github.com/pramsey/pgsql-http.git"; - // clone and checkout ref v1.5.0 - let repo_dir = &output_dir.join("pgsql-http"); - let repo = Repository::clone(repo_url, repo_dir).unwrap(); - let refname = "v1.5.0"; - let (object, reference) = repo.revparse_ext(refname).expect("Object not found"); - repo.checkout_tree(&object, None) - .expect("Failed to checkout"); - match reference { - // gref is an actual reference like branches or tags - Some(gref) => repo.set_head(gref.name().unwrap()), - // this is a commit, not a reference - None => repo.set_head_detached(object.id()), - } - .expect("Failed to set HEAD"); - let mut dockerfile_path = std::path::PathBuf::from(file!()); dockerfile_path.pop(); // Remove the file name from the path dockerfile_path.push("test_builders"); dockerfile_path.push("Dockerfile.http"); - let mut cmd = Command::cargo_bin(CARGO_BIN)?; + let mut cmd = Command::cargo_bin(TRUNK_BIN)?; cmd.arg("build"); cmd.arg("--path"); - cmd.arg(repo_dir); + cmd.arg(output_dir); cmd.arg("--output-path"); cmd.arg(output_dir); cmd.arg("--dockerfile"); cmd.arg(dockerfile_path.clone()); + cmd.arg("--install-command"); + cmd.arg("make -C pgsql-http install"); cmd.arg("--version"); cmd.arg("1.5.0"); cmd.arg("--name"); @@ -356,7 +369,7 @@ fn build_extension_custom_dockerfile() -> Result<(), Box> } // assert post installation steps contain correct CREATE EXTENSION command - let mut cmd = Command::cargo_bin(CARGO_BIN)?; + let mut cmd = Command::cargo_bin(TRUNK_BIN)?; cmd.arg("install"); cmd.arg("--pg-version"); cmd.arg("15"); @@ -381,37 +394,21 @@ fn build_pg_stat_statements() -> Result<(), Box> { let manifest_file = &output_dir.join("manifest.json"); // Example of a C extension requires another build-time requirement - // clone and checkout postgres REL_15_3 - let repo_url = "https://github.com/postgres/postgres.git"; - let repo_dir = &output_dir.join("postgres_pg_stat_statements"); - let repo = Repository::clone(repo_url, repo_dir).unwrap(); - let refname = "REL_15_3"; - let (object, reference) = repo.revparse_ext(refname).expect("Object not found"); - repo.checkout_tree(&object, None) - .expect("Failed to checkout"); - match reference { - // gref is an actual reference like branches or tags - Some(gref) => repo.set_head(gref.name().unwrap()), - // this is a commit, not a reference - None => repo.set_head_detached(object.id()), - } - .expect("Failed to set HEAD"); - let mut dockerfile_path = std::path::PathBuf::from(file!()); dockerfile_path.pop(); // Remove the file name from the path dockerfile_path.push("test_builders"); dockerfile_path.push("Dockerfile.pg_stat_statements"); - let mut cmd = Command::cargo_bin(CARGO_BIN)?; + let mut cmd = Command::cargo_bin(TRUNK_BIN)?; cmd.arg("build"); cmd.arg("--path"); - cmd.arg(repo_dir); + cmd.arg(output_dir); cmd.arg("--output-path"); cmd.arg(output_dir); cmd.arg("--dockerfile"); cmd.arg(dockerfile_path.clone()); cmd.arg("--install-command"); - cmd.arg("cd contrib/pg_stat_statements && make install && set -x && mv /usr/local/pgsql/share/extension/* /usr/share/postgresql/15/extension && mv /usr/local/pgsql/lib/* /usr/lib/postgresql/15/lib"); + cmd.arg("make -C postgres/contrib/pg_stat_statements USE_PGXS=1 install"); cmd.arg("--version"); cmd.arg("1.10"); cmd.arg("--name"); @@ -449,7 +446,7 @@ fn build_pg_cron_trunk_toml() -> Result<(), Box> { // Set up a temporary directory that will be deleted when the test finishes. let tmp_dir = TempDir::with_prefix("test_pg_cron_trunk_toml_")?; let output_dir = tmp_dir.path(); - let tarball = &output_dir.join("pg_cron-1.5.2-pg15.tar.gz"); + let tarball = &output_dir.join("pg_cron-1.6.4-pg15.tar.gz"); let manifest_file = &output_dir.join("manifest.json"); // Construct a path relative to the current file's directory @@ -458,7 +455,7 @@ fn build_pg_cron_trunk_toml() -> Result<(), Box> { trunkfile_path.push("test_trunk_toml_dirs"); trunkfile_path.push("pg_cron"); - let mut cmd = Command::cargo_bin(CARGO_BIN)?; + let mut cmd = Command::cargo_bin(TRUNK_BIN)?; cmd.arg("build"); cmd.arg("--path"); cmd.arg(trunkfile_path.as_os_str()); @@ -508,7 +505,7 @@ fn build_pg_cron_trunk_toml() -> Result<(), Box> { } // assert post installation steps contain correct CREATE EXTENSION command - let mut cmd = Command::cargo_bin(CARGO_BIN)?; + let mut cmd = Command::cargo_bin(TRUNK_BIN)?; cmd.arg("install"); cmd.arg("--pg-version"); cmd.arg("15"); @@ -550,7 +547,7 @@ fn build_pgrx_with_trunk_toml() -> Result<(), Box> { trunkfile_path.push("test_trunk_toml_dirs"); trunkfile_path.push("pgrx_with_trunk_toml"); - let mut cmd = Command::cargo_bin(CARGO_BIN)?; + let mut cmd = Command::cargo_bin(TRUNK_BIN)?; cmd.arg("build"); cmd.arg("--path"); cmd.arg(trunkfile_path.as_os_str()); @@ -591,7 +588,7 @@ fn build_pgrx_with_trunk_toml() -> Result<(), Box> { return Ok(()); } // assert post installation steps contain correct CREATE EXTENSION command - let mut cmd = Command::cargo_bin(CARGO_BIN)?; + let mut cmd = Command::cargo_bin(TRUNK_BIN)?; cmd.arg("install"); cmd.arg("--pg-version"); cmd.arg("15"); @@ -629,7 +626,7 @@ fn build_pgrx_with_trunk_toml_bad_name() -> Result<(), Box Result<(), Box Result<(), Box> { let manifest_file = &output_dir.join("manifest.json"); // Example of a C extension requires another build-time requirement - let repo_url = "https://github.com/postgres/postgres.git"; - let repo_dir = &output_dir.join("postgres"); - let repo = Repository::clone(repo_url, repo_dir).unwrap(); - let refname = "REL_15_3"; - let (object, reference) = repo.revparse_ext(refname).expect("Object not found"); - repo.checkout_tree(&object, None) - .expect("Failed to checkout"); - match reference { - // gref is an actual reference like branches or tags - Some(gref) => repo.set_head(gref.name().unwrap()), - // this is a commit, not a reference - None => repo.set_head_detached(object.id()), - } - .expect("Failed to set HEAD"); - let mut dockerfile_path = std::path::PathBuf::from(file!()); dockerfile_path.pop(); // Remove the file name from the path dockerfile_path.push("test_builders"); dockerfile_path.push("Dockerfile.auto_explain"); - let mut cmd = Command::cargo_bin(CARGO_BIN)?; + let mut cmd = Command::cargo_bin(TRUNK_BIN)?; cmd.arg("build"); cmd.arg("--path"); - cmd.arg(repo_dir); + cmd.arg(output_dir); cmd.arg("--output-path"); cmd.arg(output_dir); cmd.arg("--dockerfile"); cmd.arg(dockerfile_path.clone()); cmd.arg("--install-command"); - cmd.arg("cd contrib/auto_explain && make install && set -x && mv /usr/local/pgsql/share/extension/* /usr/share/postgresql/15/extension && mv /usr/local/pgsql/lib/* /usr/lib/postgresql/15/lib"); + cmd.arg("make -C postgres/contrib/auto_explain install USE_PGXS=1"); cmd.arg("--version"); cmd.arg("15.3.0"); cmd.arg("--name"); @@ -738,7 +720,7 @@ fn build_auto_explain() -> Result<(), Box> { return Ok(()); } - let mut cmd = Command::cargo_bin(CARGO_BIN)?; + let mut cmd = Command::cargo_bin(TRUNK_BIN)?; cmd.arg("install"); cmd.arg("--pg-version"); cmd.arg("15"); @@ -756,14 +738,14 @@ fn build_pg_unit() -> Result<(), Box> { // Set up a temporary directory that will be deleted when the test finishes. let tmp_dir = TempDir::with_prefix("test_pg_unit_")?; let output_dir = tmp_dir.path(); - let tarball = &output_dir.join("postgresql_unit-7.0.0-pg15.tar.gz"); + let tarball = &output_dir.join("postgresql_unit-7.10.0-pg15.tar.gz"); // Construct a path relative to the current file's directory let mut extension_path = std::path::PathBuf::from(file!()); extension_path.pop(); // Remove the file name from the path extension_path.push("test_postgresql_unit"); - let mut cmd = Command::cargo_bin(CARGO_BIN)?; + let mut cmd = Command::cargo_bin(TRUNK_BIN)?; cmd.arg("build"); cmd.arg("--path"); cmd.arg(extension_path.as_os_str()); @@ -799,7 +781,7 @@ fn build_install_postgis() -> Result<(), Box> { trunkfile_path.push("test_trunk_toml_dirs"); trunkfile_path.push("postgis"); - let mut cmd = Command::cargo_bin(CARGO_BIN)?; + let mut cmd = Command::cargo_bin(TRUNK_BIN)?; cmd.arg("build"); cmd.arg("--path"); cmd.arg(trunkfile_path.as_os_str()); @@ -823,7 +805,7 @@ fn build_install_postgis() -> Result<(), Box> { // Assert we recognize fuzzystrmatch as a dependency and install it // This is a dependency of postgis_tiger_geocoder, which is included in the postgis tar.gz - let mut cmd = Command::cargo_bin(CARGO_BIN)?; + let mut cmd = Command::cargo_bin(TRUNK_BIN)?; cmd.arg("install"); cmd.arg("--pg-version"); cmd.arg("15"); @@ -866,7 +848,7 @@ fn install_extension_dependency_resolution() -> Result<(), Box