feat: Support auto revisions

linz · Feb 27, 2024 · bf6c811 · bf6c811
1 parent a110502
commit bf6c811
Show file tree

Hide file tree

Showing 12 changed files with 232 additions and 301 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -17,218 +17,27 @@ jobs:
     steps:
       - name: Check out repository
         uses: actions/[email protected]
-
-      - name: Install Nix
-        uses: cachix/install-nix-action@v21
-
+      - name: Install python
+        uses: actions/setup-python@v3
       - name: Run pre-commit hooks
-        run: nix-shell --pure --run 'pre-commit run --all-files'
+        uses: pre-commit/[email protected]
 
   test-install-from-source:
     name: Test PostgreSQL ${{ matrix.pg }} source install on Ubuntu ${{ matrix.release }}
-    runs-on: ubuntu-20.04
-    strategy:
-      fail-fast: false
-      matrix:
-        pg: ['11', '12', '13', '14', '15']
-        release: [focal, jammy]
-    steps:
-      - name: Check out repository
-        uses: actions/[email protected]
-
-      - name: Build Docker container
-        run: docker build --build-arg=RELEASE=${{ matrix.release }} --tag=tester .
-
-      - name: Install from source
-        run: docker run --rm tester ./test/ci/install-from-source.bash ${{ matrix.pg }}
-
-  test-package-upgrade:
-    name: Test PostgreSQL ${{ matrix.pg }} package upgrade on Ubuntu ${{ matrix.release }}
-    runs-on: ubuntu-20.04
-    strategy:
-      fail-fast: false
-      matrix:
-        pg: ['11', '12', '13', '14'] # TODO: '15'
-        release: [focal] # TODO: jammy
-    steps:
-      - name: Check out repository
-        uses: actions/[email protected]
-        with:
-          fetch-depth: 0
-
-      - name: Build package
-        uses: linz/linz-software-repository@v15
-        with:
-          release: ${{ matrix.release }}
-          packages: jq
-
-      - name: Build Docker container
-        run: docker build --build-arg=RELEASE=${{ matrix.release }} --tag=tester .
-
-      - name: Upgrade from latest release
-        run:
-          docker run --rm --volume="${PWD}/build-area:/packages" tester
-          ./test/ci/package-upgrade.bash ${{ matrix.pg }}
-
-      - name: Archive generated packages
-        uses: actions/[email protected]
-        with:
-          name: PostgreSQL ${{ matrix.pg }} packages for Ubuntu ${{ matrix.release }}
-          path: 'build-area/*.deb'
-        if: failure()
-
-  test-source-upgrade:
-    name: Test PostgreSQL ${{ matrix.pg }} source upgrade on Ubuntu ${{ matrix.release }}
-    runs-on: ubuntu-20.04
-    strategy:
-      fail-fast: false
-      matrix:
-        pg: ['11', '12', '13', '14'] # TODO: '15'
-        release: [focal] # TODO: jammy
-    steps:
-      - name: Check out repository
-        uses: actions/[email protected]
-
-      - name: Build Docker container
-        run: docker build --build-arg=RELEASE=${{ matrix.release }} --tag=tester .
-
-      - name: Upgrade from source
-        run: docker run --rm tester ./test/ci/source-upgrade.bash ${{ matrix.pg }}
-
-  test-source-upgrade-using-loader:
-    name:
-      Test PostgreSQL ${{ matrix.pg }} package upgrade using loader on Ubuntu ${{ matrix.release }}
-    runs-on: ubuntu-20.04
-    strategy:
-      fail-fast: false
-      matrix:
-        pg: ['11', '12', '13', '14'] # TODO: '15'
-        release: [focal] # TODO: jammy
-    steps:
-      - name: Check out repository
-        uses: actions/[email protected]
-
-      - name: Build Docker container
-        run: docker build --build-arg=RELEASE=${{ matrix.release }} --tag=tester .
-
-      - name: Upgrade from latest release
-        run: docker run --rm tester ./test/ci/source-upgrade-using-loader.bash ${{ matrix.pg }}
-
-  test-source-upgrade-using-loader-without-extension-support:
-    name:
-      Test PostgreSQL ${{ matrix.pg }} package upgrade using loader without extension support on
-      Ubuntu ${{ matrix.release }}
-    runs-on: ubuntu-20.04
-    strategy:
-      fail-fast: false
-      matrix:
-        pg: ['11', '12', '13', '14'] # TODO: '15'
-        release: [focal] # TODO: jammy
-    steps:
-      - name: Check out repository
-        uses: actions/[email protected]
-
-      - name: Build Docker container
-        run: docker build --build-arg=RELEASE=${{ matrix.release }} --tag=tester .
-
-      - name: Upgrade from latest release
-        run:
-          docker run --rm tester
-          ./test/ci/source-upgrade-using-loader-without-extension-support.bash ${{ matrix.pg }}
-
-  test-package:
-    name: Test PostgreSQL ${{ matrix.pg }} package install on ${{ matrix.release }}
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     strategy:
       fail-fast: false
       matrix:
-        pg: ['11', '12', '13', '14'] # TODO: '15'
-        release: [focal] # TODO: jammy
+        pg: ['12', '13', '14', '15', '16']
+        release: [20.04, 22.04]
     steps:
       - name: Check out repository
         uses: actions/[email protected]
-        with:
-          fetch-depth: 0
-
-      - name: Build package
-        uses: linz/linz-software-repository@v15
-        with:
-          release: ${{ matrix.release }}
-          packages: jq
 
       - name: Build Docker container
-        run: docker build --build-arg=RELEASE=${{ matrix.release }} --tag=tester .
-
-      - name: Upgrade from latest release
         run:
-          docker run --rm --volume="${PWD}/build-area:/packages" tester
-          ./test/ci/install-local-package.bash ${{ matrix.pg }}
+          docker build --build-arg=RELEASE=${{ matrix.release }} --build-arg="PG_VERSION=${{
+          matrix.pg }}" --tag=tester .
 
-  package:
-    needs:
-      - lint
-      - test-install-from-source
-      - test-package
-      - test-package-upgrade
-      - test-source-upgrade
-      - test-source-upgrade-using-loader
-      - test-source-upgrade-using-loader-without-extension-support
-    name: Package for Ubuntu ${{ matrix.release }}
-    runs-on: ubuntu-20.04
-    strategy:
-      fail-fast: false
-      matrix:
-        release: [focal, jammy]
-      max-parallel: 1
-    steps:
-      - name: Check out repository
-        uses: actions/[email protected]
-        with:
-          fetch-depth: 0
-
-      - name: Determine packagecloud publication target
-        run: |
-          # TODO: it would be nice to turn this into a single-liner in
-          #       github-action syntax
-          echo "GitHub ref: ${{ github.ref }}"
-          echo "GitHub event_name: ${{ github.event_name }}"
-          REPO=
-          if test "${{ github.event_name }}" = 'push'; then
-            if expr "${{ github.ref }}" : "refs/tags/" > /dev/null; then
-              REPO=test
-              git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/$GITHUB_REPOSITORY
-            elif test "${{ github.ref }}" = 'refs/heads/packagecloud' \
-                   -o "${{ github.ref }}" = 'refs/heads/master'
-            then
-              REPO=dev
-            fi
-          fi
-          echo "REPO=$REPO" | tee --append $GITHUB_ENV
-
-      - name: Build and release package
-        uses: linz/linz-software-repository@v15
-        with:
-          release: ${{ matrix.release }}
-          packages: jq
-          packagecloud_token: ${{ secrets.LINZCI_PACKAGECLOUD_TOKEN }}
-          packagecloud_repository: ${{ env.REPO }}
-          push_to_git_remote: origin
-
-  finalise:
-    name: Verify all dependencies passed
-    if: always()
-    needs:
-      - lint
-      - package
-      - test-install-from-source
-      - test-package
-      - test-package-upgrade
-      - test-source-upgrade
-      - test-source-upgrade-using-loader
-      - test-source-upgrade-using-loader-without-extension-support
-    runs-on: ubuntu-22.04
-    steps:
-      - name: Decide whether the needed jobs succeeded or failed
-        uses: re-actors/[email protected]
-        with:
-          jobs: ${{ toJSON(needs) }}
+      - name: Install from source
+        run: docker run --rm tester ./test/ci/install-from-source.bash
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -8,12 +8,6 @@ repos:
       - id: hadolint-docker
         stages: [commit]
 
-  - repo: https://github.com/nix-community/nixpkgs-fmt
-    rev: 6740ea881d3ac5942d4fbf124f5956b896666c76 # frozen: v1.3.0
-    hooks:
-      - id: nixpkgs-fmt
-        stages: [commit]
-
   - repo: https://github.com/pre-commit/mirrors-prettier
     rev: cafd5506f18eea191804850dacc0a4264772d59d # frozen: v3.0.0-alpha.4
     hooks:

diff --git a/Dockerfile b/Dockerfile
@@ -2,28 +2,59 @@ ARG RELEASE
 FROM ubuntu:${RELEASE}
 ARG RELEASE
 
+ENV TZ=Pacific/Auckland
+
 SHELL ["/bin/bash", "-o", "errexit", "-o", "nounset", "-o", "pipefail", "-O", "failglob", "-O", "inherit_errexit", "-c"]
 
 # hadolint ignore=DL3008
 RUN apt-get update \
     && apt-get --assume-yes install --no-install-recommends \
+        build-essential \
+        libmodule-build-perl \
         ca-certificates \
+        lsb-release \
         curl \
+        vim \
         git \
         gnupg \
         make \
         jq \
     && rm -rf /var/lib/apt/lists/*
 
 # Enable PostgreSQL package repository
+ARG PG_VERSION=15
 RUN curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
-RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ ${RELEASE}-pgdg main" > /etc/apt/sources.list.d/pgdg.list
+RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list
+# hadolint ignore=DL3008
+RUN apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get -y install --no-install-recommends postgresql-${PG_VERSION} \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+# Trust postgres user connections
+RUN sed -i -e'/^local\s\+all\s\+postgres\s\+peer$/ s/peer/trust/' /etc/postgresql/${PG_VERSION}/main/pg_hba.conf
+
+# Install pgTap and pg_prove
+# hadolint ignore=DL3003
+RUN git clone https://github.com/theory/pgtap.git \
+    && cd pgtap \
+    && make \
+    && make install \
+    && make clean
+# hadolint ignore=DL3003
+RUN git clone https://github.com/theory/tap-parser-sourcehandler-pgtap.git \
+    && cd tap-parser-sourcehandler-pgtap \
+    && perl Build.PL \
+    && ./Build install
 
 # Enable LINZ package repository
-RUN curl https://packagecloud.io/install/repositories/linz/prod/script.deb.sh > script.deb.sh \
-    && chmod u+x script.deb.sh \
-    && os=ubuntu dist=${RELEASE} ./script.deb.sh \
-    && rm script.deb.sh
+RUN curl -s https://packagecloud.io/install/repositories/linz/prod/script.deb.sh | bash
+
+# hadolint ignore=DL3001
+RUN service postgresql start \
+    && su --command='createuser --superuser root' postgres \
+    && service postgresql stop
+
+ENTRYPOINT ["/bin/sh", "-c" , "service postgresql start && /bin/bash"]
 
-COPY . /src
 WORKDIR /src
diff --git a/doc/table_version.md b/doc/table_version.md
@@ -160,7 +160,7 @@ upgrade it to a properly packaged extension with:
 
     CREATE EXTENSION table_version FROM unpackaged;
 
-## Usage
+## General Usage
 
 Take the following example. We have a table `bar` in schema `foo` and insert some data:
 
@@ -261,6 +261,57 @@ Finally if you would like to remove versioning for the table call:
 
     SELECT table_version.ver_disable_versioning('foo', 'bar');
 
+## Auto revisions
+
+You can if you don't want to call the API functions of `ver_create_revision` and
+`ver_complete_revision` explicitly. This can be useful if your application can't use the call the
+API functions before editing. e.g. ongoing logical replication.
+
+Under this auto-revision mode, revision edits are grouped by transactions.
+
+    CREATE EXTENSION table_version;
+
+    CREATE SCHEMA foo;
+
+    CREATE TABLE foo.bar (
+        id INTEGER NOT NULL PRIMARY KEY,
+        baz TEXT
+    );
+
+    SELECT table_version.ver_enable_versioning('foo', 'bar');
+
+    BEGIN;
+    INSERT INTO foo.bar (id, baz) VALUES (1, 'foo bar 1');
+    INSERT INTO foo.bar (id, baz) VALUES (2, 'foo bar 2');
+    INSERT INTO foo.bar (id, baz) VALUES (3, 'foo bar 3');
+    COMMIT;
+
+    BEGIN;
+    UPDATE foo.bar
+    SET    baz = 'foo bar 1 edit'
+    WHERE  id = 1;
+    COMMIT;
+
+
+    SELECT * FROM table_version.foo_bar_revision;
+
+     _revision_created | _revision_expired | id |       baz
+    -------------------+-------------------+----+----------------
+                  1001 |                   |  2 | foo bar 2
+                  1001 |                   |  3 | foo bar 3
+                  1001 |              1002 |  1 | foo bar 1
+                  1002 |                   |  1 | foo bar 1 edit
+
+    (3 row)
+
+The revision message will be automatically created for you based on the transaction ID.
+
+      id  |       revision_time        |         start_time         | user_name | schema_change |    comment
+    ------+----------------------------+----------------------------+-----------+---------------+---------------
+     1001 | 2024-02-26 22:10:30.751895 | 2024-02-26 22:10:30.758708 | postgres  | f             | Auto Txn 4859
+     1002 | 2024-02-27 08:38:44.548215 | 2024-02-27 08:38:44.556542 | root      | f             | Auto Txn 4860
+    (2 rows)
+
 ## Replicate data using table differences
 
 If you would like to maintain a copy of table data on a remote system this is easily done with this

diff --git a/sql/01-enable_versioning.sql b/sql/01-enable_versioning.sql
@@ -120,14 +120,8 @@ BEGIN
     INTO v_table_has_data;
 
     IF v_table_has_data THEN
-        IF @extschema@._ver_get_reversion_temp_table('_changeset_revision') THEN
-            SELECT
-                max(VER.revision)
-            INTO
-                v_revision
-            FROM
-                _changeset_revision VER;
-
+        IF coalesce(current_setting('table_version.current_revision', TRUE), '') <> '' THEN
+            v_revision := current_setting('table_version.current_revision', TRUE)::INTEGER;
             v_revision_exists := TRUE;
         ELSE
             SELECT @[email protected]_create_revision(