feat: Support auto revisions (#422) (#422)

linz · Nov 7, 2024 · 1fe9a72 · 1fe9a72
1 parent e4b9948
commit 1fe9a72
Show file tree

Hide file tree

Showing 12 changed files with 248 additions and 124 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -12,44 +12,44 @@ on:
 jobs:
   lint:
     name: Lint
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
 
     steps:
       - name: Check out repository
         uses: actions/[email protected]
-
-      - name: Install Nix
-        uses: cachix/install-nix-action@v23
-
+      - name: Install python
+        uses: actions/setup-python@v3
       - name: Run pre-commit hooks
-        run: nix-shell --pure --run 'pre-commit run --all-files'
+        uses: pre-commit/[email protected]
 
   test-install-from-source:
     name: Test PostgreSQL ${{ matrix.pg }} source install on Ubuntu ${{ matrix.release }}
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        pg: ['11', '12', '13', '14', '15']
-        release: [focal, jammy]
+        pg: ['12', '13', '14', '15', '16']
+        release: [20.04, 22.04]
     steps:
       - name: Check out repository
         uses: actions/[email protected]
 
       - name: Build Docker container
-        run: docker build --build-arg=RELEASE=${{ matrix.release }} --tag=tester .
+        run:
+          docker build --build-arg=RELEASE=${{ matrix.release }} --build-arg="PG_VERSION=${{
+          matrix.pg }}" --tag=tester .
 
       - name: Install from source
-        run: docker run --rm tester ./test/ci/install-from-source.bash ${{ matrix.pg }}
+        run: docker run --rm tester ./test/ci/install-from-source.bash
 
   test-package-upgrade:
     name: Test PostgreSQL ${{ matrix.pg }} package upgrade on Ubuntu ${{ matrix.release }}
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        pg: ['11', '12', '13', '14'] # TODO: '15'
-        release: [focal] # TODO: jammy
+        pg: ['12', '13', '14', '15', '16']
+        release: [20.04]
     steps:
       - name: Check out repository
         uses: actions/[email protected]
@@ -83,8 +83,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        pg: ['11', '12', '13', '14'] # TODO: '15'
-        release: [focal] # TODO: jammy
+        pg: ['11', '12', '13', '14']
+        release: [20.04]
     steps:
       - name: Check out repository
         uses: actions/[email protected]
@@ -103,7 +103,7 @@ jobs:
       fail-fast: false
       matrix:
         pg: ['11', '12', '13', '14'] # TODO: '15'
-        release: [focal] # TODO: jammy
+        release: [20.04]
     steps:
       - name: Check out repository
         uses: actions/[email protected]
@@ -123,7 +123,7 @@ jobs:
       fail-fast: false
       matrix:
         pg: ['11', '12', '13', '14'] # TODO: '15'
-        release: [focal] # TODO: jammy
+        release: [20.04]
     steps:
       - name: Check out repository
         uses: actions/[email protected]
@@ -143,7 +143,7 @@ jobs:
       fail-fast: false
       matrix:
         pg: ['11', '12', '13', '14'] # TODO: '15'
-        release: [focal] # TODO: jammy
+        release: [20.04]
     steps:
       - name: Check out repository
         uses: actions/[email protected]
@@ -178,7 +178,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        release: [focal, jammy]
+        release: [20.04, 22.04]
       max-parallel: 1
     steps:
       - name: Check out repository
@@ -204,7 +204,6 @@ jobs:
             fi
           fi
           echo "REPO=$REPO" | tee --append $GITHUB_ENV
-
       - name: Build and release package
         uses: linz/linz-software-repository@v15
         with:

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -6,24 +6,18 @@ repos:
     rev: 02c491342ac7c7a4c0617c01ddd51f49010a77f6 # frozen: v2.12.1-beta
     hooks:
       - id: hadolint-docker
-        stages: [commit]
-
-  - repo: https://github.com/nix-community/nixpkgs-fmt
-    rev: 6740ea881d3ac5942d4fbf124f5956b896666c76 # frozen: v1.3.0
-    hooks:
-      - id: nixpkgs-fmt
-        stages: [commit]
+        stages: [pre-commit]
 
   - repo: https://github.com/pre-commit/mirrors-prettier
     rev: cafd5506f18eea191804850dacc0a4264772d59d # frozen: v3.0.0-alpha.4
     hooks:
       - id: prettier
-        stages: [commit]
+        stages: [pre-commit]
         language_version: system
 
   - repo: https://github.com/koalaman/shellcheck-precommit
     rev: 3f77b826548d8dc2d26675f077361c92773b50a7 # frozen: v0.9.0
     hooks:
       - id: shellcheck
-        stages: [commit]
+        stages: [pre-commit]
         args: ['--external-sources']
diff --git a/Dockerfile b/Dockerfile
@@ -2,28 +2,62 @@ ARG RELEASE
 FROM ubuntu:${RELEASE}
 ARG RELEASE
 
+ENV TZ=Pacific/Auckland
+
 SHELL ["/bin/bash", "-o", "errexit", "-o", "nounset", "-o", "pipefail", "-O", "failglob", "-O", "inherit_errexit", "-c"]
 
 # hadolint ignore=DL3008
 RUN apt-get update \
     && apt-get --assume-yes install --no-install-recommends \
+        build-essential \
+        libmodule-build-perl \
         ca-certificates \
+        lsb-release \
         curl \
+        vim \
         git \
         gnupg \
         make \
         jq \
     && rm -rf /var/lib/apt/lists/*
 
 # Enable PostgreSQL package repository
+ARG PG_VERSION=15
 RUN curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
-RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ ${RELEASE}-pgdg main" > /etc/apt/sources.list.d/pgdg.list
+RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list
+# hadolint ignore=DL3008
+RUN apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get -y install --no-install-recommends postgresql-${PG_VERSION} \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+# Trust postgres user connections
+RUN sed -i -e'/^local\s\+all\s\+postgres\s\+peer$/ s/peer/trust/' /etc/postgresql/${PG_VERSION}/main/pg_hba.conf \
+    && sed -i 's/host\s\+all\s\+all\s\+127\.0\.0\.1\/32\s\+scram-sha-256/host\tall\tall\tall\ttrust/g' /etc/postgresql/${PG_VERSION}/main/pg_hba.conf \
+    && echo "listen_addresses = '*'" >> /etc/postgresql/${PG_VERSION}/main/postgresql.conf \
+    && locale-gen en_NZ.UTF-8
+
+# Install pgTap and pg_prove
+# hadolint ignore=DL3003
+RUN git clone https://github.com/theory/pgtap.git \
+    && cd pgtap \
+    && make \
+    && make install \
+    && make clean
+# hadolint ignore=DL3003
+RUN git clone https://github.com/theory/tap-parser-sourcehandler-pgtap.git \
+    && cd tap-parser-sourcehandler-pgtap \
+    && perl Build.PL \
+    && ./Build install
 
 # Enable LINZ package repository
-RUN curl https://packagecloud.io/install/repositories/linz/prod/script.deb.sh > script.deb.sh \
-    && chmod u+x script.deb.sh \
-    && os=ubuntu dist=${RELEASE} ./script.deb.sh \
-    && rm script.deb.sh
+RUN curl -s https://packagecloud.io/install/repositories/linz/prod/script.deb.sh | bash
+
+# hadolint ignore=DL3001
+RUN service postgresql start \
+    && su --command='createuser --superuser root' postgres \
+    && service postgresql stop
+
+ENTRYPOINT ["/bin/sh", "-c" , "service postgresql start && /bin/bash"]
 
-COPY . /src
 WORKDIR /src
diff --git a/doc/table_version.md b/doc/table_version.md
@@ -160,7 +160,7 @@ upgrade it to a properly packaged extension with:
 
     CREATE EXTENSION table_version FROM unpackaged;
 
-## Usage
+## General Usage
 
 Take the following example. We have a table `bar` in schema `foo` and insert some data:
 
@@ -261,6 +261,57 @@ Finally if you would like to remove versioning for the table call:
 
     SELECT table_version.ver_disable_versioning('foo', 'bar');
 
+## Auto revisions
+
+You can if you don't want to call the API functions of `ver_create_revision` and
+`ver_complete_revision` explicitly. This can be useful if your application can't use the call the
+API functions before editing. e.g. ongoing logical replication.
+
+Under this auto-revision mode, revision edits are grouped by transactions.
+
+    CREATE EXTENSION table_version;
+
+    CREATE SCHEMA foo;
+
+    CREATE TABLE foo.bar (
+        id INTEGER NOT NULL PRIMARY KEY,
+        baz TEXT
+    );
+
+    SELECT table_version.ver_enable_versioning('foo', 'bar');
+
+    BEGIN;
+    INSERT INTO foo.bar (id, baz) VALUES (1, 'foo bar 1');
+    INSERT INTO foo.bar (id, baz) VALUES (2, 'foo bar 2');
+    INSERT INTO foo.bar (id, baz) VALUES (3, 'foo bar 3');
+    COMMIT;
+
+    BEGIN;
+    UPDATE foo.bar
+    SET    baz = 'foo bar 1 edit'
+    WHERE  id = 1;
+    COMMIT;
+
+
+    SELECT * FROM table_version.foo_bar_revision;
+
+     _revision_created | _revision_expired | id |       baz
+    -------------------+-------------------+----+----------------
+                  1001 |                   |  2 | foo bar 2
+                  1001 |                   |  3 | foo bar 3
+                  1001 |              1002 |  1 | foo bar 1
+                  1002 |                   |  1 | foo bar 1 edit
+
+    (3 row)
+
+The revision message will be automatically created for you based on the transaction ID.
+
+      id  |       revision_time        |         start_time         | user_name | schema_change |    comment
+    ------+----------------------------+----------------------------+-----------+---------------+---------------
+     1001 | 2024-02-26 22:10:30.751895 | 2024-02-26 22:10:30.758708 | postgres  | f             | Auto Txn 4859
+     1002 | 2024-02-27 08:38:44.548215 | 2024-02-27 08:38:44.556542 | root      | f             | Auto Txn 4860
+    (2 rows)
+
 ## Replicate data using table differences
 
 If you would like to maintain a copy of table data on a remote system this is easily done with this

diff --git a/sql/01-enable_versioning.sql b/sql/01-enable_versioning.sql
@@ -120,14 +120,8 @@ BEGIN
     INTO v_table_has_data;
 
     IF v_table_has_data THEN
-        IF @extschema@._ver_get_reversion_temp_table('_changeset_revision') THEN
-            SELECT
-                max(VER.revision)
-            INTO
-                v_revision
-            FROM
-                _changeset_revision VER;
-
+        IF coalesce(current_setting('table_version.current_revision', TRUE), '') <> '' THEN
+            v_revision := current_setting('table_version.current_revision', TRUE)::INTEGER;
             v_revision_exists := TRUE;
         ELSE
             SELECT @[email protected]_create_revision(

diff --git a/sql/03-create_revision.sql b/sql/03-create_revision.sql
@@ -9,20 +9,36 @@ $$
 DECLARE
     v_revision @extschema@.revision.id%TYPE;
 BEGIN
-    IF @extschema@._ver_get_reversion_temp_table('_changeset_revision') THEN
+    SELECT @extschema@._ver_create_revision(p_comment, p_revision_time, p_schema_change)
+    INTO v_revision;
+    PERFORM set_config('table_version.manual_revision', 't', FALSE);
+    RETURN v_revision;
+END;
+$$ LANGUAGE plpgsql SECURITY DEFINER;
+
+
+
+CREATE OR REPLACE FUNCTION _ver_create_revision(
+    p_comment       TEXT, 
+    p_revision_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, 
+    p_schema_change BOOLEAN DEFAULT FALSE
+) 
+RETURNS INTEGER AS
+$$
+DECLARE
+    v_revision @extschema@.revision.id%TYPE;
+BEGIN
+    IF coalesce(current_setting('table_version.manual_revision', TRUE), '') <> '' AND
+        coalesce(current_setting('table_version.current_revision', TRUE), '') <> ''
+    THEN
         RAISE EXCEPTION 'A revision changeset is still in progress. Please complete the revision before starting a new one';
     END IF;
 
     INSERT INTO @[email protected] (revision_time, schema_change, comment, user_name)
     VALUES (p_revision_time, p_schema_change, p_comment, SESSION_USER)
     RETURNING id INTO v_revision;
-
-    CREATE TEMP TABLE _changeset_revision(
-        revision INTEGER NOT NULL PRIMARY KEY
-    );
-    INSERT INTO _changeset_revision(revision) VALUES (v_revision);
-    ANALYSE _changeset_revision;
-
+
+    PERFORM set_config('table_version.current_revision', v_revision::VARCHAR, FALSE);
     RETURN v_revision;
 END;
 $$ LANGUAGE plpgsql SECURITY DEFINER;
diff --git a/sql/04-complete_revision.sql b/sql/04-complete_revision.sql
@@ -5,22 +5,24 @@ DECLARE
     v_user_name TEXT;
 
 BEGIN
-    IF NOT @extschema@._ver_get_reversion_temp_table('_changeset_revision') THEN
+    IF coalesce(current_setting('table_version.current_revision', TRUE), '') = '' THEN
+        RAISE EXCEPTION 'No in-progress revision';
         RETURN FALSE;
     END IF;
 
     SELECT user_name
-        FROM @[email protected] r, _changeset_revision t
-        WHERE r.id = t.revision
-        INTO v_user_name;
+    FROM @[email protected] r
+    WHERE r.id = current_setting('table_version.current_revision', TRUE)::INTEGER
+    INTO v_user_name;
 
     IF NOT pg_has_role(session_user, v_user_name, 'usage') THEN
         RAISE EXCEPTION 'In-progress revision can only be completed '
                         'by its creator user %', v_user_name;
     END IF;
 
-    DROP TABLE _changeset_revision;
-
+    PERFORM set_config('table_version.current_revision', '', FALSE);
+    PERFORM set_config('table_version.manual_revision', '', FALSE);
+
     RETURN TRUE;
 END;
 $$ LANGUAGE plpgsql SECURITY DEFINER;

diff --git a/sql/05-delete_revision.sql b/sql/05-delete_revision.sql
@@ -22,12 +22,9 @@ BEGIN
         RETURN FALSE;
     END IF;
 
-    IF @extschema@._ver_get_reversion_temp_table('_changeset_revision')
+    IF coalesce(current_setting('table_version.current_revision', TRUE), '') <> ''
     THEN
-        IF EXISTS (
-            SELECT * FROM _changeset_revision
-            WHERE revision = p_revision
-        ) THEN
+        IF current_setting('table_version.current_revision', TRUE)::INTEGER = p_revision THEN
             RAISE WARNING 'Revision % is in progress, please complete it'
                           ' before attempting to delete it.', p_revision;
             RETURN FALSE;