Merge pull request #1249 from bcgov/test-rook-AS-FOIMOD-3636

Test rook as foimod 3636

api/migrations/versions/374a9a0b07e8_.py

@@ -0,0 +1,47 @@
+"""empty message
+
+Revision ID: 374a9a0b07e8
+Revises: ae4f8e0d58b2
+Create Date: 2024-12-13 14:52:49.688416
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '374a9a0b07e8'
+down_revision = 'ae4f8e0d58b2'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('DocumentExtractionJob',   
+    sa.Column('extractionjobid', sa.Integer(), primary_key=True, autoincrement=True, nullable=False),
+    sa.Column('version', sa.Integer(), nullable=False),
+    sa.Column('documentid', sa.Integer(), nullable=False),        
+    sa.Column('ministryrequestid', sa.Integer(), nullable=False),        
+    sa.Column('status', sa.String(length=120), nullable=False),
+    sa.Column('message', sa.Text(), nullable=True),
+    sa.Column('createdat', sa.TIMESTAMP, nullable=False, server_default=sa.func.now()),
+    sa.Column('createdby', sa.String(length=120), nullable=True), 
+    sa.PrimaryKeyConstraint('extractionjobid'),
+    )
+
+    op.execute(
+        'CREATE INDEX "idx_documentextractionjob_multicolumn" ON "DocumentExtractionJob" (extractionjobid, version, documentid, ministryrequestid, status);'
+    )
+    op.execute(
+        'ALTER TABLE "DocumentExtractionJob" ADD CONSTRAINT "unique_documentextractionjob" UNIQUE (extractionjobid, version, documentid, ministryrequestid, status);'
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.execute('DROP INDEX "idx_documentextractionjob_multicolumn";')
+    op.execute('ALTER TABLE "DocumentExtractionJob" DROP CONSTRAINT "unique_documentextractionjob";')
+    op.drop_table('DocumentExtractionJob')
+    # ### end Alembic commands ###

api/migrations/versions/ae4f8e0d58b2_.py

@@ -0,0 +1,32 @@
+"""empty message
+
+Revision ID: ae4f8e0d58b2
+Revises: 9d45ce57481e
+Create Date: 2024-12-13 12:24:11.094340
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = 'ae4f8e0d58b2'
+down_revision = '9d45ce57481e'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.execute('UPDATE public."DocumentStatus" SET name=\'pushedtoqueue\', description= \'Pushed to queue\' WHERE name in (\'inprogress\');commit;')
+    op.execute('UPDATE public."DocumentStatus" SET name=\'extractioncompleted\', description= \'Extraction completed\' WHERE name in (\'saved\');commit;')
+    op.execute('INSERT INTO public."DocumentStatus"(name, description, isactive) VALUES (\'failed\', \'Failed\', true);commit;')
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.execute('UPDATE public."DocumentStatus" SET name=\'inprogress\', description=\'In progress\' WHERE name in (\'pushedtoqueue\');commit;')
+    op.execute('UPDATE public."DocumentStatus" SET name=\'saved\', description=\'Saved\' WHERE name in (\'extractioncompleted\');commit;')
+    op.execute('delete from public."DocumentStatus" WHERE name in (\'failed\');commit;')
+    # ### end Alembic commands ###

openshift/templates/documentextraction/push-to-queue/Dockerfile

@@ -0,0 +1,24 @@
+# For more information, please refer to https://aka.ms/vscode-docker-python
+# FROM python:3.8
+
+# Necessary to pull images from bcgov and not hit Dockerhub quotas.
+FROM artifacts.developer.gov.bc.ca/docker-remote/python:3.8.5-buster
+EXPOSE 6402
+
+# Keeps Python from generating .pyc files in the container
+ENV PYTHONDONTWRITEBYTECODE=1
+
+# Turns off buffering for easier container logging
+ENV PYTHONUNBUFFERED=1
+
+COPY ./ /app
+WORKDIR /app
+RUN pip3 install --no-cache-dir -r requirements.txt
+
+# Creates a non-root user and adds permission to access the /app folder
+# For more info, please refer to https://aka.ms/vscode-docker-python-configure-containers
+#RUN useradd appuser && chown -R appuser /app
+#USER appuser
+
+RUN chmod u+x /app/entrypoint.sh
+ENTRYPOINT ["/bin/sh", "/app/entrypoint.sh"]

openshift/templates/documentextraction/push-to-queue/docextractionqueuescheduler_secrets.yaml

@@ -0,0 +1,69 @@
+apiVersion: template.openshift.io/v1
+kind: Template
+metadata:
+  name: ${NAME}
+  labels:
+    app: ${NAME}
+    name: ${NAME}
+objects:
+  - apiVersion: v1
+    kind: Secret
+    metadata:
+      name: ${NAME}
+    stringData:    
+      DOCREVIEWER_DB_HOST: "${DOCREVIEWER_DB_HOST}"
+      DOCREVIEWER_DB_NAME: "${DOCREVIEWER_DB_NAME}"
+      DOCREVIEWER_DB_USER: '${DOCREVIEWER_DB_USER}'
+      DOCREVIEWER_DB_PASSWORD: '${DOCREVIEWER_DB_PASSWORD}'
+      DOCREVIEWER_DB_PORT: '${DOCREVIEWER_DB_PORT}'
+      FOI_DB_HOST: '${FOI_DB_HOST}'
+      FOI_DB_NAME: '${FOI_DB_NAME}'
+      FOI_DB_PORT: '${FOI_DB_PORT}'
+      FOI_DB_USER: '${FOI_DB_USER}'
+      FOI_DB_PASSWORD: '${FOI_DB_PASSWORD}'
+      ACTIVEMQ_USERNAME: '${ACTIVEMQ_USERNAME}'
+      ACTIVEMQ_PASSWORD: '${ACTIVEMQ_PASSWORD}'
+
+    type: Opaque
+
+parameters:
+  - name: NAME
+    description: The name for all created objects.
+    required: true
+    value: docextractionqueuescheduler-secret
+  - name: DOCREVIEWER_DB_HOST
+    description: DOCREVIEWER_DB_HOST
+    required: true
+  - name: DOCREVIEWER_DB_NAME
+    description: DOCREVIEWER_DB_NAME
+    required: true
+  - name: DOCREVIEWER_DB_USER
+    description: DOCREVIEWER_DB_USER
+    required: true
+  - name: DOCREVIEWER_DB_PASSWORD
+    description: DOCREVIEWER_DB_PASSWORD
+    required: true
+  - name: DOCREVIEWER_DB_PORT
+    description: DOCREVIEWER_DB_PORT
+    required: true
+  - name: FOI_DB_HOST
+    description: FOI_DB_HOST
+    required: true
+  - name: FOI_DB_NAME
+    description: FOI_DB_NAME
+    required: true
+  - name: FOI_DB_PORT
+    description: FOI_DB_PORT
+    required: true  
+  - name: FOI_DB_USER
+    description: FOI_DB_USER
+    required: true
+  - name: FOI_DB_PASSWORD
+    description: FOI_DB_PASSWORD
+    required: true
+  - name: ACTIVEMQ_USERNAME
+    description: ACTIVEMQ_USERNAME
+    required: true
+  - name: ACTIVEMQ_PASSWORD
+    description: ACTIVEMQ_PASSWORD
+    required: true

openshift/templates/documentextraction/push-to-queue/docextractionqueuescheduler_secrets_param.yaml

@@ -0,0 +1,12 @@
+DOCREVIEWER_DB_HOST=
+DOCREVIEWER_DB_NAME=
+DOCREVIEWER_DB_USER=
+DOCREVIEWER_DB_PASSWORD=
+DOCREVIEWER_DB_PORT=
+FOI_DB_USER=
+FOI_DB_PASSWORD=
+FOI_DB_NAME=
+FOI_DB_HOST=
+FOI_DB_PORT=
+ACTIVEMQ_USERNAME=
+ACTIVEMQ_PASSWORD=

openshift/templates/documentextraction/push-to-queue/dockerfile.local

@@ -0,0 +1,21 @@
+# For more information, please refer to https://aka.ms/vscode-docker-python
+FROM python:3.8
+EXPOSE 6402
+
+# Keeps Python from generating .pyc files in the container
+ENV PYTHONDONTWRITEBYTECODE=1
+
+# Turns off buffering for easier container logging
+ENV PYTHONUNBUFFERED=1
+
+COPY ./ /app
+WORKDIR /app
+RUN pip3 install --no-cache-dir -r requirements.txt
+
+# Creates a non-root user and adds permission to access the /app folder
+# For more info, please refer to https://aka.ms/vscode-docker-python-configure-containers
+#RUN useradd appuser && chown -R appuser /app
+#USER appuser
+
+RUN chmod u+x /app/entrypoint.sh
+ENTRYPOINT ["/bin/sh", "/app/entrypoint.sh"]

openshift/templates/documentextraction/push-to-queue/entrypoint.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+
+# python manage.py db upgrade && python wsgi.py
+python queue-documents.py

openshift/templates/documentextraction/push-to-queue/pushtoqueue-build.yaml

@@ -0,0 +1,101 @@
+---
+kind: Template
+apiVersion: v1
+metadata:
+  name: "${NAME}-build-template"
+  creationTimestamp: 
+objects:
+- kind: ImageStream
+  apiVersion: v1
+  metadata:
+    name: "${NAME}"
+- kind: BuildConfig
+  apiVersion: v1
+  metadata:
+    name: "${NAME}-build"
+    labels:
+      app: "${NAME}-build"
+  spec:
+    runPolicy: Serial
+    source:
+      type: Git
+      git:
+        uri: "${GIT_REPO_URL}"
+        ref: "${GIT_REF}"
+      contextDir: "${SOURCE_CONTEXT_DIR}"
+      # sourceSecret:
+      #   name: github-token
+    strategy:
+      type: Docker
+      dockerStrategy:
+        dockerfilePath: "${DOCKER_FILE_PATH}"
+        # pullSecret: 
+        #   name: artifacts-pull-default-jmhvkc
+      # type: Source
+      # sourceStrategy:
+      #   from:
+      #     kind: ImageStreamTag
+      #     namespace: openshift
+      #     name: 'python:3.8-ubi7'
+    output:
+      to:
+        kind: ImageStreamTag
+        name: "${NAME}:${OUTPUT_IMAGE_TAG}"
+parameters:
+- name: NAME
+  displayName: Name
+  description: The name assigned to all of the resources defined in this template.
+  required: true
+  value: docextraction-queue-scheduler
+- name: GIT_REPO_URL
+  displayName: Git Repo URL
+  description: The URL to your GIT repo.
+  required: true
+  value: https://github.com/bcgov/foi-docreviewer
+- name: GIT_REF
+  displayName: Git Reference
+  description: The git reference or branch.
+  required: true
+  value: test-rook-AS-FOIMOD-3636
+- name: SOURCE_CONTEXT_DIR
+  displayName: Source Context Directory
+  description: The source context directory.
+  required: false
+  value: openshift/templates/documentextraction/push-to-queue
+- name: DOCKER_FILE_PATH
+  displayName: Docker File Path
+  description: The path to the docker file defining the build.
+  required: false
+  value: "Dockerfile"
+- name: OUTPUT_IMAGE_TAG
+  displayName: Output Image Tag
+  description: The tag given to the built image.
+  required: true
+  value: latest
+# - name: SOURCE_IMAGE_KIND
+#   displayName: Source Image Kind
+#   description: The 'kind' (type) of the  source image; typically ImageStreamTag, or
+#     DockerImage.
+#   required: true
+#   value: ImageStreamTag
+# - name: SOURCE_IMAGE_NAME
+#   displayName: Source Image Name
+#   description: The name of the source image.
+#   required: true
+#   value: adjudication-api
+# - name: SOURCE_IMAGE_TAG
+#   displayName: Source Image Tag
+#   description: The tag of the source image.
+#   required: true
+#   value: dev
+# - name: OUTPUT_IMAGE_TAG
+#   displayName: Output Image Tag
+#   description: The tag given to the built image.
+#   required: true
+#   value: latest
+# - name: TAG_NAME
+#   displayName: Name
+#   description: The name assigned to all of the resources defined in this templates.
+#   required: true
+#   value: dev
+