diff --git a/.github/workflows/documentservice-cd.yml b/.github/workflows/documentservice-cd.yml new file mode 100644 index 000000000..4f91d0dd1 --- /dev/null +++ b/.github/workflows/documentservice-cd.yml @@ -0,0 +1,106 @@ +name: Document Services CD + + +on: + push: + branches: + - dev + - main + - dev-marshal + - test-marshal + - dev-rook + - test-rook + paths: + - "computingservices/DocumentServices/**" + - ".github/workflows/documentservice-cd.yml" + +defaults: + run: + shell: bash + working-directory: ./computingservices/DocumentServices + +env: + APP_NAME: "reviewer-documentservice" + TOOLS_NAME: "${{secrets.OPENSHIFT4_REPOSITORY}}" + +jobs: + documentServices-cd-by-push: + runs-on: ubuntu-20.04 + + if: github.event_name == 'push' && github.repository == 'bcgov/foi-docreviewer' + steps: + - uses: actions/checkout@v2 + - name: Set ENV variables for dev branch + if: ${{ github.ref_name == 'dev' }} + shell: bash + run: | + echo "For ${{ github.ref_name }} branch" + echo "TAG_NAME=dev" >> $GITHUB_ENV + echo "BRANCH_NAME=dev" >> $GITHUB_ENV + echo "ENV_NAME=dev" >> $GITHUB_ENV + + - name: Set ENV variables for main branch + if: ${{ github.ref_name == 'main' }} + shell: bash + run: | + echo "For ${{ github.ref_name }} branch" + echo "TAG_NAME=test" >> $GITHUB_ENV + echo "BRANCH_NAME=main" >> $GITHUB_ENV + echo "ENV_NAME=test" >> $GITHUB_ENV + + - name: Set ENV variables for dev-marshal branch + if: ${{ github.ref_name == 'dev-marshal' }} + run: | + echo "For ${{ github.ref_name }} branch" + echo "TAG_NAME=dev-marshal" >> $GITHUB_ENV + echo "BRANCH_NAME=dev-marshal" >> $GITHUB_ENV + echo "ENV_NAME=dev" >> $GITHUB_ENV + + - name: Set ENV variables for test-marshal branch + if: ${{ github.ref_name == 'test-marshal' }} + run: | + echo "For ${{ github.ref_name }} branch" + echo "TAG_NAME=test-marshal" >> $GITHUB_ENV + echo "BRANCH_NAME=test-marshal" >> $GITHUB_ENV + echo "ENV_NAME=test" >> $GITHUB_ENV + + - name: Set ENV variables for dev-rook branch + if: ${{ github.ref_name == 'dev-rook' }} + run: | + echo "For ${{ github.ref_name }} branch" + echo "TAG_NAME=dev-rook" >> $GITHUB_ENV + echo "BRANCH_NAME=dev-rook" >> $GITHUB_ENV + echo "ENV_NAME=dev" >> $GITHUB_ENV + + - name: Set ENV variables for test-rook branch + if: ${{ github.ref_name == 'test-rook' }} + run: | + echo "For ${{ github.ref_name }} branch" + echo "TAG_NAME=test-rook" >> $GITHUB_ENV + echo "BRANCH_NAME=test-rook" >> $GITHUB_ENV + echo "ENV_NAME=test" >> $GITHUB_ENV + + - name: Login Openshift + shell: bash + run: | + oc login --server=${{secrets.OPENSHIFT4_LOGIN_REGISTRY}} --token=${{secrets.OPENSHIFT4_SA_TOKEN}} + + - name: Tools project + shell: bash + run: | + oc project ${{ env.TOOLS_NAME }}-tools + + - name: Build from ${{ env.BRANCH_NAME }} branch + shell: bash + run: | + oc patch bc/${{ env.APP_NAME }}-build -p '{"spec":{"source":{"contextDir":"/computingservices/DocumentServices","git":{"ref":"${{ env.BRANCH_NAME }}"}}}}' + + - name: Start Build Openshift + shell: bash + run: | + oc start-build ${{ env.APP_NAME }}-build --wait + + - name: Tag+Deploy for ${{ env.TAG_NAME }} + shell: bash + run: | + oc tag ${{ env.APP_NAME }}:latest ${{ env.APP_NAME }}:${{ env.TAG_NAME }} diff --git a/.github/workflows/documentservice-ci.yml b/.github/workflows/documentservice-ci.yml new file mode 100644 index 000000000..f683c169e --- /dev/null +++ b/.github/workflows/documentservice-ci.yml @@ -0,0 +1,54 @@ +name: Document Services CI + + +on: + pull_request: + branches: + - main + - dev + - dev-marshal + - test-marshal + - dev-rook + - test-rook + paths: + - "computingservices/DocumentServices/**" + +defaults: + run: + shell: bash + working-directory: ./computingservices/DocumentServices + +jobs: + docker-build-check: + runs-on: ubuntu-20.04 + name: Build dockerfile to ensure it works + + steps: + - uses: actions/checkout@v2 + - name: docker build to check strictness + id: docker-build + run: | + docker build -f Dockerfile.local . + + python-build-check: + runs-on: ubuntu-20.04 + name: Build python to ensure it works + + strategy: + matrix: + # python-version: [3.6, 3.7, 3.8, 3.9] + python-version: [3.9] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install flake8 pytest + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + + diff --git a/api/reviewer_api/resources/foiflowmasterdata.py b/api/reviewer_api/resources/foiflowmasterdata.py index 2b553984e..9038a6874 100644 --- a/api/reviewer_api/resources/foiflowmasterdata.py +++ b/api/reviewer_api/resources/foiflowmasterdata.py @@ -272,7 +272,7 @@ def post(ministryrequestid, redactionlayer="redline", layertype="redline"): ) singlepkgpath = s3path_save data["s3path_save"] = s3path_save - + if is_single_redline_package(_bcgovcode, packagetype): for div in data["divdocumentList"]: if len(div["documentlist"]) > 0: diff --git a/api/reviewer_api/schemas/finalpackage.py b/api/reviewer_api/schemas/finalpackage.py index 8c5992065..702aeb858 100644 --- a/api/reviewer_api/schemas/finalpackage.py +++ b/api/reviewer_api/schemas/finalpackage.py @@ -10,6 +10,14 @@ class AttributeSchema(Schema): files = fields.Nested(FileSchema, many=True, required=True, allow_none=False) +class SummaryPkgSchema(Schema): + divisionid = fields.Int(data_key="divisionid", allow_none=True) + documentids = fields.List(fields.Int()) + +class SummarySchema(Schema): + pkgdocuments = fields.List(fields.Nested(SummaryPkgSchema, allow_none=True)) + sorteddocuments = fields.List(fields.Int()) + class FinalPackageSchema(Schema): ministryrequestid = fields.Str(data_key="ministryrequestid", allow_none=False) category = fields.Str(data_key="category", allow_none=False) @@ -18,3 +26,5 @@ class FinalPackageSchema(Schema): attributes = fields.Nested( AttributeSchema, many=True, required=True, allow_none=False ) + summarydocuments = fields.Nested(SummarySchema, allow_none=True) + redactionlayerid = fields.Int(data_key="redactionlayerid", allow_none=False) \ No newline at end of file diff --git a/api/reviewer_api/schemas/redline.py b/api/reviewer_api/schemas/redline.py index b92504f7f..80d7b1139 100644 --- a/api/reviewer_api/schemas/redline.py +++ b/api/reviewer_api/schemas/redline.py @@ -11,6 +11,13 @@ class AttributeSchema(Schema): divisionname = fields.Str(data_key="divisionname", allow_none=True) divisionid = fields.Int(data_key="divisionid", allow_none=True) +class SummaryPkgSchema(Schema): + divisionid = fields.Int(data_key="divisionid", allow_none=True) + documentids = fields.List(fields.Int()) + +class SummarySchema(Schema): + pkgdocuments = fields.List(fields.Nested(SummaryPkgSchema, allow_none=True)) + sorteddocuments = fields.List(fields.Int()) class RedlineSchema(Schema): ministryrequestid = fields.Str(data_key="ministryrequestid", allow_none=False) @@ -19,4 +26,6 @@ class RedlineSchema(Schema): bcgovcode = fields.Str(data_key="bcgovcode", allow_none=False) attributes = fields.Nested( AttributeSchema, many=True, required=True, allow_none=False - ) \ No newline at end of file + ) + summarydocuments = fields.Nested(SummarySchema, allow_none=True) + redactionlayerid = fields.Int(data_key="redactionlayerid", allow_none=False) \ No newline at end of file diff --git a/api/reviewer_api/services/external/documentserviceproducerservice.py b/api/reviewer_api/services/external/documentserviceproducerservice.py new file mode 100644 index 000000000..7611c5d35 --- /dev/null +++ b/api/reviewer_api/services/external/documentserviceproducerservice.py @@ -0,0 +1,27 @@ +import os +from walrus import Database +from reviewer_api.models.default_method_result import DefaultMethodResult +import logging +from os import getenv + +class documentserviceproducerservice: + """This class is reserved for integration with event queue (currently redis streams).""" + + host = os.getenv("DOCUMENTSERVICE_REDIS_HOST") + port = os.getenv("DOCUMENTSERVICE_REDIS_PORT") + password = os.getenv("DOCUMENTSERVICE_REDIS_PASSWORD") + + db = Database(host=host, port=port, db=0, password=password) + + def add(self, payload): + try: + stream = self.db.Stream(self.__streamkey()) + msgid = stream.add(payload, id="*") + return DefaultMethodResult(True, "Added to stream", msgid.decode("utf-8")) + except Exception as err: + logging.error("Error in contacting Redis Stream") + logging.error(err) + return DefaultMethodResult(False, err, -1) + + def __streamkey(self): + return getenv("DOCUMENTSERVICE_STREAM_KEY") diff --git a/api/reviewer_api/services/external/zipperproducerservice.py b/api/reviewer_api/services/external/zipperproducerservice.py index b007caebb..11f699911 100644 --- a/api/reviewer_api/services/external/zipperproducerservice.py +++ b/api/reviewer_api/services/external/zipperproducerservice.py @@ -2,6 +2,7 @@ from walrus import Database from reviewer_api.models.default_method_result import DefaultMethodResult import logging +from os import getenv class zipperproducerservice: @@ -13,12 +14,15 @@ class zipperproducerservice: db = Database(host=host, port=port, db=0, password=password) - def add(self, streamkey, payload): + def add(self, payload): try: - stream = self.db.Stream(streamkey) + stream = self.db.Stream(self.__streamkey()) msgid = stream.add(payload, id="*") return DefaultMethodResult(True, "Added to stream", msgid.decode("utf-8")) except Exception as err: logging.error("Error in contacting Redis Stream") logging.error(err) return DefaultMethodResult(False, err, -1) + + def __streamkey(self): + return getenv("ZIPPER_STREAM_KEY") \ No newline at end of file diff --git a/api/reviewer_api/services/radactionservice.py b/api/reviewer_api/services/radactionservice.py index b4bb21ccd..36a87d55e 100644 --- a/api/reviewer_api/services/radactionservice.py +++ b/api/reviewer_api/services/radactionservice.py @@ -8,16 +8,15 @@ from reviewer_api.services.annotationservice import annotationservice from reviewer_api.services.documentpageflagservice import documentpageflagservice from reviewer_api.services.jobrecordservice import jobrecordservice -from reviewer_api.services.external.zipperproducerservice import zipperproducerservice +from reviewer_api.services.external.documentserviceproducerservice import documentserviceproducerservice from reviewer_api.utils.util import to_json from datetime import datetime - +import json class redactionservice: """FOI Document management service""" - zipperstreamkey = getenv("ZIPPER_STREAM_KEY") def getannotationsbyrequest( @@ -193,13 +192,13 @@ def triggerdownloadredlinefinalpackage(self, finalpackageschema, userinfo): _jobmessage, userinfo["userid"] ) if job.success: - _message = self.__preparemessageforzipservice( + _message = self.__preparemessageforsummaryservice( finalpackageschema, userinfo, job ) - return zipperproducerservice().add(self.zipperstreamkey, _message) + return documentserviceproducerservice().add(_message) # redline/final package download: prepare message for zipping service - def __preparemessageforzipservice(self, messageschema, userinfo, job): + def __preparemessageforsummaryservice(self, messageschema, userinfo, job): _message = { "jobid": job.identifier, "requestid": -1, @@ -211,8 +210,10 @@ def __preparemessageforzipservice(self, messageschema, userinfo, job): "filestozip": to_json( self.__preparefilestozip(messageschema["attributes"]) ), - "finaloutput": to_json({}), + "finaloutput": to_json(""), "attributes": to_json(messageschema["attributes"]), + "summarydocuments": json.dumps(messageschema["summarydocuments"]), + "redactionlayerid": json.dumps(messageschema["redactionlayerid"]) } return _message diff --git a/computingservices/DocumentServices/.gitignore b/computingservices/DocumentServices/.gitignore new file mode 100644 index 000000000..0fd8a721b --- /dev/null +++ b/computingservices/DocumentServices/.gitignore @@ -0,0 +1,2 @@ +__pycache__/* +*.pyc \ No newline at end of file diff --git a/computingservices/DocumentServices/.sampleenv b/computingservices/DocumentServices/.sampleenv new file mode 100644 index 000000000..426268c39 --- /dev/null +++ b/computingservices/DocumentServices/.sampleenv @@ -0,0 +1,29 @@ + +#Properties of Document Service - Begin +DOCUMENTSERVICE_REDIS_HOST= +DOCUMENTSERVICE_REDIS_PORT= +DOCUMENTSERVICE_REDIS_PASSWORD= +DOCUMENTSERVICE_STREAM_KEY=DOCUMENTSERVICE + +ZIPPER_REDIS_HOST= +ZIPPER_REDIS_PORT= +ZIPPER_REDIS_PASSWORD= +ZIPPER_STREAM_KEY=ZIPPER_STREAM + +DOCUMENTSERVICE_DB_HOST= +DOCUMENTSERVICE_DB_NAME= +DOCUMENTSERVICE_DB_PORT= +DOCUMENTSERVICE_DB_USER= +DOCUMENTSERVICE_DB_PASSWORD= + +DOCUMENTSERVICE_S3_HOST= +DOCUMENTSERVICE_S3_REGION= +DOCUMENTSERVICE_S3_SERVICE= +DOCUMENTSERVICE_S3_ENV= + +FOI_DB_USER= +FOI_DB_PASSWORD= +FOI_DB_NAME= +FOI_DB_HOST= +FOI_DB_PORT= +#Properties of Document Service - End diff --git a/computingservices/DocumentServices/DockerFile.bcgov b/computingservices/DocumentServices/DockerFile.bcgov new file mode 100644 index 000000000..427e6e598 --- /dev/null +++ b/computingservices/DocumentServices/DockerFile.bcgov @@ -0,0 +1,18 @@ +FROM artifacts.developer.gov.bc.ca/docker-remote/python:3.10.8-buster + +# Keeps Python from generating .pyc files in the container +ENV PYTHONDONTWRITEBYTECODE=1 + +# Turns off buffering for easier container logging +ENV PYTHONUNBUFFERED=1 + +RUN useradd --create-home --shell /bin/bash app_user +WORKDIR /home/app_user +COPY requirements.txt ./ +RUN apt-get update \ + && apt-get -y install libpq-dev gcc \ + && pip install psycopg2 +RUN pip install --no-cache-dir -r requirements.txt +USER app_user +COPY . . +ENTRYPOINT ["/bin/sh", "./entrypoint.sh"] diff --git a/computingservices/DocumentServices/Dockerfile.local b/computingservices/DocumentServices/Dockerfile.local new file mode 100644 index 000000000..179ae3680 --- /dev/null +++ b/computingservices/DocumentServices/Dockerfile.local @@ -0,0 +1,18 @@ +FROM python:3.10.8 + +# Keeps Python from generating .pyc files in the container +ENV PYTHONDONTWRITEBYTECODE=1 + +# Turns off buffering for easier container logging +ENV PYTHONUNBUFFERED=1 + +RUN useradd --create-home --shell /bin/bash app_user +WORKDIR /home/app_user +COPY requirements.txt ./ +RUN apt-get update \ + && apt-get -y install libpq-dev gcc \ + && pip install psycopg2 +RUN pip install --no-cache-dir -r requirements.txt +USER app_user +COPY . . +ENTRYPOINT ["/bin/sh", "./entrypoint.sh"] diff --git a/computingservices/DocumentServices/__init__.py b/computingservices/DocumentServices/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/computingservices/DocumentServices/__main__.py b/computingservices/DocumentServices/__main__.py new file mode 100644 index 000000000..0a285b7ae --- /dev/null +++ b/computingservices/DocumentServices/__main__.py @@ -0,0 +1,6 @@ +from rstreamio.reader import documentservicestreamreader + + + +if __name__ == '__main__': + documentservicestreamreader.app() \ No newline at end of file diff --git a/computingservices/DocumentServices/entrypoint.sh b/computingservices/DocumentServices/entrypoint.sh new file mode 100644 index 000000000..bfa3d8512 --- /dev/null +++ b/computingservices/DocumentServices/entrypoint.sh @@ -0,0 +1,2 @@ +#!/bin/bash +python __main__.py $ \ No newline at end of file diff --git a/computingservices/DocumentServices/models/__init__.py b/computingservices/DocumentServices/models/__init__.py new file mode 100644 index 000000000..b87e20fea --- /dev/null +++ b/computingservices/DocumentServices/models/__init__.py @@ -0,0 +1,2 @@ +from .redactionsummary import redactionsummary +from .s3credentials import s3credentials \ No newline at end of file diff --git a/computingservices/DocumentServices/models/redactionsummary.py b/computingservices/DocumentServices/models/redactionsummary.py new file mode 100644 index 000000000..35a194ef1 --- /dev/null +++ b/computingservices/DocumentServices/models/redactionsummary.py @@ -0,0 +1,13 @@ +class redactionsummary(object): + + def __init__(self,jobid,requestid,category,requestnumber,bcgovcode,createdby,ministryrequestid,filestozip,finaloutput,attributes) -> None: + self.jobid = jobid + self.requestid = requestid + self.category=category + self.requestnumber = requestnumber + self.bcgovcode = bcgovcode + self.createdby = createdby + self.ministryrequestid = ministryrequestid + self.filestozip = filestozip + self.finaloutput = finaloutput + self.attributes = attributes diff --git a/computingservices/DocumentServices/models/s3credentials.py b/computingservices/DocumentServices/models/s3credentials.py new file mode 100644 index 000000000..97746abbb --- /dev/null +++ b/computingservices/DocumentServices/models/s3credentials.py @@ -0,0 +1,4 @@ +class s3credentials(object): + def __init__(self,s3accesskey,s3secretkey) -> None: + self.s3accesskey = s3accesskey + self.s3secretkey=s3secretkey diff --git a/computingservices/DocumentServices/requirements.txt b/computingservices/DocumentServices/requirements.txt new file mode 100644 index 000000000..ca7e1b33c Binary files /dev/null and b/computingservices/DocumentServices/requirements.txt differ diff --git a/computingservices/DocumentServices/rstreamio/__init__.py b/computingservices/DocumentServices/rstreamio/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/computingservices/DocumentServices/rstreamio/message/__init__.py b/computingservices/DocumentServices/rstreamio/message/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/computingservices/DocumentServices/rstreamio/message/schemas/__init__.py b/computingservices/DocumentServices/rstreamio/message/schemas/__init__.py new file mode 100644 index 000000000..a444092a5 --- /dev/null +++ b/computingservices/DocumentServices/rstreamio/message/schemas/__init__.py @@ -0,0 +1,16 @@ +# Copyright © 2021 Province of British Columbia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Schema package.""" + + diff --git a/computingservices/DocumentServices/rstreamio/message/schemas/baseinfo.py b/computingservices/DocumentServices/rstreamio/message/schemas/baseinfo.py new file mode 100644 index 000000000..0aad9ac37 --- /dev/null +++ b/computingservices/DocumentServices/rstreamio/message/schemas/baseinfo.py @@ -0,0 +1,9 @@ +import json + +class baseobj(object): + def __init__(self, dict_): + self.__dict__.update(dict_) + + +def dict2obj(d): + return json.loads(json.dumps(d), object_hook=baseobj) \ No newline at end of file diff --git a/computingservices/DocumentServices/rstreamio/message/schemas/redactionsummary.py b/computingservices/DocumentServices/rstreamio/message/schemas/redactionsummary.py new file mode 100644 index 000000000..e79f10427 --- /dev/null +++ b/computingservices/DocumentServices/rstreamio/message/schemas/redactionsummary.py @@ -0,0 +1,70 @@ + +from marshmallow import EXCLUDE, fields, Schema +import json +from rstreamio.message.schemas.baseinfo import baseobj, dict2obj +from models.redactionsummary import redactionsummary +""" +This class consolidates schemas of RedactionSummary Service. + +__author__ = "sumathi.thirumani@aot-technologies.com" + +""" + + +class FileSchema(Schema): + class Meta: + unknown = EXCLUDE + filename = fields.Str(data_key="filename",allow_none=False) + s3uripath = fields.Str(data_key="s3uripath",allow_none=False) + +class SummaryPkgSchema(Schema): + class Meta: + unknown = EXCLUDE + divisionid = fields.Int(data_key="divisionid", allow_none=True) + documentids = fields.List(fields.Int()) + +class SummarySchema(Schema): + class Meta: + unknown = EXCLUDE + sorteddocuments = fields.List(fields.Int()) + pkgdocuments = fields.List(fields.Nested(SummaryPkgSchema, allow_none=True)) + +class AttributeSchema(Schema): + class Meta: + unknown = EXCLUDE + divisionid = fields.Int(data_key="divisionid",allow_none=True) + files = fields.List(fields.Nested(FileSchema, allow_none=True)) + divisionname = fields.Str(data_key="divisionname",allow_none=True) + +class RedactionSummaryIncomingSchema(Schema): + class Meta: + unknown = EXCLUDE + jobid = fields.Int(data_key="jobid",allow_none=False) + requestid = fields.Int(data_key="requestid",allow_none=False) + ministryrequestid = fields.Int(data_key="ministryrequestid",allow_none=False) + category = fields.Str(data_key="category",allow_none=False) + requestnumber = fields.Str(data_key="requestnumber",allow_none=False) + bcgovcode = fields.Str(data_key="bcgovcode",allow_none=False) + createdby = fields.Str(data_key="createdby",allow_none=False) + filestozip = fields.List(fields.Nested(FileSchema, allow_none=True)) + finaloutput = fields.Str(data_key="finaloutput",allow_none=False) + attributes = fields.List(fields.Nested(AttributeSchema, allow_none=True)) + summarydocuments = fields.Nested(SummarySchema, allow_none=True) + redactionlayerid = fields.Int(data_key="redactionlayerid", allow_none=False) + + +def get_in_redactionsummary_msg(producer_json): + messageobject = RedactionSummaryIncomingSchema().load(__formatmsg(producer_json), unknown=EXCLUDE) + return dict2obj(messageobject) + +def __formatmsg(producer_json): + j = json.loads(producer_json) + return j + +def decodesummarymsg(_message): + _message = _message.encode().decode('unicode-escape') + _message = _message.replace("b'","'").replace('"\'','"').replace('\'"','"') + _message = _message.replace('"[','[').replace(']"',"]").replace("\\","") + _message = _message.replace('"{','{').replace('}"',"}") + _message = _message.replace('""','"') + return _message \ No newline at end of file diff --git a/computingservices/DocumentServices/rstreamio/reader/documentservicestreamreader.py b/computingservices/DocumentServices/rstreamio/reader/documentservicestreamreader.py new file mode 100644 index 000000000..74b9edebe --- /dev/null +++ b/computingservices/DocumentServices/rstreamio/reader/documentservicestreamreader.py @@ -0,0 +1,61 @@ +import json +import typer +import random +import time +import logging +from enum import Enum +from utils import redisstreamdb +from utils.foidocumentserviceconfig import documentservice_stream_key +from rstreamio.message.schemas.redactionsummary import decodesummarymsg +from services.redactionsummaryservice import redactionsummaryservice +from services.zippingservice import zippingservice + + +LAST_ID_KEY = "{consumer_id}:lastid" +BLOCK_TIME = 5000 +STREAM_KEY = documentservice_stream_key + +app = typer.Typer() + +class StartFrom(str, Enum): + beginning = "0" + latest = "$" + +@app.command() +def start(consumer_id: str, start_from: StartFrom = StartFrom.latest): + rdb = redisstreamdb + stream = rdb.Stream(STREAM_KEY) + last_id = rdb.get(LAST_ID_KEY.format(consumer_id=consumer_id)) + if last_id: + print(f"Resume from ID: {last_id}") + else: + last_id = start_from.value + print(f"Starting from {start_from.name}") + + while True: + print("Reading stream...") + messages = stream.read(last_id=last_id, block=BLOCK_TIME) + print("*********** Messages ***********") + print(messages) + if messages: + for _message in messages: + # message_id is the random id created to identify the message + # message is the actual data passed to the stream + message_id, message = _message + print(f"processing {message_id}::{message}") + if message is not None: + message = json.dumps({str(key): str(value) for (key, value) in message.items()}) + message = decodesummarymsg(message) + summaryfiles = [] + try: + summaryfiles = redactionsummaryservice().processmessage(message) + except(Exception) as error: + logging.exception(error) + zippingservice().sendtozipper(summaryfiles, message) + # simulate processing + time.sleep(random.randint(1, 3)) + last_id = message_id + rdb.set(LAST_ID_KEY.format(consumer_id=consumer_id), last_id) + print(f"finished processing {message_id}") + else: + print(f"No new messages after ID: {last_id}") \ No newline at end of file diff --git a/computingservices/DocumentServices/rstreamio/writer/__init__.py b/computingservices/DocumentServices/rstreamio/writer/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/computingservices/DocumentServices/rstreamio/writer/zipperstreamwriter.py b/computingservices/DocumentServices/rstreamio/writer/zipperstreamwriter.py new file mode 100644 index 000000000..8de678519 --- /dev/null +++ b/computingservices/DocumentServices/rstreamio/writer/zipperstreamwriter.py @@ -0,0 +1,19 @@ +import logging +from utils import redisstreamdb, zipper_stream_key + + +class zipperstreamwriter: + + rdb = redisstreamdb + zipperstream = rdb.Stream(zipper_stream_key) + + def sendmessage(self, message): + try: + msgid = self.zipperstream.add(message, id="*") + logging.info("zipper message for msgid = %s ", msgid) + except RuntimeError as error: + print("Exception while sending message for zipping , Error : {0} ".format(error)) + logging.error("Unable to write to message stream for zipper %s | ministryrequestid=%i", message.ministryrequestid, message.ministryrequestid) + logging.error(error) + + \ No newline at end of file diff --git a/computingservices/DocumentServices/services/__init__.py b/computingservices/DocumentServices/services/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/computingservices/DocumentServices/services/cdogsapiservice.py b/computingservices/DocumentServices/services/cdogsapiservice.py new file mode 100644 index 000000000..78a042c98 --- /dev/null +++ b/computingservices/DocumentServices/services/cdogsapiservice.py @@ -0,0 +1,90 @@ +"""Service for pdf generation.""" +import base64 +import json +import os +import re + +import requests +from utils.foidocumentserviceconfig import cdogs_base_url,cdogs_token_url,cdogs_service_client,cdogs_service_client_secret + + +class cdogsapiservice: + """cdogs api Service class.""" + + + def generate_pdf(self, template_hash_code, data, access_token): + request_body = { + "options": { + "cachereport": False, + "convertTo": "pdf", + "overwrite": True, + "reportName": "Summary" + }, + "data": data + } + json_request_body = json.dumps(request_body) + print("json_request_body:",json_request_body) + headers = { + 'Content-Type': 'application/json', + 'Authorization': f'Bearer {access_token}' + } + url = f"{cdogs_base_url}/api/v2/template/{template_hash_code}/render" + print("url:",url) + return self._post_generate_pdf(json_request_body, headers, url) + + def _post_generate_pdf(self, json_request_body, headers, url): + return requests.post(url, data= json_request_body, headers= headers) + + def upload_template(self, template_path, access_token): + headers = { + "Authorization": f'Bearer {access_token}' + } + url = f"{cdogs_base_url}/api/v2/template" + if os.path.exists(template_path): + print("Exists!!") + template = {'template':('template', open(template_path, 'rb'), "multipart/form-data")} + response = self._post_upload_template(headers, url, template) + if response.status_code == 200: + print('Returning new hash %s', response.headers['X-Template-Hash']) + return response.headers['X-Template-Hash']; + + response_json = json.loads(response.content) + if response.status_code == 405 and response_json['detail'] is not None: + match = re.findall(r"Hash '(.*?)'", response_json['detail']); + if match: + print('Template already hashed with code %s', match[0]) + return match[0] + + + def _post_upload_template(self, headers, url, template): + response = requests.post(url, headers= headers, files= template) + return response + + def check_template_cached(self, template_hash_code, access_token): + + headers = { + "Authorization": f'Bearer {access_token}' + } + url = f"{cdogs_base_url}/api/v2/template/{template_hash_code}" + response = requests.get(url, headers= headers) + return response.status_code == 200 + + + def _get_access_token(self): + token_url = cdogs_token_url + service_client = cdogs_service_client + service_client_secret = cdogs_service_client_secret + basic_auth_encoded = base64.b64encode( + bytes(service_client + ':' + service_client_secret, 'utf-8')).decode('utf-8') + data = 'grant_type=client_credentials' + response = requests.post( + token_url, + data=data, + headers={ + 'Authorization': f'Basic {basic_auth_encoded}', + 'Content-Type': 'application/x-www-form-urlencoded' + } + ) + + response_json = response.json() + return response_json['access_token'] \ No newline at end of file diff --git a/computingservices/DocumentServices/services/dal/__init__.py b/computingservices/DocumentServices/services/dal/__init__.py new file mode 100644 index 000000000..eeeba930d --- /dev/null +++ b/computingservices/DocumentServices/services/dal/__init__.py @@ -0,0 +1 @@ +from utils import getdbconnection \ No newline at end of file diff --git a/computingservices/DocumentServices/services/dal/documentpageflag.py b/computingservices/DocumentServices/services/dal/documentpageflag.py new file mode 100644 index 000000000..934a93a86 --- /dev/null +++ b/computingservices/DocumentServices/services/dal/documentpageflag.py @@ -0,0 +1,186 @@ +from utils import getdbconnection +import logging +import json + +class documentpageflag: + + @classmethod + def get_all_pageflags(cls): + conn = getdbconnection() + pageflags = [] + try: + cursor = conn.cursor() + cursor.execute( + """select pageflagid, name, description from "Pageflags" + where isactive = true and name not in ('Consult') order by sortorder + """ + ) + + result = cursor.fetchall() + cursor.close() + if result is not None: + for entry in result: + pageflags.append({"pageflagid": entry[0], "name": entry[1], "description": entry[2]}) + return pageflags + return None + + except Exception as error: + logging.error("Error in getting page flags") + logging.error(error) + raise + finally: + if conn is not None: + conn.close() + + @classmethod + def get_all_programareas(cls): + conn = getdbconnection() + programareas = {} + try: + cursor = conn.cursor() + cursor.execute( + """select programareaid, bcgovcode, iaocode from "ProgramAreas" + where isactive = true order by programareaid + """ + ) + + result = cursor.fetchall() + cursor.close() + if result is not None: + for entry in result: + programareas[entry[0]] = {"bcgovcode": entry[1], "iaocode": entry[2]} + return programareas + return None + + except Exception as error: + logging.error("Error in getting program areas") + logging.error(error) + raise + finally: + if conn is not None: + conn.close() + + @classmethod + def get_documentpageflag(cls, ministryrequestid, redactionlayerid, documentids): + conn = getdbconnection() + documentpageflags = {} + try: + cursor = conn.cursor() + cursor.execute( + """select documentid, documentversion, pageflag, attributes + from "DocumentPageflags" dp where + foiministryrequestid = %s::integer and redactionlayerid = %s::integer and documentid in %s + order by documentid ;""", + (ministryrequestid, redactionlayerid, tuple(documentids)), + ) + + result = cursor.fetchall() + cursor.close() + if result is not None: + for entry in result: + documentpageflags[entry[0]] = {"pageflag": entry[2], "attributes": entry[3]} + return documentpageflags + return None + + except Exception as error: + logging.error("Error in getting document page flags") + logging.error(error) + raise + finally: + if conn is not None: + conn.close() + + @classmethod + def get_documents_lastmodified(cls, ministryrequestid, documentids): + conn = getdbconnection() + docids = [] + try: + cursor = conn.cursor() + cursor.execute( + """select d.documentid from "Documents" d join "DocumentMaster" dm on d.foiministryrequestid = dm.ministryrequestid and d.documentmasterid = dm.documentmasterid + join "DocumentAttributes" da on (da.documentmasterid = d.documentmasterid or da.documentmasterid = dm.processingparentid) + where documentid in %s + and d.foiministryrequestid = %s::integer + group by d.documentid, da."attributes" ->> 'lastmodified' ::text + order by da."attributes" ->> 'lastmodified'""", + (tuple(documentids),ministryrequestid), + ) + + result = cursor.fetchall() + cursor.close() + if result is not None: + for entry in result: + docids.append(entry[0]) + return docids + return None + + except Exception as error: + logging.error("Error in getting documentids for requestid") + logging.error(error) + raise + finally: + if conn is not None: + conn.close() + + + @classmethod + def getpagecount_by_documentid(cls, ministryrequestid, documentids): + conn = getdbconnection() + docpgs = {} + try: + cursor = conn.cursor() + cursor.execute( + """select documentid, pagecount from "Documents" d + where foiministryrequestid = %s::integer + and documentid in %s;""", + (ministryrequestid, tuple(documentids)), + ) + + result = cursor.fetchall() + cursor.close() + if result is not None: + for entry in result: + docpgs[entry[0]] = {"pagecount": entry[1]} + return docpgs + return None + + except Exception as error: + logging.error("Error in getting pagecount for requestid") + logging.error(error) + raise + finally: + if conn is not None: + conn.close() + + @classmethod + def getsections_by_documentid_pageno(cls, redactionlayerid, documentid, pagenos): + conn = getdbconnection() + sections = [] + try: + cursor = conn.cursor() + cursor.execute( + """select pagenumber , unnest(xpath('//contents/text()', annotation::xml))::text as sections + from "Annotations" a + where annotation like '%%freetext%%' and isactive = true + and redactionlayerid = %s::integer + and documentid = %s::integer + and pagenumber in %s + order by pagenumber;""", + (redactionlayerid, documentid, tuple(pagenos)), + ) + + result = cursor.fetchall() + cursor.close() + if result is not None: + for entry in result: + sections.append({"pageno": entry[0], "section": entry[1]}) + return sections + return None + + except Exception as error: + logging.error("Error in getting sections for requestid") + logging.error(error) + raise + finally: + if conn is not None: + conn.close() \ No newline at end of file diff --git a/computingservices/DocumentServices/services/dal/documenttemplate.py b/computingservices/DocumentServices/services/dal/documenttemplate.py new file mode 100644 index 000000000..ee6cdca51 --- /dev/null +++ b/computingservices/DocumentServices/services/dal/documenttemplate.py @@ -0,0 +1,73 @@ +from utils import getfoidbconnection +import logging +import json + + +class documenttemplate: + + @classmethod + def gettemplatebytype(cls, documenttypeid, extension= "docx"): + conn = getfoidbconnection() + try: + cursor = conn.cursor() + query = ''' + SELECT cdogs_hash_code + FROM public."DocumentTemplates" + WHERE document_type_id = %s and extension = %s; + ''' + parameters = (documenttypeid,extension,) + cursor.execute(query, parameters) + documenttemplate = cursor.fetchone()[0] + return documenttemplate + except Exception as error: + logging.error("Error in gettemplatebytype") + logging.error(error) + raise + finally: + cursor.close() + if conn is not None: + conn.close() + + @classmethod + def updatecdogshashcode(cls, documenttypeid, cdogshashcode): + conn = getfoidbconnection() + try: + cursor = conn.cursor() + query = ''' + UPDATE public."DocumentTemplates" SET cdogs_hash_code = %s + WHERE document_type_id = %s; + ''' + parameters = (cdogshashcode, documenttypeid,) + cursor.execute(query, parameters) + print("DB updated") + conn.commit() + except(Exception) as error: + print("Exception while executing func updatecdogshashcode, Error : {0} ".format(error)) + raise + finally: + cursor.close() + if conn is not None: + conn.close() + + @classmethod + def getdocumenttypebyname(cls, document_type_name): + conn = getfoidbconnection() + try: + cursor = conn.cursor() + query = ''' + SELECT * + FROM public."DocumentTypes" + WHERE document_type_name = %s; + ''' + parameters = (document_type_name,) + cursor.execute(query, parameters) + documenttemplate = cursor.fetchone()[0] + return documenttemplate + except Exception as error: + logging.error("Error in getdocumenttypebyname") + logging.error(error) + raise + finally: + cursor.close() + if conn is not None: + conn.close() \ No newline at end of file diff --git a/computingservices/DocumentServices/services/dal/pdfstitchjobactivity.py b/computingservices/DocumentServices/services/dal/pdfstitchjobactivity.py new file mode 100644 index 000000000..14cc66514 --- /dev/null +++ b/computingservices/DocumentServices/services/dal/pdfstitchjobactivity.py @@ -0,0 +1,42 @@ +from utils import getdbconnection +import logging +import json + +def to_json(obj): + return json.dumps(obj, default=lambda obj: obj.__dict__) +class pdfstitchjobactivity: + + @classmethod + def recordjobstatus(cls,message,version,status,error=None,errormessage=None): + category = message.category.lower() + "-summary" + conn = getdbconnection() + try: + cursor = conn.cursor() + status = "error" if error else status + + cursor.execute( + """INSERT INTO public."PDFStitchJob" + (pdfstitchjobid,version, ministryrequestid, category, inputfiles, status, message, createdby) + VALUES (%s::integer,%s::integer, %s::integer, %s, %s, %s, %s, %s) on conflict (pdfstitchjobid,version) do nothing returning pdfstitchjobid;""", + ( + message.jobid, + version, + message.ministryrequestid, + category, + to_json(message.attributes), + status, + errormessage if error else None, + message.createdby, + ), + ) + + conn.commit() + cursor.close() + except Exception as error: + logging.error("Error in recordjobstatus") + logging.error(error) + raise + finally: + if conn is not None: + conn.close() + diff --git a/computingservices/DocumentServices/services/documentgenerationservice.py b/computingservices/DocumentServices/services/documentgenerationservice.py new file mode 100644 index 000000000..c88f2116a --- /dev/null +++ b/computingservices/DocumentServices/services/documentgenerationservice.py @@ -0,0 +1,65 @@ +#from reviewer_api.services.cdogs_api_service import cdogsapiservice +from services.dal.documenttemplate import documenttemplate +from .cdogsapiservice import cdogsapiservice +import json +import logging +from aws_requests_auth.aws_auth import AWSRequestsAuth +import os +import boto3 +from botocore.exceptions import ClientError +from botocore.config import Config +import requests +import mimetypes + + + +s3host = os.getenv("OSS_S3_HOST") +s3region = os.getenv("OSS_S3_REGION") + +class documentgenerationservice: + """document generation Service class.""" + + # def __init__(self,documenttypename='redaction_summary'): + # self.cdgos_api_service = cdogsapiservice() + # self.documenttypename = documenttypename + # receipt_document_type : DocumentType = DocumentType.get_document_type_by_name(self.documenttypename) + # if receipt_document_type is None: + # raise BusinessException(Error.DATA_NOT_FOUND) + + # self.receipt_template : DocumentTemplate = DocumentTemplate \ + # .get_template_by_type(document_type_id = receipt_document_type.document_type_id) + # if self.receipt_template is None: + # raise BusinessException(Error.DATA_NOT_FOUND) + + + def generate_pdf(self, data, documenttypename='redline_redaction_summary', template_path='templates/redline_redaction_summary.docx'): + access_token= cdogsapiservice()._get_access_token() + template_cached = False + templatefromdb= self.__gettemplate(documenttypename) + if templatefromdb is not None and templatefromdb["cdogs_hash_code"] is not None: + template_cached = cdogsapiservice().check_template_cached(templatefromdb["cdogs_hash_code"], access_token) + templatecdogshashcode = templatefromdb["cdogs_hash_code"] + print("template_cached:",template_cached) + + if templatefromdb is None or templatefromdb["cdogs_hash_code"] is None or not template_cached: + templatecdogshashcode = cdogsapiservice().upload_template(template_path, access_token) + print("templatecdogshashcode:",templatecdogshashcode) + if templatefromdb is not None and templatefromdb["document_type_id"] is not None: + templatefromdb["cdogs_hash_code"] = templatecdogshashcode + documenttemplate().updatecdogshashcode(templatefromdb["document_type_id"], templatefromdb["cdogs_hash_code"]) + return cdogsapiservice().generate_pdf(templatecdogshashcode, data,access_token) + + def __gettemplate(self,documenttypename='redline_redaction_summary'): + try: + templatefromdb=None + summary_cdogs_hash_code=None + summary_document_type_id =documenttemplate().getdocumenttypebyname(documenttypename) + print("summary_document_type_id:",summary_document_type_id) + if summary_document_type_id is not None: + summary_cdogs_hash_code=documenttemplate().gettemplatebytype(summary_document_type_id) + templatefromdb = {"document_type_id": summary_document_type_id, "cdogs_hash_code":summary_cdogs_hash_code} + print("templatefromdb:",templatefromdb) + return templatefromdb + except (Exception) as error: + print('error occured in document generation service - gettemplate method: ', error) + diff --git a/computingservices/DocumentServices/services/dts/redactionsummary.py b/computingservices/DocumentServices/services/dts/redactionsummary.py new file mode 100644 index 000000000..bae052f39 --- /dev/null +++ b/computingservices/DocumentServices/services/dts/redactionsummary.py @@ -0,0 +1,140 @@ +from services.dal.documentpageflag import documentpageflag + +class redactionsummary(): + + def prepareredactionsummary(self, message, documentids, pageflags, programareas): + redactionsummary = self.prepare_pkg_redactionsummary(message, documentids, pageflags, programareas) + if message.category == "responsepackage": + consolidated_redactions = [] + for entry in redactionsummary['data']: + consolidated_redactions += entry['sections'] + sortedredactions = sorted(consolidated_redactions, key=lambda x: self.__getrangenumber(x["range"])) + return {"requestnumber": message.requestnumber, "data": sortedredactions} + return redactionsummary + + def __getrangenumber(self, rangeval): + rangestart = str(rangeval).split('-')[0] + rangestart = str(rangestart).split('(')[0] + return int(rangestart) + + def prepare_pkg_redactionsummary(self, message, documentids, pageflags, programareas): + try: + redactionlayerid = message.redactionlayerid + summarymsg = message.summarydocuments + ordereddocids = summarymsg.sorteddocuments + stitchedpagedata = documentpageflag().getpagecount_by_documentid(message.ministryrequestid, ordereddocids) + totalpagecount = self.__calculate_totalpages(stitchedpagedata) + if totalpagecount <=0: + return + _pageflags = self.__transformpageflags(pageflags) + + summarydata = [] + docpageflags = documentpageflag().get_documentpageflag(message.ministryrequestid, redactionlayerid, ordereddocids) + + pagecount = 0 + for docid in ordereddocids: + if docid in documentids: + docpageflag = docpageflags[docid] + for pageflag in _pageflags: + filteredpages = self.__get_pages_by_flagid(docpageflag["pageflag"], pagecount, pageflag["pageflagid"]) + if len(filteredpages) > 0: + originalpagenos = [pg['originalpageno'] for pg in filteredpages] + docpagesections = documentpageflag().getsections_by_documentid_pageno(redactionlayerid, docid, originalpagenos) + docpageconsults = self.__get_consults_by_pageno(programareas, docpageflag["pageflag"], filteredpages) + pageflag['docpageflags'] = pageflag['docpageflags'] + self.__get_pagesection_mapping(filteredpages, docpagesections, docpageconsults) + pagecount = pagecount+stitchedpagedata[docid]["pagecount"] + + for pageflag in _pageflags: + _data = {} + if len(pageflag['docpageflags']) > 0: + _data = {} + _data["flagname"] = pageflag["description"].upper() + _data["pagecount"] = len(pageflag['docpageflags']) + _data["sections"] = self.__format_redaction_summary(pageflag["description"], pageflag['docpageflags']) + summarydata.append(_data) + return {"requestnumber": message.requestnumber, "data": summarydata} + except (Exception) as error: + print('error occured in redaction summary service: ', error) + + def __transformpageflags(self, pageflags): + for entry in pageflags: + entry['docpageflags']= [] + return pageflags + def __get_consults_by_pageno(self, programareas, docpageflag, pagenos): + consults = {} + for entry in docpageflag: + for pg in pagenos: + if entry['flagid'] == 4 and entry['page']-1 == pg['originalpageno']: + additional_consults = entry["other"] if "other" in entry else [] + consults[pg['originalpageno']] = self.__format_consults(programareas,entry['programareaid'], additional_consults) + return consults + + def __format_consults(self, programareas, consultids, others): + formatted = [] + for cid in consultids: + formatted.append(programareas[cid]['iaocode']) + if len(others) > 0: + formatted = formatted+others + return ",".join(formatted) + + def __format_redaction_summary(self, pageflag, pageredactions): + totalpages = len(pageredactions) + _sorted_pageredactions = sorted(pageredactions, key=lambda x: x["stitchedpageno"]) + #prepare ranges: Begin + formatted = [] + range_start, range_end = 0, 0 + range_sections = [] + range_consults = None + for pgindex, pgentry in enumerate(_sorted_pageredactions): + currentpg = _sorted_pageredactions[pgindex] + nextindex = pgindex+1 if pgindex < totalpages-1 else pgindex + nextpg = _sorted_pageredactions[nextindex] + range_sections = currentpg["sections"] if range_start == 0 else range_sections + range_start = currentpg["stitchedpageno"] if range_start == 0 else range_start + range_consults = currentpg["consults"] + if currentpg["stitchedpageno"]+1 == nextpg["stitchedpageno"] and currentpg["consults"] == nextpg["consults"]: + range_sections.extend(nextpg["sections"]) + range_end = nextpg["stitchedpageno"] + else: + rangepg = str(range_start) if range_end == 0 else str(range_start)+" - "+str(range_end) + rangepg = rangepg if range_consults is None else rangepg+" ("+range_consults+")" + formatted.append({"range": rangepg, "section": self.__formatsections(pageflag, range_sections)}) + range_start, range_end = 0, 0, + range_consults = None + range_sections = [] + #prepare ranges: End + return formatted + + + def __formatsections(self, pageflag, sections): + if pageflag in ("Duplicate", "Not Responsive"): + return pageflag + distinct_sections = list(set(sections)) + return pageflag+" under "+", ".join(distinct_sections) if len(distinct_sections) > 0 else pageflag + + def __get_pagesection_mapping(self, docpages, docpagesections, docpageconsults): + for entry in docpages: + entry["sections"] = self.__get_sections(docpagesections, entry['originalpageno']) + entry["consults"] = docpageconsults[entry['originalpageno']] if entry['originalpageno'] in docpageconsults else None + return docpages + + def __get_sections(self, docpagesections, pageno): + sections = [] + filtered = [x for x in docpagesections if x['pageno'] == pageno] + for dta in filtered: + sections += [x.strip() for x in dta['section'].split(",")] + return list(filter(None, sections)) + + def __get_pages_by_flagid(self, _docpageflags, totalpages, flagid): + pagenos = [] + for x in _docpageflags: + if x["flagid"] == flagid: + pagenos.append({'originalpageno':x["page"]-1, 'stitchedpageno':x["page"]+totalpages}) + return pagenos + + def __calculate_totalpages(self, data): + totalpages = 0 + for entry in data: + totalpages=totalpages+data[entry]['pagecount'] + return totalpages + diff --git a/computingservices/DocumentServices/services/redactionsummaryservice.py b/computingservices/DocumentServices/services/redactionsummaryservice.py new file mode 100644 index 000000000..2861e94d2 --- /dev/null +++ b/computingservices/DocumentServices/services/redactionsummaryservice.py @@ -0,0 +1,64 @@ + +import traceback +import json +from services.dal.pdfstitchjobactivity import pdfstitchjobactivity +from services.dts.redactionsummary import redactionsummary +from .documentgenerationservice import documentgenerationservice +from .s3documentservice import uploadbytes +from services.dts.redactionsummary import redactionsummary +from services.dal.documentpageflag import documentpageflag +from rstreamio.message.schemas.redactionsummary import get_in_redactionsummary_msg +class redactionsummaryservice(): + + def processmessage(self,incomingmessage): + summaryfilestozip = [] + message = get_in_redactionsummary_msg(incomingmessage) + try: + pdfstitchjobactivity().recordjobstatus(message,3,"redactionsummarystarted") + summarymsg = message.summarydocuments + #Condition for handling oipcredline category + category = message.category + documenttypename= category+"_redaction_summary" if category == 'responsepackage' else "redline_redaction_summary" + #print('documenttypename', documenttypename) + upload_responses=[] + pageflags = documentpageflag().get_all_pageflags() + programareas = documentpageflag().get_all_programareas() + divisiondocuments = summarymsg.pkgdocuments + for entry in divisiondocuments: + divisionid = entry.divisionid + documentids = entry.documentids + formattedsummary = redactionsummary().prepareredactionsummary(message, documentids, pageflags, programareas) + print('formattedsummary', formattedsummary) + template_path='templates/'+documenttypename+'.docx' + redaction_summary= documentgenerationservice().generate_pdf(formattedsummary, documenttypename,template_path) + messageattributes= message.attributes + print("attributes length:",len(messageattributes)) + if len(messageattributes)>1: + filesobj=(next(item for item in messageattributes if item.divisionid == divisionid)).files[0] + else: + filesobj= messageattributes[0].files[0] + stitcheddocs3uri = filesobj.s3uripath + stitcheddocfilename = filesobj.filename + s3uricategoryfolder= "oipcreview" if category == 'oipcreviewredline' else category + s3uri = stitcheddocs3uri.split(s3uricategoryfolder+"/")[0] + s3uricategoryfolder+"/" + filename = stitcheddocfilename.replace(".pdf","- summary.pdf") + print('s3uri:', s3uri) + uploadobj= uploadbytes(filename,redaction_summary.content, s3uri) + upload_responses.append(uploadobj) + if uploadobj["uploadresponse"].status_code == 200: + summaryuploaderror= False + summaryuploaderrormsg="" + else: + summaryuploaderror= True + summaryuploaderrormsg = uploadobj.uploadresponse.text + pdfstitchjobactivity().recordjobstatus(message,4,"redactionsummaryuploaded",summaryuploaderror,summaryuploaderrormsg) + summaryfilestozip.append({"filename": uploadobj["filename"], "s3uripath":uploadobj["documentpath"]}) + return summaryfilestozip + except (Exception) as error: + print('error occured in redaction summary service: ', error) + pdfstitchjobactivity().recordjobstatus(message,4,"redactionsummaryfailed",str(error),"summary generation failed") + return summaryfilestozip + + + + diff --git a/computingservices/DocumentServices/services/s3documentservice.py b/computingservices/DocumentServices/services/s3documentservice.py new file mode 100644 index 000000000..bf5c5f8b7 --- /dev/null +++ b/computingservices/DocumentServices/services/s3documentservice.py @@ -0,0 +1,69 @@ +from utils.dbconnection import getdbconnection +from psycopg2 import sql +from os import path +import psycopg2 +import requests +from aws_requests_auth.aws_auth import AWSRequestsAuth +from utils.jsonmessageparser import gets3credentialsobject +from utils.foidocumentserviceconfig import docservice_db_user,docservice_db_port,docservice_s3_host,docservice_db_name,docservice_s3_region,docservice_s3_service,docservice_failureattempt +import logging + +def getcredentialsbybucketname(bucketname): + _conn = getdbconnection() + s3cred = None + #bucket = '{0}-{1}-e'.format(bcgovcode.lower(),pdfstitch_s3_env.lower()) + try: + cur = _conn.cursor() + _sql = sql.SQL("SELECT attributes FROM {0} WHERE bucket='{1}'and category='Records'".format('public."DocumentPathMapper"',bucketname)) + cur.execute(_sql) + attributes = cur.fetchone() + if attributes is not None: + s3cred = gets3credentialsobject(str(attributes[0])) + cur.close() + except (Exception, psycopg2.DatabaseError) as error: + logging.error(error) + finally: + if _conn is not None: + _conn.close() + + return s3cred + +def uploadbytes(filename, filebytes, s3uri): + bucketname= s3uri.split("/")[3] + s3credentials= getcredentialsbybucketname(bucketname) + s3_access_key_id= s3credentials.s3accesskey + s3_secret_access_key= s3credentials.s3secretkey + retry = 0 + while True: + try: + auth = AWSRequestsAuth(aws_access_key=s3_access_key_id, + aws_secret_access_key=s3_secret_access_key, + aws_host=docservice_s3_host, + aws_region=docservice_s3_region, + aws_service=docservice_s3_service) + s3uri= s3uri+filename + response = requests.put(s3uri, data=None, auth=auth) + header = { + 'X-Amz-Date': response.request.headers['x-amz-date'], + 'Authorization': response.request.headers['Authorization'], + 'Content-Type': 'application/octet-stream' #mimetypes.MimeTypes().guess_type(filename)[0] + } + + #upload to S3 + uploadresponse= requests.put(s3uri, data=filebytes, headers=header) + uploadobj = {"uploadresponse": uploadresponse, "filename": filename, "documentpath": s3uri} + return uploadobj + except Exception as ex: + if retry > int(docservice_failureattempt): + logging.error("Error in uploading document to S3") + logging.error(ex) + uploadobj = {"success": False, "filename": filename, "documentpath": None} + raise ValueError(uploadobj, ex) + logging.info(f"uploadbytes s3retry = {retry}") + + retry += 1 + continue + finally: + if filebytes: + filebytes = None + del filebytes \ No newline at end of file diff --git a/computingservices/DocumentServices/services/zippingservice.py b/computingservices/DocumentServices/services/zippingservice.py new file mode 100644 index 000000000..2a9489513 --- /dev/null +++ b/computingservices/DocumentServices/services/zippingservice.py @@ -0,0 +1,28 @@ + +import json +from rstreamio.writer.zipperstreamwriter import zipperstreamwriter + +class zippingservice(): + + def sendtozipper(self, summaryfiles, message): + updatedmessage = zippingservice().preparemessageforzipperservice(summaryfiles, message) + zipperstreamwriter().sendmessage(updatedmessage) + + def preparemessageforzipperservice(self,summaryfiles, message): + try: + msgjson= json.loads(message) + if summaryfiles and len(summaryfiles) > 0: + filestozip_list = msgjson['filestozip']+summaryfiles + else: + filestozip_list = msgjson['filestozip'] + print('filestozip_list: ', filestozip_list) + msgjson['filestozip'] = self.to_json(filestozip_list) + msgjson['attributes'] = self.to_json(msgjson['attributes']) + msgjson['summarydocuments'] = self.to_json(msgjson['summarydocuments']) + return msgjson + except (Exception) as error: + print('error occured in redaction summary service: ', error) + + def to_json(self, obj): + return json.dumps(obj, default=lambda obj: obj.__dict__) + diff --git a/computingservices/DocumentServices/templates/redline_redaction_summary.docx b/computingservices/DocumentServices/templates/redline_redaction_summary.docx new file mode 100644 index 000000000..e08da9f00 Binary files /dev/null and b/computingservices/DocumentServices/templates/redline_redaction_summary.docx differ diff --git a/computingservices/DocumentServices/templates/responsepackage_redaction_summary.docx b/computingservices/DocumentServices/templates/responsepackage_redaction_summary.docx new file mode 100644 index 000000000..86a3730ab Binary files /dev/null and b/computingservices/DocumentServices/templates/responsepackage_redaction_summary.docx differ diff --git a/computingservices/DocumentServices/unittests/__init__.py b/computingservices/DocumentServices/unittests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/computingservices/DocumentServices/unittests/files/sample.pdf b/computingservices/DocumentServices/unittests/files/sample.pdf new file mode 100644 index 000000000..af2949c43 Binary files /dev/null and b/computingservices/DocumentServices/unittests/files/sample.pdf differ diff --git a/computingservices/DocumentServices/unittests/testredactionsummary.py b/computingservices/DocumentServices/unittests/testredactionsummary.py new file mode 100644 index 000000000..abe150346 --- /dev/null +++ b/computingservices/DocumentServices/unittests/testredactionsummary.py @@ -0,0 +1,27 @@ +import os +import random +import redis +import time +from walrus import Database +import json +from dotenv import load_dotenv +load_dotenv() + +STREAM_KEY = os.getenv('DOCUMENTSERVICE_STREAM_KEY') + +redishost = os.getenv('DOCUMENTSERVICE_REDIS_HOST') +redisport = os.getenv('DOCUMENTSERVICE_REDIS_PORT') +redispassword = os.getenv('DOCUMENTSERVICE_REDIS_PASSWORD') + +db = Database(host=redishost, port=redisport, db=0,password=redispassword) +stream = db.Stream(STREAM_KEY) +encoder = json.JSONEncoder() +while True: + + jobs_dict = {'jobid': 51, 'category': 'harms', 'requestnumber': 'EDU-2023-04040757', 'bcgovcode': 'EDU', 'createdby': 'foiedu@idir', 'requestid': '518', 'ministryrequestid': '520', 'attributes': '[{"divisionname": "Learning and Education Programs", "divisionid": 3, "files": [{"s3uripath": "https://citz-foi-prod.objectstore.gov.bc.ca/edu-dev-e/EDU-2023-28030430/5770c34d-c16e-4bbc-bcf5-837edab9a94c.png", "lastmodified": "2023-03-15T18:09:35.883Z", "recordid": 98, "filename": "download.png"},{"s3uripath": "https://citz-foi-prod.objectstore.gov.bc.ca/edu-dev-e/EDU-2023-29030230/a512a03f-bdb0-439c-bb39-6cb868ef3887.jpg", "lastmodified": "2023-03-29T18:44:26.245Z", "recordid": 92, "filename": "blueprint.jpg"},{"s3uripath":"https://citz-foi-prod.objectstore.gov.bc.ca/edu-dev-e/EDU-2023-03041156/b9efa6e2-d7fd-4bfd-9c9f-f549164f35e2.png","filename":"MicrosoftTeams-image.png", "lastmodified": "11/23/2022 23:26:21", "recordid": null}]}]'} + + job_id = stream.add(jobs_dict, id="*") + print(f"Created job {job_id}:") + + time.sleep(random.randint(5, 10)) + \ No newline at end of file diff --git a/computingservices/DocumentServices/utils/__init__.py b/computingservices/DocumentServices/utils/__init__.py new file mode 100644 index 000000000..59dc1f2f3 --- /dev/null +++ b/computingservices/DocumentServices/utils/__init__.py @@ -0,0 +1,3 @@ +from .foiredisstreamdb import redisstreamdb +from .foidocumentserviceconfig import * +from .dbconnection import getdbconnection, getfoidbconnection diff --git a/computingservices/DocumentServices/utils/dbconnection.py b/computingservices/DocumentServices/utils/dbconnection.py new file mode 100644 index 000000000..7dd1f67fb --- /dev/null +++ b/computingservices/DocumentServices/utils/dbconnection.py @@ -0,0 +1,18 @@ +import psycopg2 +from . import docservice_db_user,docservice_db_port,docservice_db_host,docservice_db_name,docservice_db_password,foi_db_user,foi_db_port,foi_db_host,foi_db_name,foi_db_password + +def getdbconnection(): + conn = psycopg2.connect( + host=docservice_db_host, + database=docservice_db_name, + user=docservice_db_user, + password=docservice_db_password,port=docservice_db_port) + return conn + +def getfoidbconnection(): + conn = psycopg2.connect( + host=foi_db_host, + database=foi_db_name, + user=foi_db_user, + password=foi_db_password,port=foi_db_port) + return conn \ No newline at end of file diff --git a/computingservices/DocumentServices/utils/foidocumentserviceconfig.py b/computingservices/DocumentServices/utils/foidocumentserviceconfig.py new file mode 100644 index 000000000..6c4cce8f4 --- /dev/null +++ b/computingservices/DocumentServices/utils/foidocumentserviceconfig.py @@ -0,0 +1,45 @@ +import os +import logging +import requests + +from dotenv import load_dotenv + +load_dotenv() + + +redishost = os.getenv("REDIS_HOST") +redisport = os.getenv("REDIS_PORT") +redispassword = os.getenv("REDIS_PASSWORD") +documentservice_stream_key = os.getenv("DOCUMENTSERVICE_STREAM_KEY") + +docservice_db_host = os.getenv("DOCUMENTSERVICE_DB_HOST") +docservice_db_name = os.getenv("DOCUMENTSERVICE_DB_NAME") +docservice_db_port = os.getenv("DOCUMENTSERVICE_DB_PORT") +docservice_db_user = os.getenv("DOCUMENTSERVICE_DB_USER") +docservice_db_password = os.getenv("DOCUMENTSERVICE_DB_PASSWORD") + +foi_db_host = os.getenv("FOI_DB_HOST") +foi_db_name = os.getenv("FOI_DB_NAME") +foi_db_port = os.getenv("FOI_DB_PORT") +foi_db_user = os.getenv("FOI_DB_USER") +foi_db_password = os.getenv("FOI_DB_PASSWORD") + +docservice_s3_host = os.getenv("DOCUMENTSERVICE_S3_HOST") +docservice_s3_region = os.getenv("DOCUMENTSERVICE_S3_REGION") +docservice_s3_service = os.getenv("DOCUMENTSERVICE_S3_SERVICE") +docservice_s3_env = os.getenv("DOCUMENTSERVICE_S3_ENV") + +docservice_failureattempt = os.getenv('DOCUMENTSERVICE_FAILUREATTEMPT', 3) + + +# Zipper stream config +zipperredishost = os.getenv("ZIPPER_REDIS_HOST") +zipperredisport = os.getenv("ZIPPER_REDIS_PORT") +zipperredispassword = os.getenv("ZIPPER_REDIS_PASSWORD") +zipper_stream_key = os.getenv("ZIPPER_STREAM_KEY") + + +cdogs_base_url = os.getenv("CDOGS_BASE_URL") +cdogs_token_url = os.getenv("CDOGS_TOKEN_URL") +cdogs_service_client = os.getenv("CDOGS_SERVICE_CLIENT") +cdogs_service_client_secret = os.getenv("CDOGS_SERVICE_CLIENT_SECRET") diff --git a/computingservices/DocumentServices/utils/foiredisstreamdb.py b/computingservices/DocumentServices/utils/foiredisstreamdb.py new file mode 100644 index 000000000..09af2d8ee --- /dev/null +++ b/computingservices/DocumentServices/utils/foiredisstreamdb.py @@ -0,0 +1,5 @@ +from walrus import Database +from .foidocumentserviceconfig import redishost, redisport, redispassword + + +redisstreamdb = Database(host=str(redishost), port=str(redisport), db=0,password=str(redispassword)) \ No newline at end of file diff --git a/computingservices/DocumentServices/utils/jsonmessageparser.py b/computingservices/DocumentServices/utils/jsonmessageparser.py new file mode 100644 index 000000000..a66d80ad4 --- /dev/null +++ b/computingservices/DocumentServices/utils/jsonmessageparser.py @@ -0,0 +1,7 @@ +import json +from models import s3credentials + +def gets3credentialsobject(s3cred_json): + j = json.loads(s3cred_json) + messageobject = s3credentials(**j) + return messageobject diff --git a/computingservices/ZippingServices/models/zipperproducermessage.py b/computingservices/ZippingServices/models/zipperproducermessage.py index 55f7aac2a..068642334 100644 --- a/computingservices/ZippingServices/models/zipperproducermessage.py +++ b/computingservices/ZippingServices/models/zipperproducermessage.py @@ -1,5 +1,5 @@ class zipperproducermessage(object): - def __init__(self,jobid,requestid,category,requestnumber,bcgovcode,createdby,ministryrequestid,filestozip,finaloutput,attributes) -> None: + def __init__(self,jobid,requestid,category,requestnumber,bcgovcode,createdby,ministryrequestid,filestozip,finaloutput,attributes,summarydocuments=None,redactionlayerid=None) -> None: self.jobid = jobid self.requestid = requestid self.category=category @@ -10,3 +10,5 @@ def __init__(self,jobid,requestid,category,requestnumber,bcgovcode,createdby,min self.filestozip = filestozip self.finaloutput = finaloutput self.attributes = attributes + self.summarydocuments = summarydocuments + self.redactionlayerid = redactionlayerid diff --git a/computingservices/ZippingServices/services/zipperservice.py b/computingservices/ZippingServices/services/zipperservice.py index dda549e79..8f7a2f3d7 100644 --- a/computingservices/ZippingServices/services/zipperservice.py +++ b/computingservices/ZippingServices/services/zipperservice.py @@ -106,9 +106,10 @@ def __zipfilesandupload(_message, s3credentials): tp, "w", zipfile.ZIP_DEFLATED, compresslevel=9, allowZip64=True ) as zip: _jsonfiles = json.loads(_message.filestozip) + print("\n_jsonfiles:",_jsonfiles) for fileobj in _jsonfiles: filename = fileobj["filename"] - + print("\nfilename:",filename) zip.writestr( filename, __getdocumentbytearray(fileobj, s3credentials) ) @@ -116,7 +117,7 @@ def __zipfilesandupload(_message, s3credentials): tp.seek(0) zipped_bytes = tp.read() filepath = __getzipfilepath(_message.category, _message.requestnumber) - logging.info("zipfilename = %s", filepath) + print("zipfilename = %s", filepath) docobj = uploadbytes( filepath, zipped_bytes, diff --git a/docker-compose.yml b/docker-compose.yml index 675d4b166..e5324a0be 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -74,6 +74,10 @@ services: - ZIPPER_REDIS_PASSWORD=${DEDUPE_REDIS_PASSWORD} - ZIPPER_REDIS_PORT=${DEDUPE_REDIS_PORT} - ZIPPER_STREAM_KEY=${ZIPPER_STREAM_KEY} + - DOCUMENTSERVICE_REDIS_HOST=${DEDUPE_REDIS_HOST} + - DOCUMENTSERVICE_REDIS_PASSWORD=${DEDUPE_REDIS_PASSWORD} + - DOCUMENTSERVICE_REDIS_PORT=${DEDUPE_REDIS_PORT} + - DOCUMENTSERVICE_STREAM_KEY=${DOCUMENTSERVICE_STREAM_KEY} - BATCH_CONFIG=${BATCH_CONFIG} - REDLINE_SINGLE_PKG_MINISTRIES=${REDLINE_SINGLE_PKG_MINISTRIES} @@ -232,6 +236,51 @@ services: - ZIPPER_REDIS_PASSWORD=${DEDUPE_REDIS_PASSWORD} - ZIPPER_REDIS_PORT=${DEDUPE_REDIS_PORT} - ZIPPER_STREAM_KEY=${ZIPPER_STREAM_KEY} + + foi-docreviewer-documentservice: + container_name: foi-docreviewer-documentservice + depends_on: + - foi-docreviewer-db + - foi-docreviewer-redis + build: + context: ./computingservices/DocumentServices + dockerfile: Dockerfile.local + image: docreviewerdocumentserviceimage + stdin_open: true + tty: true + networks: + services-network: + aliases: + - docreviewerdocumentservice + environment: + - REDIS_HOST=${DOCUMENTSERVICE_REDIS_HOST} + - REDIS_PASSWORD=${DOCUMENTSERVICE_REDIS_PASSWORD} + - REDIS_PORT=${DOCUMENTSERVICE_REDIS_PORT} + - DOCUMENTSERVICE_STREAM_KEY=${DOCUMENTSERVICE_STREAM_KEY} + - ZIPPER_REDIS_HOST=${ZIPPER_REDIS_HOST} + - ZIPPER_REDIS_PASSWORD=${ZIPPER_REDIS_PASSWORD} + - ZIPPER_REDIS_PORT=${ZIPPER_REDIS_PORT} + - ZIPPER_STREAM_KEY=${ZIPPER_STREAM_KEY} + - DOCUMENTSERVICE_DB_HOST=${DOCUMENTSERVICE_DB_HOST} + - DOCUMENTSERVICE_DB_NAME=${DOCUMENTSERVICE_DB_NAME} + - DOCUMENTSERVICE_DB_PORT=${DOCUMENTSERVICE_DB_PORT} + - DOCUMENTSERVICE_DB_USER=${DOCUMENTSERVICE_DB_USER} + - DOCUMENTSERVICE_DB_PASSWORD=${DOCUMENTSERVICE_DB_PASSWORD} + - DOCUMENTSERVICE_S3_HOST=${DOCUMENTSERVICE_S3_HOST} + - DOCUMENTSERVICE_S3_REGION=${DOCUMENTSERVICE_S3_REGION} + - DOCUMENTSERVICE_S3_SERVICE=${DOCUMENTSERVICE_S3_SERVICE} + - DOCUMENTSERVICE_S3_ENV=${DOCUMENTSERVICE_S3_ENV} + - CDOGS_BASE_URL=${CDOGS_BASE_URL} + - CDOGS_TOKEN_URL=${CDOGS_TOKEN_URL} + - CDOGS_SERVICE_CLIENT=${CDOGS_SERVICE_CLIENT} + - CDOGS_SERVICE_CLIENT_SECRET=${CDOGS_SERVICE_CLIENT_SECRET} + - FOI_DB_HOST=${FOI_DB_HOST} + - FOI_DB_NAME=${FOI_DB_NAME} + - FOI_DB_PORT=${FOI_DB_PORT} + - FOI_DB_USER=${FOI_DB_USER} + - FOI_DB_PASSWORD=${FOI_DB_PASSWORD} + + volumes: dbdata: diff --git a/openshift/templates/documentservice/documentservice-build.yaml b/openshift/templates/documentservice/documentservice-build.yaml new file mode 100644 index 000000000..11edaecb6 --- /dev/null +++ b/openshift/templates/documentservice/documentservice-build.yaml @@ -0,0 +1,62 @@ +--- +kind: Template +apiVersion: template.openshift.io/v1 +metadata: + name: "${APP_NAME}-build-template" + creationTimestamp: +objects: +- kind: ImageStream + apiVersion: v1 + metadata: + name: "${APP_NAME}" +- kind: BuildConfig + apiVersion: v1 + metadata: + name: "${APP_NAME}-build" + labels: + app: "${APP_NAME}-build" + spec: + runPolicy: Serial + source: + type: Git + git: + uri: "${GIT_REPO_URL}" + ref: "${GIT_REF}" + contextDir: "${SOURCE_CONTEXT_DIR}" + strategy: + type: Docker + dockerStrategy: + dockerfilePath: "${DOCKER_FILE_PATH}" + pullSecret: + name: artifacts-pull-default-jmhvkc + output: + to: + kind: ImageStreamTag + name: "${APP_NAME}:latest" +parameters: +- name: APP_NAME + displayName: Name + description: The name assigned to all of the resources defined in this template. + required: true + value: reviewer-documentservice +- name: GIT_REPO_URL + displayName: Git Repo URL + description: The URL to your GIT repo. + required: true + value: https://github.com/bcgov/foi-docreviewer +- name: GIT_REF + displayName: Git Reference + description: The git reference or branch. + required: true + value: dev-marshal +- name: SOURCE_CONTEXT_DIR + displayName: Source Context Directory + description: The source context directory. + required: false + value: computingservices/DocumentServices +- name: DOCKER_FILE_PATH + displayName: Docker File Path + description: The path to the docker file defining the build. + required: false + value: "DockerFile.bcgov" + diff --git a/openshift/templates/documentservice/documentservice-deploy.yaml b/openshift/templates/documentservice/documentservice-deploy.yaml new file mode 100644 index 000000000..1bce18441 --- /dev/null +++ b/openshift/templates/documentservice/documentservice-deploy.yaml @@ -0,0 +1,272 @@ +--- +kind: Template +apiVersion: template.openshift.io/v1 +metadata: + annotations: + description: Deployment template for a reviewer-documentservice. + tags: "${APP_NAME}" + name: "${APP_NAME}-deploy" +objects: +- kind: DeploymentConfig + apiVersion: v1 + metadata: + name: "${APP_NAME}" + labels: + app: "${APP_NAME}" + app-group: "${APP_GROUP}" + template: "${APP_NAME}-deploy" + spec: + strategy: + type: Rolling + rollingParams: + updatePeriodSeconds: 1 + intervalSeconds: 1 + timeoutSeconds: 600 + maxUnavailable: 25% + maxSurge: 25% + triggers: + - type: ImageChange + imageChangeParams: + automatic: true + containerNames: + - "${APP_NAME}" + from: + kind: ImageStreamTag + namespace: "${IMAGE_NAMESPACE}" + name: "${IMAGE_APP_NAME}:${TAG_NAME}" + - type: ConfigChange + replicas: 1 + test: false + selector: + app: "${APP_NAME}" + deploymentconfig: "${APP_NAME}" + template: + metadata: + labels: + app: "${APP_NAME}" + app-group: "${APP_GROUP}" + deploymentconfig: "${APP_NAME}" + template: "${APP_NAME}-deploy" + spec: + containers: + - name: "${APP_NAME}" + image: "${APP_NAME}" + imagePullPolicy: Always + env: + - name: REDIS_HOST + valueFrom: + secretKeyRef: + name: "${SECRETS}" + key: DOCUMENTSERVICE_REDIS_HOST + - name: REDIS_PORT + valueFrom: + secretKeyRef: + name: "${SECRETS}" + key: DOCUMENTSERVICE_REDIS_PORT + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: "${REDIS_SECRETS}" + key: database-password + - name: DOCUMENTSERVICE_STREAM_KEY + valueFrom: + secretKeyRef: + name: "${SECRETS}" + key: DOCUMENTSERVICE_STREAM_KEY + - name: ZIPPER_REDIS_HOST + valueFrom: + secretKeyRef: + name: "${SECRETS}" + key: ZIPPER_REDIS_HOST + - name: ZIPPER_REDIS_PORT + valueFrom: + secretKeyRef: + name: "${SECRETS}" + key: ZIPPER_REDIS_PORT + - name: ZIPPER_REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: "${REDIS_SECRETS}" + key: database-password + - name: ZIPPER_STREAM_KEY + valueFrom: + secretKeyRef: + name: "${SECRETS}" + key: ZIPPER_STREAM_KEY + - name: DOCUMENTSERVICE_DB_HOST + valueFrom: + secretKeyRef: + name: "${SECRETS}" + key: DOCUMENTSERVICE_DB_HOST + - name: DOCUMENTSERVICE_DB_NAME + valueFrom: + secretKeyRef: + name: "${SECRETS}" + key: DOCUMENTSERVICE_DB_NAME + - name: DOCUMENTSERVICE_DB_USER + valueFrom: + secretKeyRef: + name: "${DB_SECRETS}" + key: app-db-username + - name: DOCUMENTSERVICE_DB_PASSWORD + valueFrom: + secretKeyRef: + name: "${DB_SECRETS}" + key: app-db-password + - name: DOCUMENTSERVICE_S3_HOST + valueFrom: + secretKeyRef: + name: "${SECRETS}" + key: DOCUMENTSERVICE_S3_HOST + - name: DOCUMENTSERVICE_S3_REGION + valueFrom: + secretKeyRef: + name: "${SECRETS}" + key: DOCUMENTSERVICE_S3_REGION + - name: DOCUMENTSERVICE_S3_SERVICE + valueFrom: + secretKeyRef: + name: "${SECRETS}" + key: DOCUMENTSERVICE_S3_SERVICE + - name: DOCUMENTSERVICE_S3_ENV + valueFrom: + secretKeyRef: + name: "${SECRETS}" + key: DOCUMENTSERVICE_S3_ENV + - name: CDOGS_BASE_URL + valueFrom: + secretKeyRef: + name: "${SECRETS}" + key: CDOGS_BASE_URL + - name: CDOGS_TOKEN_URL + valueFrom: + secretKeyRef: + name: "${SECRETS}" + key: CDOGS_TOKEN_URL + - name: CDOGS_SERVICE_CLIENT + valueFrom: + secretKeyRef: + name: "${SECRETS}" + key: CDOGS_SERVICE_CLIENT + - name: CDOGS_SERVICE_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: "${SECRETS}" + key: CDOGS_SERVICE_CLIENT_SECRET + - name: FOI_DB_HOST + valueFrom: + secretKeyRef: + name: "${SECRETS}" + key: FOI_DB_HOST + - name: FOI_DB_PORT + valueFrom: + secretKeyRef: + name: "${SECRETS}" + key: FOI_DB_PORT + - name: FOI_DB_NAME + valueFrom: + secretKeyRef: + name: "${SECRETS}" + key: FOI_DB_NAME + - name: FOI_DB_USER + valueFrom: + secretKeyRef: + name: "${SECRETS}" + key: FOI_DB_USER + - name: FOI_DB_PASSWORD + valueFrom: + secretKeyRef: + name: "${SECRETS}" + key: FOI_DB_PASSWORD + resources: + requests: + cpu: "50m" + memory: "250Mi" + limits: + cpu: "150m" + memory: "500Mi" + terminationMessagePath: "/dev/termination-log" + terminationMessagePolicy: File + imagePullPolicy: Always + restartPolicy: Always + terminationGracePeriodSeconds: 30 + dnsPolicy: ClusterFirst + securityContext: {} + schedulerName: default-scheduler +- kind: Service + apiVersion: v1 + metadata: + name: "${APP_NAME}" + creationTimestamp: + labels: + app: "${APP_NAME}" + app-group: "${APP_GROUP}" + template: "${APP_NAME}-deploy" + spec: + ports: + - name: 5000-tcp + protocol: TCP + port: 5000 + targetPort: 5000 + selector: + deploymentconfig: "${APP_NAME}" + type: ClusterIP + sessionAffinity: None + # status: + # loadBalancer: {} +- kind: Route + apiVersion: v1 + metadata: + name: "${APP_NAME}" + labels: + app: "${APP_NAME}" + app-group: "${APP_GROUP}" + template: "${APP_NAME}-deploy" + spec: + to: + kind: Service + name: "${APP_NAME}" + weight: 100 + port: + targetPort: 5000-tcp + tls: + termination: edge + wildcardPolicy: None + host: "${APP_NAME}-${TAG_NAME}.apps.silver.devops.gov.bc.ca" +parameters: +- name: APP_NAME + displayName: Name + description: The name assigned to all of the OpenShift resources associated to the + server instance. + required: true +- name: IMAGE_APP_NAME + displayName: Name + description: The name assigned to all of the OpenShift resources associated to the + server instance. + required: true +- name: APP_GROUP + displayName: App Group + description: The name assigned to all of the deployments in this project. + required: true + value: foi-docreviewer +- name: IMAGE_NAMESPACE + displayName: Image Namespace + required: true + description: The namespace of the OpenShift project containing the imagestream for + the application. +- name: TAG_NAME + displayName: Environment TAG name + description: The TAG name for this environment, e.g., dev, test, prod + required: true +- name: DB_SECRETS + displayName: Patroni DB Secrets + description: Name of secrets for all db values + required: true +- name: SECRETS + displayName: Documentservice Secrets + description: Name of secrets for all documentservice values + required: true +- name: REDIS_SECRETS + displayName: Redis Secrets + description: Name of secrets for redis + required: true \ No newline at end of file diff --git a/openshift/templates/documentservice/documentservice_deploy_param.yaml b/openshift/templates/documentservice/documentservice_deploy_param.yaml new file mode 100644 index 000000000..b84cbc87e --- /dev/null +++ b/openshift/templates/documentservice/documentservice_deploy_param.yaml @@ -0,0 +1,7 @@ +IMAGE_NAMESPACE= +IMAGE_APP_NAME=reviewer-documentservice +TAG_NAME= +DB_SECRETS= +APP_NAME=reviewer-documentservice +SECRETS= +REDIS_SECRETS= \ No newline at end of file diff --git a/openshift/templates/documentservice/documentservice_secrets.yaml b/openshift/templates/documentservice/documentservice_secrets.yaml new file mode 100644 index 000000000..2ca23a592 --- /dev/null +++ b/openshift/templates/documentservice/documentservice_secrets.yaml @@ -0,0 +1,103 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: ${NAME} + labels: + app: ${NAME} + name: ${NAME} +objects: + - apiVersion: v1 + kind: Secret + metadata: + name: ${NAME} + stringData: + DOCUMENTSERVICE_REDIS_HOST: "${DOCUMENTSERVICE_REDIS_HOST}" + DOCUMENTSERVICE_REDIS_PORT: "${DOCUMENTSERVICE_REDIS_PORT}" + DOCUMENTSERVICE_STREAM_KEY: '${DOCUMENTSERVICE_STREAM_KEY}' + ZIPPER_REDIS_HOST: '${ZIPPER_REDIS_HOST}' + ZIPPER_REDIS_PORT: '${ZIPPER_REDIS_PORT}' + ZIPPER_STREAM_KEY: '${ZIPPER_STREAM_KEY}' + DOCUMENTSERVICE_DB_HOST: '${DOCUMENTSERVICE_DB_HOST}' + DOCUMENTSERVICE_DB_NAME: '${DOCUMENTSERVICE_DB_NAME}' + DOCUMENTSERVICE_S3_HOST: '${DOCUMENTSERVICE_S3_HOST}' + DOCUMENTSERVICE_S3_REGION: '${DOCUMENTSERVICE_S3_REGION}' + DOCUMENTSERVICE_S3_SERVICE: '${DOCUMENTSERVICE_S3_SERVICE}' + DOCUMENTSERVICE_S3_ENV: '${DOCUMENTSERVICE_S3_ENV}' + CDOGS_BASE_URL: '${CDOGS_BASE_URL}' + CDOGS_TOKEN_URL: '${CDOGS_TOKEN_URL}' + CDOGS_SERVICE_CLIENT: '${CDOGS_SERVICE_CLIENT}' + CDOGS_SERVICE_CLIENT_SECRET: '${CDOGS_SERVICE_CLIENT_SECRET}' + FOI_DB_HOST: '${FOI_DB_HOST}' + FOI_DB_PORT: '${FOI_DB_PORT}' + FOI_DB_NAME: '${FOI_DB_NAME}' + FOI_DB_USER: '${FOI_DB_USER}' + FOI_DB_PASSWORD: '${FOI_DB_PASSWORD}' + type: Opaque + +parameters: + - name: NAME + description: The name for all created objects. + required: true + - name: DOCUMENTSERVICE_REDIS_HOST + description: DOCUMENTSERVICE_REDIS_HOST + required: true + - name: DOCUMENTSERVICE_REDIS_PORT + description: DOCUMENTSERVICE_REDIS_PORT + required: true + - name: DOCUMENTSERVICE_STREAM_KEY + description: DOCUMENTSERVICE_STREAM_KEY + required: true + - name: ZIPPER_REDIS_HOST + description: ZIPPER_REDIS_HOST + required: true + - name: ZIPPER_REDIS_PORT + description: ZIPPER_REDIS_PORT + required: true + - name: ZIPPER_STREAM_KEY + description: ZIPPER_STREAM_KEY + required: true + - name: DOCUMENTSERVICE_DB_HOST + description: DOCUMENTSERVICE_DB_HOST + required: true + - name: DOCUMENTSERVICE_DB_NAME + description: DOCUMENTSERVICE_DB_NAME + required: true + - name: DOCUMENTSERVICE_S3_HOST + description: DOCUMENTSERVICE_S3_HOST + required: true + - name: DOCUMENTSERVICE_S3_REGION + description: DOCUMENTSERVICE_S3_REGION + required: true + - name: DOCUMENTSERVICE_S3_SERVICE + description: DOCUMENTSERVICE_S3_SERVICE + required: true + - name: DOCUMENTSERVICE_S3_ENV + description: DOCUMENTSERVICE_S3_ENV + required: true + - name: FOI_DB_HOST + description: FOI_DB_HOST + required: true + - name: FOI_DB_PORT + description: FOI_DB_PORT + required: true + - name: FOI_DB_NAME + description: FOI_DB_NAME + required: true + - name: FOI_DB_USER + description: FOI_DB_USER + required: true + - name: FOI_DB_PASSWORD + description: FOI_DB_PASSWORD + required: true + - name: CDOGS_SERVICE_CLIENT_SECRET + description: CDOGS_SERVICE_CLIENT_SECRET + required: true + - name: CDOGS_SERVICE_CLIENT + description: CDOGS_SERVICE_CLIENT + required: true + - name: CDOGS_TOKEN_URL + description: CDOGS_TOKEN_URL + required: true + - name: CDOGS_BASE_URL + description: CDOGS_BASE_URL + required: true \ No newline at end of file diff --git a/openshift/templates/documentservice/documentservice_secrets_param.yaml b/openshift/templates/documentservice/documentservice_secrets_param.yaml new file mode 100644 index 000000000..92ba7d1fe --- /dev/null +++ b/openshift/templates/documentservice/documentservice_secrets_param.yaml @@ -0,0 +1,22 @@ +NAME=documentservice-secrets +DOCUMENTSERVICE_REDIS_HOST= +DOCUMENTSERVICE_REDIS_PORT= +DOCUMENTSERVICE_STREAM_KEY= +ZIPPER_REDIS_HOST= +ZIPPER_REDIS_PORT= +ZIPPER_STREAM_KEY=ZIPPERSERVICE +DOCUMENTSERVICE_DB_HOST= +DOCUMENTSERVICE_DB_NAME= +DOCUMENTSERVICE_S3_HOST= +DOCUMENTSERVICE_S3_REGION= +DOCUMENTSERVICE_S3_SERVICE= +DOCUMENTSERVICE_S3_ENV= +CDOGS_BASE_URL= +CDOGS_SERVICE_CLIENT= +CDOGS_SERVICE_CLIENT_SECRET= +CDOGS_TOKEN_URL= +FOI_DB_HOST= +FOI_DB_PORT= +FOI_DB_NAME= +FOI_DB_USER= +FOI_DB_PASSWORD= \ No newline at end of file diff --git a/web/src/components/FOI/Home/Redlining.js b/web/src/components/FOI/Home/Redlining.js index 9ac2d1d5d..8787e330c 100644 --- a/web/src/components/FOI/Home/Redlining.js +++ b/web/src/components/FOI/Home/Redlining.js @@ -689,9 +689,10 @@ const Redlining = React.forwardRef( set ) => { slicedsetofdoclist.forEach(async (filerow) => { - await createDocument(filerow.s3url, - { useDownloader: false } // Added to fix BLANK page issue - ).then(async (newDoc) => { + await createDocument(filerow.s3url, { + useDownloader: false, // Added to fix BLANK page issue + loadAsPDF: true, // Added to fix jpeg/pdf stitiching issue #2941 + }).then(async (newDoc) => { setpdftronDocObjects((_arr) => [ ..._arr, { @@ -2075,7 +2076,7 @@ const Redlining = React.forwardRef( const zipDocObj = { divisionid: null, divisionname: null, - files: [], + files: [], }; if (stitchedDocPath) { const stitchedDocPathArray = stitchedDocPath?.split("/"); @@ -2100,6 +2101,7 @@ const Redlining = React.forwardRef( zipDocObj.divisionname = divObj["divisionname"]; } zipServiceMessage.attributes.push(zipDocObj); + //zipServiceMessage.summarydocuments = redlineStitchInfo[divObj["divisionid"]]["documentids"]; if (divisionCountForToast === zipServiceMessage.attributes.length) { triggerDownloadRedlines(zipServiceMessage, (error) => { console.log(error); @@ -2749,13 +2751,7 @@ const Redlining = React.forwardRef( let IncompatableList = prepareRedlineIncompatibleMapping(res); setIncompatableList(IncompatableList); fetchDocumentRedlineAnnotations(requestid, documentids, currentLayer.name.toLowerCase()); - setRedlineZipperMessage({ - ministryrequestid: requestid, - category: getzipredlinecategory(layertype), - attributes: [], - requestnumber: res.requestnumber, - bcgovcode: res.bcgovcode, - }); + let stitchDocuments = {}; let documentsObjArr = []; let divisionstitchpages = []; @@ -2837,6 +2833,15 @@ const Redlining = React.forwardRef( setRedlineStitchInfo(stitchDoc); setIssingleredlinepackage(res.issingleredlinepackage); + setRedlineZipperMessage({ + ministryrequestid: requestid, + category: getzipredlinecategory(layertype), + attributes: [], + requestnumber: res.requestnumber, + bcgovcode: res.bcgovcode, + summarydocuments: prepareredlinesummarylist(stitchDocuments), + redactionlayerid: currentLayer.redactionlayerid + }); if(res.issingleredlinepackage == 'Y' || divisions.length == 1){ stitchSingleDivisionRedlineExport( _instance, @@ -2863,6 +2868,33 @@ const Redlining = React.forwardRef( ); }; + const prepareredlinesummarylist = (stitchDocuments) => { + let summarylist = [] + let alldocuments = [] + for (const [key, value] of Object.entries(stitchDocuments)) { + let summary_division = {}; + summary_division["divisionid"] = key + let documentlist = stitchDocuments[key]; + if(documentlist.length > 0) { + let summary_divdocuments = [] + for (let doc of documentlist) { + summary_divdocuments.push(doc.documentid); + alldocuments.push(doc); + } + summary_division["documentids"] = summary_divdocuments; + } + summarylist.push(summary_division); + } + let sorteddocids = [] + let sorteddocs = sortByLastModified(alldocuments) + for (const sorteddoc of sorteddocs) { + sorteddocids.push(sorteddoc['documentid']); + } + + return {"sorteddocuments": sorteddocids, "pkgdocuments": summarylist} + } + + const stitchForRedlineExport = async ( _instance, divisionDocuments, @@ -3128,7 +3160,7 @@ const Redlining = React.forwardRef( redlineSinglePackage ); } else { - let formattedAnnotationXML = formatAnnotationsForRedline( + let formattedAnnotationXML = formatAnnotationsForRedline( redlineDocumentAnnotations, redlinepageMappings["divpagemappings"][divisionid], redlineStitchInfo[divisionid]["documentids"] @@ -3312,7 +3344,7 @@ const Redlining = React.forwardRef( }); }; - const triggerRedlineZipper = ( + const triggerRedlineZipper = ( divObj, stitchedDocPath, divisionCountForToast, @@ -3335,13 +3367,14 @@ const Redlining = React.forwardRef( _instance ) => { const downloadType = "pdf"; - let zipServiceMessage = { ministryrequestid: requestid, category: "responsepackage", attributes: [], requestnumber: "", bcgovcode: "", + summarydocuments : prepareresponseredlinesummarylist(documentList), + redactionlayerid: currentLayer.redactionlayerid }; getResponsePackagePreSignedUrl( @@ -3492,6 +3525,27 @@ const Redlining = React.forwardRef( ); }; + const prepareresponseredlinesummarylist = (documentlist) => { + let summarylist = [] + let summary_division = {}; + let summary_divdocuments = []; + let alldocuments = []; + summary_division["divisionid"] = '0'; + for (let doc of documentlist) { + summary_divdocuments.push(doc.documentid); + alldocuments.push(doc); + } + summary_division["documentids"] = summary_divdocuments; + summarylist.push(summary_division); + + let sorteddocids = [] + let sorteddocs = sortByLastModified(alldocuments) + for (const sorteddoc of sorteddocs) { + sorteddocids.push(sorteddoc['documentid']); + } + return {"sorteddocuments": sorteddocids, "pkgdocuments": summarylist} + } + const compareValues = (a, b) => { if (modalSortNumbered) { if (modalSortAsc) {