Merge pull request #1009 from bcgov/dev-AS-FOIMOD-3073

FOIMOD-3073 - Fees - Remaining Balance Check

api/migrations/versions/9d45ce57481e_.py

@@ -0,0 +1,35 @@
+"""empty message
+
+Revision ID: 9d45ce57481e
+Revises: 18a45d1b33cc
+Create Date: 2024-06-06 10:19:45.739225
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '9d45ce57481e'
+down_revision = '18a45d1b33cc'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table('PDFStitchJobAttributes',
+        sa.Column('attributesid', sa.Integer(), autoincrement=True, nullable=False),
+        sa.Column('pdfstitchjobid', sa.Integer(), primary_key=True, autoincrement=True, nullable=False),
+        sa.Column('version', sa.Integer(), nullable=False),
+        sa.Column('ministryrequestid', sa.Integer(), nullable=False),        
+        sa.Column('attributes', postgresql.JSON(astext_type=sa.Text()), nullable=False),
+        sa.Column('createdat', sa.TIMESTAMP, nullable=False, server_default=sa.func.now()),
+        sa.Column('createdby', sa.String(length=120), nullable=True), 
+        sa.PrimaryKeyConstraint('attributesid'),
+        sa.ForeignKeyConstraint(['pdfstitchjobid', 'version'], ['PDFStitchJob.pdfstitchjobid', 'PDFStitchJob.version'], )
+    )
+
+
+def downgrade():
+    op.drop_table('PDFStitchJobAttributes')
+

api/reviewer_api/models/DocumentPageflagHistory.py

@@ -33,7 +33,10 @@ def createpageflag(cls, documentpageflaghistory)->DefaultMethodResult:
         # no db close or commit because this function is called in a loop
         try:
             db.session.add(documentpageflaghistory)
+            db.session.commit()
             return DefaultMethodResult(True, 'Page Flag history is saved', documentpageflaghistory.id)  
         except Exception as ex:
             logging.error(ex)
             return DefaultMethodResult(False, 'Page Flag history is not saved', documentpageflaghistory.id)
+        finally:
+            db.session.close()

api/reviewer_api/models/Keywords.py

@@ -3,7 +3,7 @@
 from sqlalchemy import or_
 from datetime import datetime as datetime2
 from sqlalchemy import or_, and_
-
+import logging
 class Keyword(db.Model):
     __tablename__ = 'Keywords' 
     # Defining the columns
@@ -19,10 +19,14 @@ class Keyword(db.Model):
 
     @classmethod
     def getall(cls):
-        keyword_schema = KeywordSchema(many=True)
-        query = db.session.query(Keyword).filter_by(isactive=True).all()
-        return keyword_schema.dump(query)
-
+        try:
+            keyword_schema = KeywordSchema(many=True)
+            query = db.session.query(Keyword).filter_by(isactive=True).all()
+            return keyword_schema.dump(query)
+        except Exception as ex:
+            logging.error(ex)
+        finally:
+            db.session.close()
 
 class KeywordSchema(ma.Schema):
     class Meta:

api/reviewer_api/models/PDFStitchJobAttributes.py

@@ -0,0 +1,66 @@
+from .db import db, ma
+from datetime import datetime as datetime2
+from sqlalchemy.dialects.postgresql import JSON
+from sqlalchemy import func, and_
+from .default_method_result import DefaultMethodResult
+from .DocumentDeleted import DocumentDeleted
+from .DocumentMaster import DocumentMaster
+import logging
+
+
+class PDFStitchJobAttributes(db.Model):
+    __tablename__ = "PDFStitchJobAttributes"
+    # Defining the columns
+    attributesid = db.Column(db.Integer, primary_key=True, autoincrement=True)
+    pdfstitchjobid = db.Column(db.Integer, db.ForeignKey("PDFStitchJob.pdfstitchjobid"))
+    version = db.Column(db.Integer, db.ForeignKey("PDFStitchJob.version"))
+    ministryrequestid = db.Column(db.Integer, nullable=False)
+    attributes = db.Column(JSON, unique=False, nullable=False)
+    createdat = db.Column(db.DateTime, default=datetime2.now, nullable=False)
+    createdby = db.Column(db.String(120), nullable=False)
+
+
+    @classmethod
+    def insert(cls, row):
+        try:
+            db.session.add(row)
+            db.session.commit()
+            return DefaultMethodResult(
+                True,
+                "PDF Stitch Job Attributes recorded for ministryrequestid: {0}".format(
+                    row.ministryrequestid
+                ),
+                row.pdfstitchjobid,
+            )
+        except Exception as ex:
+            logging.error(ex)
+        finally:
+            db.session.close()
+
+    @classmethod
+    def getpdfstitchjobattributesbyid(cls, requestid):
+        try:
+            pdfstitchjobattributesschema = PDFStitchJobAttributesSchema(many=False)
+            query = db.session.query(PDFStitchJobAttributes).filter(
+                    PDFStitchJobAttributes.ministryrequestid == requestid
+                ).first()
+            return pdfstitchjobattributesschema.dump(query)
+        except Exception as ex:
+            logging.error(ex)
+        finally:
+            db.session.close()
+
+
+
+
+class PDFStitchJobAttributesSchema(ma.Schema):
+    class Meta:
+        fields = (
+            "attributesid",
+            "pdfstitchjobid",
+            "version",
+            "ministryrequestid",
+            "attributes",
+            "createdat",
+            "createdby",
+        )

api/reviewer_api/resources/document.py

@@ -29,6 +29,7 @@
 
 from reviewer_api.services.documentservice import documentservice
 from reviewer_api.services.docdeletedpageservice import docdeletedpageservice
+from reviewer_api.services.jobrecordservice import jobrecordservice
 
 API = Namespace('Document Services', description='Endpoints for deleting and replacing documents')
 TRACER = Tracer.get_instance()
@@ -99,13 +100,15 @@ def get(requestid):
             response.raise_for_status()
             # get request status
             jsonobj = response.json()
+            balancefeeoverrodforrequest = jobrecordservice().isbalancefeeoverrodforrequest(requestid)
             requestinfo = {
                 "bcgovcode": jsonobj["bcgovcode"],
                 "requesttype": jsonobj["requestType"],
                 "validoipcreviewlayer": documentservice().validate_oipcreviewlayer(jsonobj, requestid),
+                "balancefeeoverrodforrequest": balancefeeoverrodforrequest
             }
-            result = documentservice().getdocuments(requestid, requestinfo["bcgovcode"])
-            return json.dumps({"requeststatuslabel": jsonobj["requeststatuslabel"], "documents": result, "requestnumber":jsonobj["axisRequestId"], "requestinfo":requestinfo}), 200
+            documentdivisionslist,result = documentservice().getdocuments(requestid, requestinfo["bcgovcode"])
+            return json.dumps({"requeststatuslabel": "", "documents": result, "requestnumber":"", "requestinfo":requestinfo,"documentdivisions":documentdivisionslist}), 200
         except KeyError as error:
             return {'status': False, 'message': CUSTOM_KEYERROR_MESSAGE + str(error)}, 400
         except BusinessException as exception:

api/reviewer_api/schemas/finalpackage.py

@@ -9,6 +9,8 @@ class FileSchema(Schema):
 class AttributeSchema(Schema):
     files = fields.Nested(FileSchema, many=True, required=True, allow_none=False)
 
+class FeeOverrideSchema(Schema):
+    feeoverridereason = fields.Str(data_key="feeoverridereason", allow_none=True)
 
 class SummaryPkgSchema(Schema):
     divisionid = fields.Int(data_key="divisionid", allow_none=True)
@@ -27,4 +29,5 @@ class FinalPackageSchema(Schema):
         AttributeSchema, many=True, required=True, allow_none=False
     )
     summarydocuments = fields.Nested(SummarySchema, allow_none=True)
-    redactionlayerid = fields.Int(data_key="redactionlayerid", allow_none=False)
+    redactionlayerid = fields.Int(data_key="redactionlayerid", allow_none=False)
+    pdfstitchjobattributes = fields.Nested(FeeOverrideSchema, allow_none=True, many=False)

api/reviewer_api/services/jobrecordservice.py

@@ -6,6 +6,7 @@
 from reviewer_api.models.DocumentAttributes import DocumentAttributes
 from reviewer_api.services.annotationservice import annotationservice
 from reviewer_api.services.documentpageflagservice import documentpageflagservice
+from reviewer_api.models.PDFStitchJobAttributes import PDFStitchJobAttributes
 from reviewer_api.auth import auth, AuthHelper
 from datetime import datetime as datetime2
 from reviewer_api.utils.constants import FILE_CONVERSION_FILE_TYPES, DEDUPE_FILE_TYPES
@@ -129,3 +130,19 @@ def insertpagecalculatorjobstatus(self, message, userid):
                 )
         job = PageCalculatorJob.insert(row)
         return job
+
+    def insertfeeoverridereason(self, message, pdfstitchjobid, userid):
+        row = PDFStitchJobAttributes(
+                    pdfstitchjobid=pdfstitchjobid,
+                    version=1,
+                    ministryrequestid=message['ministryrequestid'],
+                    attributes=message['pdfstitchjobattributes'],
+                    createdby=userid
+                )
+        job = PDFStitchJobAttributes.insert(row)
+        return job
+
+    def isbalancefeeoverrodforrequest(self, requestid):
+        pdfstitchjobattributes= PDFStitchJobAttributes().getpdfstitchjobattributesbyid(requestid)
+        isbalancefeeoverrode= False if pdfstitchjobattributes is None or not pdfstitchjobattributes else True
+        return isbalancefeeoverrode

api/reviewer_api/services/radactionservice.py

@@ -123,13 +123,24 @@ def triggerdownloadredlinefinalpackage(self, finalpackageschema, userinfo):
             _jobmessage, userinfo["userid"]
         )
         if job.success:
+            if finalpackageschema['pdfstitchjobattributes'] is not None:
+                if 'feeoverridereason' in finalpackageschema['pdfstitchjobattributes']:
+                    feeoverridereason= finalpackageschema['pdfstitchjobattributes']['feeoverridereason']
+                    if feeoverridereason is not None and feeoverridereason != '':
+                        jobrecordservice().insertfeeoverridereason(finalpackageschema,job.identifier,userinfo["userid"])
             _message = self.__preparemessageforsummaryservice(
                 finalpackageschema, userinfo, job
             )
             return documentserviceproducerservice().add(_message)
 
     # redline/final package download: prepare message for zipping service
     def __preparemessageforsummaryservice(self, messageschema, userinfo, job):
+        feeoverridereason= None
+        pdf_stitch_job_attributes = to_json(messageschema['pdfstitchjobattributes'])
+        if pdf_stitch_job_attributes is not None:
+            feeoverridereason= json.loads(pdf_stitch_job_attributes).get("feeoverridereason", None) 
+            if feeoverridereason is not None and feeoverridereason != '':
+                feeoverridereason= userinfo["firstname"]+" "+userinfo["lastname"]+" overrode balance outstanding warning for the following reason: "+feeoverridereason
         _message = {
             "jobid": job.identifier,
             "requestid": -1,
@@ -144,7 +155,8 @@ def __preparemessageforsummaryservice(self, messageschema, userinfo, job):
             "finaloutput": to_json(""),
             "attributes": to_json(messageschema["attributes"]),
             "summarydocuments": json.dumps(messageschema["summarydocuments"]),
-            "redactionlayerid": json.dumps(messageschema["redactionlayerid"])
+            "redactionlayerid": json.dumps(messageschema["redactionlayerid"]),
+            "feeoverridereason":feeoverridereason
         }
         return _message
 

computingservices/DocumentServices/services/dal/documentpageflag.py

@@ -214,3 +214,26 @@ def getdeletedpages(cls, ministryrequestid, docids):
         finally:
             if conn is not None:
                 conn.close()
+
+    @classmethod
+    def getrecentredactionlayerid(cls, ministryrequestid):
+        conn = getdbconnection()
+        layerid = 1
+        try:
+            cursor = conn.cursor()
+            query = '''
+                select redactionlayerid  
+                from "DocumentPageflags" 
+                where foiministryrequestid = %s::integer
+                order by created_at desc limit 1;
+            '''
+            cursor.execute(query, (ministryrequestid,))
+            layerid = cursor.fetchone()
+            return layerid
+        except Exception as error:
+            logging.error("Error in getting recentredactionlayerid for requestid")
+            logging.error(error)
+        finally:
+            cursor.close()
+            if conn is not None:
+                conn.close()

computingservices/DocumentServices/services/dts/redactionsummary.py

@@ -20,7 +20,7 @@ def __getrangenumber(self, rangeval):
 
     def __packaggesummary(self, message, documentids, pageflags, programareas):
         try:
-            redactionlayerid = message.redactionlayerid
+            redactionlayerid = self.__getredactionlayerid(message)
             summarymsg = message.summarydocuments
             summaryobject = get_in_summary_object(summarymsg)
             ordereddocids = summaryobject.sorteddocuments
@@ -48,7 +48,7 @@ def __packaggesummary(self, message, documentids, pageflags, programareas):
                             pageflag['docpageflags'] = pageflag['docpageflags'] + self.__get_pagesection_mapping(filteredpages, docpagesections, docpageconsults)
                     skippages = self.__get_skippagenos(docpageflag['pageflag'], message.category)
                 pagecount = (pagecount+stitchedpagedata[docid]["pagecount"])-len(skippages)
-                
+
             for pageflag in _pageflags:
                 _data = {}
                 if len(pageflag['docpageflags']) > 0:
@@ -61,6 +61,11 @@ def __packaggesummary(self, message, documentids, pageflags, programareas):
         except (Exception) as error:
             print('error occured in redaction dts service: ', error)
 
+    def __getredactionlayerid(self, message):
+        if message.category == "responsepackage":
+            return documentpageflag().getrecentredactionlayerid(message.ministryrequestid)
+        return message.redactionlayerid 
+
     def __getdeletedpages(self, ministryid, ordereddocids):
         deletedpages = documentpageflag().getdeletedpages(ministryid, ordereddocids)
         documentpages = {}

computingservices/DocumentServices/services/redactionsummaryservice.py

@@ -33,16 +33,21 @@ def processmessage(self,incomingmessage):
                     template_path='templates/'+documenttypename+'.docx'
                     redaction_summary= documentgenerationservice().generate_pdf(formattedsummary, documenttypename,template_path)
                     messageattributes= json.loads(message.attributes)
+                    divisioname = None
                     if len(messageattributes)>1:
                         filesobj=(next(item for item in messageattributes if item['divisionid'] == divisionid))['files'][0]
+                        divisioname=(next(item for item in messageattributes if item['divisionid'] == divisionid))['divisionname'] if category not in ('responsepackage','oipcreviewredline') else None
+
                     else:
                         filesobj= messageattributes[0]['files'][0]
+                        divisioname =  messageattributes[0]['divisionname'] if category not in ('responsepackage','oipcreviewredline') else None  
+
                     stitcheddocs3uri = filesobj['s3uripath']
-                    stitcheddocfilename = filesobj['filename']
+                    stitcheddocfilename = filesobj['filename']                    
                     s3uricategoryfolder= "oipcreview" if category == 'oipcreviewredline' else category
                     s3uri = stitcheddocs3uri.split(s3uricategoryfolder+"/")[0] + s3uricategoryfolder+"/"
-                    filename = stitcheddocfilename.replace(".pdf","- summary.pdf")
-                    uploadobj= uploadbytes(filename,redaction_summary.content, s3uri)
+                    filename =self.__get_summaryfilename(message.requestnumber, category, divisioname, stitcheddocfilename) 
+                    uploadobj= uploadbytes(filename,redaction_summary.content,s3uri)
                     upload_responses.append(uploadobj)
                     if uploadobj["uploadresponse"].status_code == 200:
                         summaryuploaderror= False    
@@ -57,8 +62,19 @@ def processmessage(self,incomingmessage):
             print('error occured in redaction summary service: ', error)
             pdfstitchjobactivity().recordjobstatus(message,4,"redactionsummaryfailed",str(error),"summary generation failed")
             return summaryfilestozip
-
-
+
+    def __get_summaryfilename(self, requestnumber, category, divisionname, stitcheddocfilename):
+        stitchedfilepath = stitcheddocfilename[:stitcheddocfilename.rfind( '/')+1]
+        if category == 'responsepackage':
+            _filename = requestnumber
+        elif category == 'oipcreviewredline':
+            _filename = requestnumber+ ' - Redline'
+        else:
+            _filename = requestnumber+" - "+category
+            if divisionname not in (None, ''):
+                _filename = _filename+" - "+divisionname       
+        return stitchedfilepath+_filename+" - summary.pdf"
+
     def __get_pageflags(self, category):
         if category == "responsepackage":            
             return documentpageflag().get_all_pageflags(['Consult', 'Not Responsive', 'Duplicate'])

computingservices/DocumentServices/services/zippingservice.py

@@ -18,7 +18,8 @@ def preparemessageforzipperservice(self,summaryfiles, message):
             print('filestozip_list: ', filestozip_list)
             msgjson['filestozip'] = self.to_json(filestozip_list)   
             msgjson['attributes'] = self.to_json(msgjson['attributes'])
-            msgjson['summarydocuments'] = self.to_json(msgjson['summarydocuments'])         
+            msgjson['summarydocuments'] = self.to_json(msgjson['summarydocuments'])
+
             return msgjson
         except (Exception) as error:
             print('error occured in zipping service: ', error)

computingservices/ZippingServices/models/redlineresponsenotificationmessage.py

@@ -1,6 +1,7 @@
 class redlineresponsenotificationmessage(object):
-    def __init__(self, ministryrequestid, serviceid, errorflag, createdby) -> None:
+    def __init__(self, ministryrequestid, serviceid, errorflag, createdby,feeoverridereason) -> None:
         self.ministryrequestid = ministryrequestid
         self.serviceid = serviceid
         self.errorflag = errorflag
         self.createdby = createdby
+        self.feeoverridereason=feeoverridereason

computingservices/ZippingServices/models/zipperproducermessage.py

@@ -1,5 +1,5 @@
 class zipperproducermessage(object):
-    def __init__(self,jobid,requestid,category,requestnumber,bcgovcode,createdby,ministryrequestid,filestozip,finaloutput,attributes,summarydocuments=None,redactionlayerid=None,foldername=None) -> None:
+    def __init__(self,jobid,requestid,category,requestnumber,bcgovcode,createdby,ministryrequestid,filestozip,finaloutput,attributes,feeoverridereason,summarydocuments=None,redactionlayerid=None,foldername=None) -> None:
         self.jobid = jobid
         self.requestid = requestid
         self.category=category
@@ -13,3 +13,4 @@ def __init__(self,jobid,requestid,category,requestnumber,bcgovcode,createdby,min
         self.foldername = foldername
         self.summarydocuments = summarydocuments
         self.redactionlayerid = redactionlayerid
+        self.feeoverridereason= feeoverridereason

computingservices/ZippingServices/services/notificationservice.py

@@ -44,6 +44,7 @@ def __responsepackagepublishtostream(self, message, error=False):
                 serviceid="pdfstitchforresponsepackage",
                 createdby=message.createdby,
                 errorflag=self.__booltostr(error),
+                feeoverridereason= message.feeoverridereason
             )
 
             logging.info(