Merge branch 'dev' of https://github.com/bcgov/foi-docreviewer into dev

MCS.FOI.S3FileConversion/MCS.FOI.MSGToPDF/MSGFileProcessor.cs

@@ -57,15 +57,25 @@ public MSGFileProcessor(Stream sourceStream)
                                     var _attachment = (Storage.Message)attachment;
                                     var filename = _attachment.FileName;
                                     var extension = Path.GetExtension(filename);
+                                    var baseFilename = Path.GetFileNameWithoutExtension(filename);
                                     if (!string.IsNullOrEmpty(extension))
                                     {
                                         _attachment.Save(attachmentStream);
                                         Dictionary<string, string> attachmentInfo = new Dictionary<string, string>();
-
+
+                                        // If the filename already exists, increment the duplicate count to create a unique filename
                                         if (fileNameHash.ContainsKey(filename))
                                         {
-
-                                            filename = Path.GetFileNameWithoutExtension(filename) + '1' + extension;
+                                            int duplicateCount = 1; // Initialize the duplicate count
+                                            string newFilename;
+
+                                            // Loop until a unique filename is found
+                                            do
+                                            {
+                                                newFilename = baseFilename + duplicateCount.ToString() + extension;
+                                                duplicateCount++;
+                                            } while (fileNameHash.ContainsKey(newFilename));
+                                            filename = newFilename;
                                         }
                                         fileNameHash.Add(filename, true);
 
@@ -91,19 +101,27 @@ public MSGFileProcessor(Stream sourceStream)
                                     var _attachment = (Storage.Attachment)attachment;
                                     var filename = _attachment.FileName;
                                     var extension = Path.GetExtension(filename);
-
+                                    var baseFilename = Path.GetFileNameWithoutExtension(filename);
                                     if (!string.IsNullOrEmpty(extension))
                                     {
                                         attachmentStream.Write(_attachment.Data, 0, _attachment.Data.Length);
                                         Dictionary<string, string> attachmentInfo = new Dictionary<string, string>();
-
+
+                                        // If the filename already exists, increment the duplicate count to create a unique filename
                                         if (fileNameHash.ContainsKey(filename))
                                         {
-
-                                            filename = Path.GetFileNameWithoutExtension(filename) + '1' + extension;
+                                            int duplicateCount = 1;  // Initialize the duplicate count
+                                            string newFilename;
+                                            // Loop until a unique filename is found
+                                            do
+                                            {
+                                                newFilename = baseFilename + '-' +duplicateCount.ToString() + extension;
+                                                duplicateCount++;
+                                            } while (fileNameHash.ContainsKey(newFilename));
+                                            filename = newFilename; // Set the unique filename
                                         }
                                         fileNameHash.Add(filename, true);
-                                        attachmentInfo.Add("filename", _attachment.FileName);
+                                        attachmentInfo.Add("filename", filename);
                                         attachmentInfo.Add("s3filename", filename);
                                         attachmentInfo.Add("cid", _attachment.ContentId);
                                         attachmentInfo.Add("size", _attachment.Data.Length.ToString());
@@ -521,6 +539,10 @@ private string GenerateHtmlfromMsg(Storage.Message msg)
                             <td>" + msg.Subject + "</td></tr>");
 
                 DateTime sentDate = Convert.ToDateTime(msg.SentOn);
+                if(sentDate == DateTime.MinValue)
+                {
+                    sentDate = Convert.ToDateTime(msg.CreationTime);
+                }
                 if (TimeZone.CurrentTimeZone.StandardName != "Pacific Standard Time")
                 {
 

api/migrations/versions/9d45ce57481e_.py

@@ -0,0 +1,35 @@
+"""empty message
+
+Revision ID: 9d45ce57481e
+Revises: 18a45d1b33cc
+Create Date: 2024-06-06 10:19:45.739225
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '9d45ce57481e'
+down_revision = '18a45d1b33cc'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table('PDFStitchJobAttributes',
+        sa.Column('attributesid', sa.Integer(), autoincrement=True, nullable=False),
+        sa.Column('pdfstitchjobid', sa.Integer(), nullable=False),
+        sa.Column('version', sa.Integer(), nullable=False),
+        sa.Column('ministryrequestid', sa.Integer(), nullable=False),        
+        sa.Column('attributes', postgresql.JSON(astext_type=sa.Text()), nullable=False),
+        sa.Column('createdat', sa.TIMESTAMP, nullable=False, server_default=sa.func.now()),
+        sa.Column('createdby', sa.String(length=120), nullable=True), 
+        sa.PrimaryKeyConstraint('attributesid'),
+        sa.ForeignKeyConstraint(['pdfstitchjobid', 'version'], ['PDFStitchJob.pdfstitchjobid', 'PDFStitchJob.version'], )
+    )
+
+
+def downgrade():
+    op.drop_table('PDFStitchJobAttributes')
+

api/reviewer_api/models/PDFStitchJobAttributes.py

@@ -0,0 +1,66 @@
+from .db import db, ma
+from datetime import datetime as datetime2
+from sqlalchemy.dialects.postgresql import JSON
+from sqlalchemy import func, and_
+from .default_method_result import DefaultMethodResult
+from .DocumentDeleted import DocumentDeleted
+from .DocumentMaster import DocumentMaster
+import logging
+
+
+class PDFStitchJobAttributes(db.Model):
+    __tablename__ = "PDFStitchJobAttributes"
+    # Defining the columns
+    attributesid = db.Column(db.Integer, primary_key=True, autoincrement=True)
+    pdfstitchjobid = db.Column(db.Integer, db.ForeignKey("PDFStitchJob.pdfstitchjobid"))
+    version = db.Column(db.Integer, db.ForeignKey("PDFStitchJob.version"))
+    ministryrequestid = db.Column(db.Integer, nullable=False)
+    attributes = db.Column(JSON, unique=False, nullable=False)
+    createdat = db.Column(db.DateTime, default=datetime2.now, nullable=False)
+    createdby = db.Column(db.String(120), nullable=False)
+
+
+    @classmethod
+    def insert(cls, row):
+        try:
+            db.session.add(row)
+            db.session.commit()
+            return DefaultMethodResult(
+                True,
+                "PDF Stitch Job Attributes recorded for ministryrequestid: {0}".format(
+                    row.ministryrequestid
+                ),
+                row.pdfstitchjobid,
+            )
+        except Exception as ex:
+            logging.error(ex)
+        finally:
+            db.session.close()
+
+    @classmethod
+    def getpdfstitchjobattributesbyid(cls, requestid):
+        try:
+            pdfstitchjobattributesschema = PDFStitchJobAttributesSchema(many=False)
+            query = db.session.query(PDFStitchJobAttributes).filter(
+                    PDFStitchJobAttributes.ministryrequestid == requestid
+                ).first()
+            return pdfstitchjobattributesschema.dump(query)
+        except Exception as ex:
+            logging.error(ex)
+        finally:
+            db.session.close()
+
+
+
+
+class PDFStitchJobAttributesSchema(ma.Schema):
+    class Meta:
+        fields = (
+            "attributesid",
+            "pdfstitchjobid",
+            "version",
+            "ministryrequestid",
+            "attributes",
+            "createdat",
+            "createdby",
+        )

api/reviewer_api/resources/document.py

@@ -29,6 +29,7 @@
 
 from reviewer_api.services.documentservice import documentservice
 from reviewer_api.services.docdeletedpageservice import docdeletedpageservice
+from reviewer_api.services.jobrecordservice import jobrecordservice
 
 API = Namespace('Document Services', description='Endpoints for deleting and replacing documents')
 TRACER = Tracer.get_instance()
@@ -122,10 +123,17 @@ def get(requestid):
             response.raise_for_status()
             # get request status
             jsonobj = response.json()
+            balancefeeoverrodforrequest = jobrecordservice().isbalancefeeoverrodforrequest(requestid)
+            outstandingbalance=0
+            if 'cfrfee' in jsonobj and 'feedata' in jsonobj['cfrfee'] and "balanceDue" in jsonobj['cfrfee']['feedata']:
+                outstandingbalancestr = jsonobj['cfrfee']['feedata']["balanceDue"]
+                outstandingbalance = float(outstandingbalancestr)
             requestinfo = {
                 "bcgovcode": jsonobj["bcgovcode"],
                 "requesttype": jsonobj["requestType"],
                 "validoipcreviewlayer": documentservice().validate_oipcreviewlayer(jsonobj, requestid),
+                "outstandingbalance": outstandingbalance,
+                "balancefeeoverrodforrequest": balancefeeoverrodforrequest
             }
             documentdivisionslist,result = documentservice().getdocuments(requestid, requestinfo["bcgovcode"])
             return json.dumps({"requeststatuslabel": jsonobj["requeststatuslabel"], "documents": result, "requestnumber":jsonobj["axisRequestId"], "requestinfo":requestinfo, "documentdivisions":documentdivisionslist}), 200

api/reviewer_api/resources/foiflowmasterdata.py

@@ -200,6 +200,8 @@ def post(ministryrequestid, redactionlayer="redline", layertype="redline"):
             packagetype = "redline"
             if redactionlayer == "oipc":
                 packagetype = "oipcreview" if layertype == "oipcreview" else "oipcredline"
+            if layertype == "consult":
+                packagetype = "consult"
 
             #check if is single redline package
             is_single_redline = is_single_redline_package(_bcgovcode, packagetype, requesttype)
@@ -215,7 +217,10 @@ def post(ministryrequestid, redactionlayer="redline", layertype="redline"):
                         filepath_put = "{0}/{2}/{1}/{0} - {2} - {1}.pdf".format(
                             filepathlist[0], division_name, packagetype
                         )
-
+                        if packagetype == "consult":
+                            filepath_put = "{0}/{2}/{2} - {1} - {0}.pdf".format(
+                                filepathlist[0], division_name, 'Consult'
+                            )
                         s3path_save = s3client.generate_presigned_url(
                             ClientMethod="get_object",
                             Params={

api/reviewer_api/resources/redaction.py

@@ -231,7 +231,7 @@ class AnnotationMetadata(Resource):
     @staticmethod
     @TRACER.trace()
     @cross_origin(origins=allowedorigins())
-    @auth.require
+    #@auth.require
     def get(ministryrequestid, redactionlayer):
         try:
             result = redactionservice().getannotationinfobyrequest(ministryrequestid, redactionlayer)

api/reviewer_api/schemas/finalpackage.py

@@ -10,6 +10,8 @@ class FileSchema(Schema):
 class AttributeSchema(Schema):
     files = fields.Nested(FileSchema, many=True, required=True, allow_none=False)
 
+class FeeOverrideSchema(Schema):
+    feeoverridereason = fields.Str(data_key="feeoverridereason", allow_none=True)
 
 class SummaryPkgSchema(Schema):
     divisionid = fields.Int(data_key="divisionid", allow_none=True)
@@ -30,7 +32,9 @@ class FinalPackageSchema(Schema):
     )
     summarydocuments = fields.Nested(SummarySchema, allow_none=True)
     redactionlayerid = fields.Int(data_key="redactionlayerid", allow_none=False)
+    pdfstitchjobattributes = fields.Nested(FeeOverrideSchema, allow_none=True, many=False)
     requesttype = fields.Str(data_key="requesttype", allow_none=False)
+    pdfstitchjobattributes = fields.Nested(FeeOverrideSchema, allow_none=True, many=False)
 
 class SummaryRecordSchema(Schema):
     recordname = fields.Str(data_key="recordname", allow_none=True)
@@ -55,4 +59,5 @@ class MCFFinalPackageSchema(Schema):
     )
     summarydocuments = fields.Nested(MCFSummarySchema, allow_none=True)
     redactionlayerid = fields.Int(data_key="redactionlayerid", allow_none=False)
-    requesttype = fields.Str(data_key="requesttype", allow_none=False)
+    pdfstitchjobattributes = fields.Nested(FeeOverrideSchema, allow_none=True, many=False)
+    requesttype = fields.Str(data_key="requesttype", allow_none=False)

api/reviewer_api/services/jobrecordservice.py

@@ -6,6 +6,7 @@
 from reviewer_api.models.DocumentAttributes import DocumentAttributes
 from reviewer_api.services.annotationservice import annotationservice
 from reviewer_api.services.documentpageflagservice import documentpageflagservice
+from reviewer_api.models.PDFStitchJobAttributes import PDFStitchJobAttributes
 from reviewer_api.auth import auth, AuthHelper
 from datetime import datetime as datetime2
 from reviewer_api.utils.constants import FILE_CONVERSION_FILE_TYPES, DEDUPE_FILE_TYPES
@@ -129,3 +130,19 @@ def insertpagecalculatorjobstatus(self, message, userid):
                 )
         job = PageCalculatorJob.insert(row)
         return job
+
+    def insertfeeoverridereason(self, message, pdfstitchjobid, userid):
+        row = PDFStitchJobAttributes(
+                    pdfstitchjobid=pdfstitchjobid,
+                    version=1,
+                    ministryrequestid=message['ministryrequestid'],
+                    attributes=message['pdfstitchjobattributes'],
+                    createdby=userid
+                )
+        job = PDFStitchJobAttributes.insert(row)
+        return job
+
+    def isbalancefeeoverrodforrequest(self, requestid):
+        pdfstitchjobattributes= PDFStitchJobAttributes().getpdfstitchjobattributesbyid(requestid)
+        isbalancefeeoverrode= False if pdfstitchjobattributes is None or not pdfstitchjobattributes else True
+        return isbalancefeeoverrode

api/reviewer_api/services/radactionservice.py

@@ -124,13 +124,26 @@ def triggerdownloadredlinefinalpackage(self, finalpackageschema, userinfo):
             _jobmessage, userinfo["userid"]
         )
         if job.success:
+            if 'pdfstitchjobattributes' in finalpackageschema and finalpackageschema['pdfstitchjobattributes'] is not None:
+                if 'feeoverridereason' in finalpackageschema['pdfstitchjobattributes']:
+                    feeoverridereason= finalpackageschema['pdfstitchjobattributes']['feeoverridereason']
+                    if feeoverridereason is not None and feeoverridereason != '':
+                        jobrecordservice().insertfeeoverridereason(finalpackageschema,job.identifier,userinfo["userid"])
             _message = self.__preparemessageforsummaryservice(
                 finalpackageschema, userinfo, job
             )
             return documentserviceproducerservice().add(_message)
 
     # redline/final package download: prepare message for zipping service
     def __preparemessageforsummaryservice(self, messageschema, userinfo, job):
+        feeoverridereason= ''
+        pdf_stitch_job_attributes = None
+        if 'pdfstitchjobattributes' in messageschema:
+            pdf_stitch_job_attributes = to_json(messageschema['pdfstitchjobattributes'])
+        if pdf_stitch_job_attributes is not None:
+            feeoverridereason= json.loads(pdf_stitch_job_attributes).get("feeoverridereason", None) 
+            if feeoverridereason is not None and feeoverridereason != '':
+                feeoverridereason= userinfo["firstname"]+" "+userinfo["lastname"]+" overrode balance outstanding warning for the following reason: "+feeoverridereason
         _message = {
             "jobid": job.identifier,
             "requestid": -1,
@@ -146,7 +159,9 @@ def __preparemessageforsummaryservice(self, messageschema, userinfo, job):
             "attributes": to_json(messageschema["attributes"]),
             "summarydocuments": json.dumps(messageschema["summarydocuments"]),
             "redactionlayerid": json.dumps(messageschema["redactionlayerid"]),
+            "feeoverridereason":feeoverridereason,
             "requesttype": messageschema["requesttype"],
+            "feeoverridereason":feeoverridereason
         }
         return _message
 

api/reviewer_api/utils/enums.py

@@ -74,7 +74,8 @@ class MinistryTeamWithKeycloackGroup(Enum):
     ECC = "ECC Ministry Team"
     JED = "JED Ministry Team"
     COR = "COR Ministry Team"
-    HSG = "HSG Ministry Team"
+    HSG = "HSG Ministry Team",
+    LSB = "LSB Ministry Team"
 
     @staticmethod
     def list():

api/reviewer_api/utils/util.py

@@ -137,6 +137,8 @@ def getbatchconfig():
     return _begin, _size, _limit
 
 def is_single_redline_package(bcgovcode, packagetype, requesttype):
+    if packagetype == "consult":
+        return False
     if (packagetype == "oipcreview"):
         return True
     if REDLINE_SINGLE_PKG_MINISTRIES not in (None, ""):

computingservices/DocumentServices/rstreamio/message/schemas/redactionsummary.py

@@ -30,7 +30,7 @@ def __init__(self, sorteddocuments, pkgdocuments) -> None:
 
 class RedactionSummaryMessage(object):
     def __init__(self, jobid, requestid, ministryrequestid, category, requestnumber, 
-                 bcgovcode, createdby, filestozip, finaloutput, attributes, summarydocuments ,redactionlayerid,requesttype) -> None:
+                 bcgovcode, createdby, filestozip, finaloutput, attributes, summarydocuments ,redactionlayerid, requesttype, feeoverridereason) -> None:
         self.jobid = jobid
         self.requestid = requestid
         self.ministryrequestid = ministryrequestid
@@ -43,7 +43,9 @@ def __init__(self, jobid, requestid, ministryrequestid, category, requestnumber,
         self.attributes = attributes
         self.summarydocuments = summarydocuments
         self.redactionlayerid = redactionlayerid
+        self.feeoverridereason = feeoverridereason
         self.requesttype = requesttype
+        self.feeoverridereason = feeoverridereason
 
 
 def get_in_redactionsummary_msg(producer_json):