Merge pull request #1202 from bcgov/dev

Dev <> Main Merge - Release 11.0 (Consults)

MCS.FOI.S3FileConversion/MCS.FOI.MSGToPDF/MSGFileProcessor.cs

@@ -57,15 +57,25 @@ public MSGFileProcessor(Stream sourceStream)
                                     var _attachment = (Storage.Message)attachment;
                                     var filename = _attachment.FileName;
                                     var extension = Path.GetExtension(filename);
+                                    var baseFilename = Path.GetFileNameWithoutExtension(filename);
                                     if (!string.IsNullOrEmpty(extension))
                                     {
                                         _attachment.Save(attachmentStream);
                                         Dictionary<string, string> attachmentInfo = new Dictionary<string, string>();
-
+
+                                        // If the filename already exists, increment the duplicate count to create a unique filename
                                         if (fileNameHash.ContainsKey(filename))
                                         {
-
-                                            filename = Path.GetFileNameWithoutExtension(filename) + '1' + extension;
+                                            int duplicateCount = 1; // Initialize the duplicate count
+                                            string newFilename;
+
+                                            // Loop until a unique filename is found
+                                            do
+                                            {
+                                                newFilename = baseFilename + duplicateCount.ToString() + extension;
+                                                duplicateCount++;
+                                            } while (fileNameHash.ContainsKey(newFilename));
+                                            filename = newFilename;
                                         }
                                         fileNameHash.Add(filename, true);
 
@@ -91,19 +101,27 @@ public MSGFileProcessor(Stream sourceStream)
                                     var _attachment = (Storage.Attachment)attachment;
                                     var filename = _attachment.FileName;
                                     var extension = Path.GetExtension(filename);
-
+                                    var baseFilename = Path.GetFileNameWithoutExtension(filename);
                                     if (!string.IsNullOrEmpty(extension))
                                     {
                                         attachmentStream.Write(_attachment.Data, 0, _attachment.Data.Length);
                                         Dictionary<string, string> attachmentInfo = new Dictionary<string, string>();
-
+
+                                        // If the filename already exists, increment the duplicate count to create a unique filename
                                         if (fileNameHash.ContainsKey(filename))
                                         {
-
-                                            filename = Path.GetFileNameWithoutExtension(filename) + '1' + extension;
+                                            int duplicateCount = 1;  // Initialize the duplicate count
+                                            string newFilename;
+                                            // Loop until a unique filename is found
+                                            do
+                                            {
+                                                newFilename = baseFilename + '-' +duplicateCount.ToString() + extension;
+                                                duplicateCount++;
+                                            } while (fileNameHash.ContainsKey(newFilename));
+                                            filename = newFilename; // Set the unique filename
                                         }
                                         fileNameHash.Add(filename, true);
-                                        attachmentInfo.Add("filename", _attachment.FileName);
+                                        attachmentInfo.Add("filename", filename);
                                         attachmentInfo.Add("s3filename", filename);
                                         attachmentInfo.Add("cid", _attachment.ContentId);
                                         attachmentInfo.Add("size", _attachment.Data.Length.ToString());

api/reviewer_api/resources/document.py

@@ -126,7 +126,7 @@ def get(requestid):
             balancefeeoverrodforrequest = jobrecordservice().isbalancefeeoverrodforrequest(requestid)
             outstandingbalance=0
             if 'cfrfee' in jsonobj and 'feedata' in jsonobj['cfrfee'] and "balanceDue" in jsonobj['cfrfee']['feedata']:
-                outstandingbalancestr= jsonobj['cfrfee']['feedata']["balanceDue"]
+                outstandingbalancestr = jsonobj['cfrfee']['feedata']["balanceDue"]
                 outstandingbalance = float(outstandingbalancestr)
             requestinfo = {
                 "bcgovcode": jsonobj["bcgovcode"],

api/reviewer_api/resources/foiflowmasterdata.py

@@ -200,6 +200,8 @@ def post(ministryrequestid, redactionlayer="redline", layertype="redline"):
             packagetype = "redline"
             if redactionlayer == "oipc":
                 packagetype = "oipcreview" if layertype == "oipcreview" else "oipcredline"
+            if layertype == "consult":
+                packagetype = "consult"
 
             #check if is single redline package
             is_single_redline = is_single_redline_package(_bcgovcode, packagetype, requesttype)
@@ -215,7 +217,10 @@ def post(ministryrequestid, redactionlayer="redline", layertype="redline"):
                         filepath_put = "{0}/{2}/{1}/{0} - {2} - {1}.pdf".format(
                             filepathlist[0], division_name, packagetype
                         )
-
+                        if packagetype == "consult":
+                            filepath_put = "{0}/{2}/{2} - {1} - {0}.pdf".format(
+                                filepathlist[0], division_name, 'Consult'
+                            )
                         s3path_save = s3client.generate_presigned_url(
                             ClientMethod="get_object",
                             Params={

api/reviewer_api/resources/redaction.py

@@ -231,7 +231,7 @@ class AnnotationMetadata(Resource):
     @staticmethod
     @TRACER.trace()
     @cross_origin(origins=allowedorigins())
-    @auth.require
+    #@auth.require
     def get(ministryrequestid, redactionlayer):
         try:
             result = redactionservice().getannotationinfobyrequest(ministryrequestid, redactionlayer)

api/reviewer_api/schemas/finalpackage.py

@@ -32,6 +32,7 @@ class FinalPackageSchema(Schema):
     )
     summarydocuments = fields.Nested(SummarySchema, allow_none=True)
     redactionlayerid = fields.Int(data_key="redactionlayerid", allow_none=False)
+    pdfstitchjobattributes = fields.Nested(FeeOverrideSchema, allow_none=True, many=False)
     requesttype = fields.Str(data_key="requesttype", allow_none=False)
     pdfstitchjobattributes = fields.Nested(FeeOverrideSchema, allow_none=True, many=False)
 
@@ -58,5 +59,5 @@ class MCFFinalPackageSchema(Schema):
     )
     summarydocuments = fields.Nested(MCFSummarySchema, allow_none=True)
     redactionlayerid = fields.Int(data_key="redactionlayerid", allow_none=False)
-    requesttype = fields.Str(data_key="requesttype", allow_none=False)
     pdfstitchjobattributes = fields.Nested(FeeOverrideSchema, allow_none=True, many=False)
+    requesttype = fields.Str(data_key="requesttype", allow_none=False)

api/reviewer_api/services/radactionservice.py

@@ -159,6 +159,7 @@ def __preparemessageforsummaryservice(self, messageschema, userinfo, job):
             "attributes": to_json(messageschema["attributes"]),
             "summarydocuments": json.dumps(messageschema["summarydocuments"]),
             "redactionlayerid": json.dumps(messageschema["redactionlayerid"]),
+            "feeoverridereason":feeoverridereason,
             "requesttype": messageschema["requesttype"],
             "feeoverridereason":feeoverridereason
         }

api/reviewer_api/utils/enums.py

@@ -74,7 +74,8 @@ class MinistryTeamWithKeycloackGroup(Enum):
     ECC = "ECC Ministry Team"
     JED = "JED Ministry Team"
     COR = "COR Ministry Team"
-    HSG = "HSG Ministry Team"
+    HSG = "HSG Ministry Team",
+    LSB = "LSB Ministry Team"
 
     @staticmethod
     def list():

api/reviewer_api/utils/util.py

@@ -137,6 +137,8 @@ def getbatchconfig():
     return _begin, _size, _limit
 
 def is_single_redline_package(bcgovcode, packagetype, requesttype):
+    if packagetype == "consult":
+        return False
     if (packagetype == "oipcreview"):
         return True
     if REDLINE_SINGLE_PKG_MINISTRIES not in (None, ""):

computingservices/DocumentServices/rstreamio/message/schemas/redactionsummary.py

@@ -43,6 +43,7 @@ def __init__(self, jobid, requestid, ministryrequestid, category, requestnumber,
         self.attributes = attributes
         self.summarydocuments = summarydocuments
         self.redactionlayerid = redactionlayerid
+        self.feeoverridereason = feeoverridereason
         self.requesttype = requesttype
         self.feeoverridereason = feeoverridereason
 

computingservices/DocumentServices/services/redactionsummaryservice.py

@@ -15,11 +15,14 @@ def processmessage(self,incomingmessage):
         message = get_in_redactionsummary_msg(incomingmessage)
         print('\n 1. get_in_redactionsummary_msg is : {0}'.format(message))
         try:
+            category = message.category 
+            #Condition to handle consults packaages (no summary files to be created)
+            if category == "consultpackage": 
+                return summaryfilestozip
             pdfstitchjobactivity().recordjobstatus(message,3,"redactionsummarystarted")                      
             summarymsg = message.summarydocuments
             #Condition for handling oipcredline category
             bcgovcode= message.bcgovcode
-            category = message.category 
             requesttype = message.requesttype
             if bcgovcode == 'mcf' and requesttype == 'personal' and category == 'responsepackage':
                 documenttypename= 'CFD_responsepackage_redaction_summary'

computingservices/DocumentServices/services/zippingservice.py

@@ -15,7 +15,7 @@ def preparemessageforzipperservice(self,summaryfiles, message):
             if summaryfiles and len(summaryfiles) > 0:                
                 filestozip_list = json.loads(msgjson['filestozip'])+summaryfiles
             else:
-                filestozip_list = msgjson['filestozip']
+                filestozip_list = json.loads(msgjson['filestozip'])
             print('filestozip_list: ', filestozip_list)
             msgjson['filestozip'] = self.to_json(filestozip_list)   
             msgjson['attributes'] = self.to_json(msgjson['attributes'])

computingservices/ZippingServices/services/zipperservice.py

@@ -16,7 +16,6 @@
 import traceback
 import PyPDF2
 
-
 def processmessage(message):
     try:
         s3credentials = getcredentialsbybcgovcode(message.bcgovcode)
@@ -174,3 +173,20 @@ def __getzipfilepath(foldername, filename):
         if foldername is not None
         else filename + ".zip"
     )
+
+
+def __removesensitivecontent(documentbytes):
+    # clear metadata
+    reader2 = PyPDF2.PdfReader(BytesIO(documentbytes))
+    # Check if metadata exists.
+    if reader2.metadata is not None:
+        # Create a new PDF file without metadata.
+        writer = PyPDF2.PdfWriter()
+        # Copy pages from the original PDF to the new PDF.
+        for page_num in range(len(reader2.pages)):
+            page = reader2.pages[page_num]                
+            writer.add_page(page)        
+        #writer.remove_links() # to remove comments.
+        buffer = BytesIO()
+        writer.write(buffer)
+        return buffer.getvalue()

web/public/stylesheets/webviewer.css

@@ -19,6 +19,11 @@
     cursor: not-allowed !important;
 }
 
+.consult_package:disabled {
+    color: #999999 !important;
+    cursor: not-allowed !important;
+}
+
 .file-upload-toast {
     .Toastify__toast-body {
         > div:last-child {

web/src/actions/actionConstants.ts

@@ -20,6 +20,7 @@ const ACTION_CONSTANTS = {
   INC_REDACTION_LAYER: "INC_REDACTION_LAYER",
   SET_REQUEST_NUMBER:"SET_REQUEST_NUMBER",
   SET_DELETED_PAGES: "SET_DELETED_PAGES",
+  SET_PUBLIC_BODIES: "SET_PUBLIC_BODIES",
   FOI_PERSONAL_SECTIONS: "FOI_PERSONAL_SECTIONS",
   FOI_PERSONAL_PEOPLE: "FOI_PERSONAL_PEOPLE",
   FOI_PERSONAL_FILETYPES: "FOI_PERSONAL_FILETYPES",

web/src/actions/documentActions.ts

@@ -1,5 +1,13 @@
 import ACTION_CONSTANTS from "./actionConstants";
 
+type PublicBody = {
+    bcgovcode: string,
+    iaocode: string,
+    name: string,
+    isactive: boolean,
+    type: string,
+    programareaid: number
+}
 
 export const setRedactionInfo = (data: any) => (dispatch:any) =>{
     dispatch({
@@ -8,7 +16,6 @@ export const setRedactionInfo = (data: any) => (dispatch:any) =>{
     })
 }
 
-
 export const setIsPageLeftOff = (data: any) => (dispatch:any) =>{
     dispatch({
         type:ACTION_CONSTANTS.SET_IS_PAGE_LEFT_OFF,
@@ -93,6 +100,13 @@ export const setDeletedPages = (data: any) => (dispatch:any) =>{
     })
 }
 
+export const setPublicBodies = (data: PublicBody[]) => (dispatch:any) =>{
+    dispatch({
+        type:ACTION_CONSTANTS.SET_PUBLIC_BODIES,
+        payload:data
+    })
+}
+
 export const setFOIPersonalSections = (data: any) => (dispatch:any) =>{
     dispatch({
         type:ACTION_CONSTANTS.FOI_PERSONAL_SECTIONS,
@@ -116,4 +130,4 @@ export const setFOIPersonalSections = (data: any) => (dispatch:any) =>{
         type:ACTION_CONSTANTS.FOI_PERSONAL_VOLUMES,
         payload:data      
     })
-  }
+  }

web/src/apiManager/services/docReviewerService.tsx

@@ -4,7 +4,7 @@ import API from "../endpoints";
 import UserService from "../../services/UserService";
 import { setRedactionInfo, setIsPageLeftOff, setSections, 
   setDocumentList, setRequestStatus, setRedactionLayers, incrementLayerCount, setRequestNumber, setRequestInfo, setDeletedPages,
-  setFOIPersonalSections, setFOIPersonalPeople, setFOIPersonalFiletypes, setFOIPersonalVolumes
+  setFOIPersonalSections, setFOIPersonalPeople, setFOIPersonalFiletypes, setFOIPersonalVolumes, setPublicBodies
 } from "../../actions/documentActions";
 import { store } from "../../services/StoreService";
 import { number } from "yargs";
@@ -36,7 +36,6 @@ export const fetchDocuments = (
         store.dispatch(setRequestNumber(res.data.requestnumber) as any);
         store.dispatch(setRequestStatus(res.data.requeststatuslabel) as any);
         store.dispatch(setRequestInfo(res.data.requestinfo) as any);
-        // callback(__files, res.data.documentdivisions, res.data.requestinfo);
         callback(res.data.documents, res.data.documentdivisions, res.data.requestinfo);
       } else {
         throw new Error();
@@ -295,6 +294,7 @@ export const fetchPageFlagsMasterData = (
     .then((res:any) => {
       if (res.data || res.data === "") {
         callback(res.data);
+        store.dispatch(setPublicBodies(res.data.find((flag: any) => flag.name === 'Consult').programareas));
       } else {
         throw new Error();
       }

web/src/apiManager/services/foiOSSService.tsx

@@ -59,6 +59,8 @@ export const getFOIS3DocumentRedlinePreSignedUrl = (
 
     if (layertype === "oipcreview") {
         apiurl = apiurl + "/oipcreview"
+    } else if (layertype === "consult") {
+        apiurl = apiurl + "/consult"
     } else {
         apiurl = apiurl + "/" + layer
     }

web/src/components/FOI/App.scss

@@ -228,6 +228,10 @@ li.modal-message-list-item {
   min-height: 350px !important;
 }
 
+.consult-modal {
+  min-height: 500px !important;
+}
+
 .redline-checkmark {
   height:14px;
   width:14px;