From b85017889da4c273614657cc0f0dd8c97592ce91 Mon Sep 17 00:00:00 2001
From: Aman-Hundal <aman.hundal@gov.bc.ca>
Date: Tue, 20 Aug 2024 15:16:27 -0700
Subject: [PATCH 01/18] bug fix. no page flag being applied to page with no
 flags

---
 web/src/components/FOI/Home/utils.js | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/web/src/components/FOI/Home/utils.js b/web/src/components/FOI/Home/utils.js
index fe926edf5..ef736a94f 100644
--- a/web/src/components/FOI/Home/utils.js
+++ b/web/src/components/FOI/Home/utils.js
@@ -783,6 +783,9 @@ export const skipNRDocument = (documentPageFlags, pagecount, pageFlagTypes) => {
 
 export const findNROrDuplicatePageFlag = (pageFlags, docObj, pageFlagTypes) => {
   const docPageFlags = pageFlags.find(pageFlagObj => pageFlagObj.documentid === docObj.docid);
+  if (!docPageFlags) {
+    return false;
+  }
   for (let pageFlag of docPageFlags.pageflag) {
     if ((pageFlag.page === docObj.page && pageFlag.flagid === pageFlagTypes["Duplicate"]) || (pageFlag.page === docObj.page && pageFlag.flagid === pageFlagTypes["Not Responsive"])) {
       return pageFlag;

From 38502d83df454a813369a5e3e6afd8079f63f28f Mon Sep 17 00:00:00 2001
From: Abin Antony <abin.antony@aot-technologies.com>
Date: Tue, 20 Aug 2024 17:51:04 -0700
Subject: [PATCH 02/18] #FOIMOD-3416 Logs enabling

---
 .../services/dts/redactionsummary.py          | 57 +++++++++++--------
 .../services/redactionsummaryservice.py       | 14 +++--
 2 files changed, 40 insertions(+), 31 deletions(-)

diff --git a/computingservices/DocumentServices/services/dts/redactionsummary.py b/computingservices/DocumentServices/services/dts/redactionsummary.py
index a70208a3f..0a05f7098 100644
--- a/computingservices/DocumentServices/services/dts/redactionsummary.py
+++ b/computingservices/DocumentServices/services/dts/redactionsummary.py
@@ -2,6 +2,7 @@
 from rstreamio.message.schemas.redactionsummary import get_in_summary_object,get_in_summarypackage_object
 import json
 from collections import defaultdict
+import traceback
 
 class redactionsummary():
 
@@ -25,48 +26,54 @@ def __getrangenumber(self, rangeval):
 
     def __packaggesummary(self, message, documentids, pageflags, programareas):
         try:
-            # print("\nInside __packaggesummary")
+            print("\nInside __packaggesummary")
             redactionlayerid = self.__getredactionlayerid(message)
             summarymsg = message.summarydocuments
             summaryobject = get_in_summary_object(summarymsg)
             ordereddocids = summaryobject.sorteddocuments
             stitchedpagedata = documentpageflag().getpagecount_by_documentid(message.ministryrequestid, ordereddocids)
             totalpagecount = self.__calculate_totalpages(stitchedpagedata)
-            # print("\ntotalpagecount",totalpagecount)
+            print("\n __packaggesummary stitchedpagedata",stitchedpagedata)
+            print("\n __packaggesummary totalpagecount",totalpagecount)
             
             if totalpagecount <=0:
                 return 
             _pageflags = self.__transformpageflags(pageflags)
-            # print("\n_pageflags",_pageflags)
+            print("\n_pageflags",_pageflags)
             summarydata = []
             docpageflags = documentpageflag().get_documentpageflag(message.ministryrequestid, redactionlayerid, ordereddocids)
             deletedpages = self.__getdeletedpages(message.ministryrequestid, ordereddocids)
             skippages= []     
             pagecount = 0
-            for docid in ordereddocids:
-                if docid in documentids:
-                    docdeletedpages = deletedpages[docid] if docid in deletedpages else []
-                    docpageflag = docpageflags[docid]
-                    for pageflag in _pageflags:
-                        filteredpages = self.__get_pages_by_flagid(docpageflag["pageflag"], docdeletedpages, pagecount, pageflag["pageflagid"], message.category)
-                        if len(filteredpages) > 0:
-                            originalpagenos = [pg['originalpageno'] for pg in filteredpages]
-                            docpagesections = documentpageflag().getsections_by_documentid_pageno(redactionlayerid, docid, originalpagenos)
-                            docpageconsults = self.__get_consults_by_pageno(programareas, docpageflag["pageflag"], filteredpages)
-                            pageflag['docpageflags'] = pageflag['docpageflags'] + self.__get_pagesection_mapping(filteredpages, docpagesections, docpageconsults)
-                    skippages = self.__get_skippagenos(docpageflag['pageflag'], message.category)
-                pagecount = (pagecount+stitchedpagedata[docid]["pagecount"])-len(skippages)
-            # print("\n_pageflags1",_pageflags)
-            for pageflag in _pageflags:
-                _data = {}
-                if len(pageflag['docpageflags']) > 0:
+            try:
+                for docid in ordereddocids:
+                    if docid in documentids:
+                        docdeletedpages = deletedpages[docid] if docid in deletedpages else []
+                        docpageflag = docpageflags[docid]
+                        for pageflag in _pageflags:
+                            filteredpages = self.__get_pages_by_flagid(docpageflag["pageflag"], docdeletedpages, pagecount, pageflag["pageflagid"], message.category)
+                            if len(filteredpages) > 0:
+                                originalpagenos = [pg['originalpageno'] for pg in filteredpages]
+                                docpagesections = documentpageflag().getsections_by_documentid_pageno(redactionlayerid, docid, originalpagenos)
+                                docpageconsults = self.__get_consults_by_pageno(programareas, docpageflag["pageflag"], filteredpages)
+                                pageflag['docpageflags'] = pageflag['docpageflags'] + self.__get_pagesection_mapping(filteredpages, docpagesections, docpageconsults)
+                        skippages = self.__get_skippagenos(docpageflag['pageflag'], message.category)
+                    pagecount = (pagecount+stitchedpagedata[docid]["pagecount"])-len(skippages)
+                print("\n_pageflags1",_pageflags)
+                for pageflag in _pageflags:
                     _data = {}
-                    _data["flagname"] = pageflag["header"].upper()
-                    _data["pagecount"] = len(pageflag['docpageflags'])  
-                    _data["sections"] = self.__format_redaction_summary(pageflag["description"], pageflag['docpageflags'], message.category)
-                    summarydata.append(_data)
+                    if len(pageflag['docpageflags']) > 0:
+                        _data = {}
+                        _data["flagname"] = pageflag["header"].upper()
+                        _data["pagecount"] = len(pageflag['docpageflags'])  
+                        _data["sections"] = self.__format_redaction_summary(pageflag["description"], pageflag['docpageflags'], message.category)
+                        summarydata.append(_data)
+            except (Exception) as err:
+                traceback.print_exc()
+                print('error occured in __packaggesummary redaction dts service: ', err)
             return {"requestnumber": message.requestnumber, "data": summarydata}
         except (Exception) as error:
+            traceback.print_exc()
             print('error occured in redaction dts service: ', error)
 
 
@@ -117,7 +124,7 @@ def __packagesummaryforcfdrequests(self, message, documentids):
             return {"requestnumber": message.requestnumber, "data": summarydata}
 
         except Exception as error:
-            print('Error occurred in redaction dts service: ', error)
+            print('CFD Error occurred in redaction dts service: ', error)
 
 
     def __calculate_range(self, mapped_flags, docids):
diff --git a/computingservices/DocumentServices/services/redactionsummaryservice.py b/computingservices/DocumentServices/services/redactionsummaryservice.py
index 9facce3e0..7193978bb 100644
--- a/computingservices/DocumentServices/services/redactionsummaryservice.py
+++ b/computingservices/DocumentServices/services/redactionsummaryservice.py
@@ -13,6 +13,7 @@ class redactionsummaryservice():
     def processmessage(self,incomingmessage):
         summaryfilestozip = []
         message = get_in_redactionsummary_msg(incomingmessage)
+        print('\n 1. get_in_redactionsummary_msg is : {0}'.format(message))
         try:
             pdfstitchjobactivity().recordjobstatus(message,3,"redactionsummarystarted")                      
             summarymsg = message.summarydocuments
@@ -23,22 +24,22 @@ def processmessage(self,incomingmessage):
                 documenttypename= 'CFD_responsepackage_redaction_summary'
             else:
                 documenttypename= category+"_redaction_summary" if category == 'responsepackage' else "redline_redaction_summary"
-            #print('documenttypename', documenttypename)
+            print('\n 2. documenttypename', documenttypename)
             upload_responses=[]
             pageflags = self.__get_pageflags(category)
             programareas = documentpageflag().get_all_programareas()
             messageattributes= json.loads(message.attributes)
-            #print("\nmessageattributes:",messageattributes)
+            print("\n 3. messageattributes:",messageattributes)
             divisiondocuments = get_in_summary_object(summarymsg).pkgdocuments
-            #print("\n divisiondocuments:",divisiondocuments)
+            print("\n 4. divisiondocuments:",divisiondocuments)
             for entry in divisiondocuments:
                 #print("\n entry:",entry)
                 if 'documentids' in entry and len(entry['documentids']) > 0 :
-                    # print("\n entry['divisionid']:",entry['divisionid'])
+                    print("\n 5. entry['divisionid']:",entry['divisionid'])
                     divisionid = entry['divisionid']
                     documentids = entry['documentids']
                     formattedsummary = redactionsummary().prepareredactionsummary(message, documentids, pageflags, programareas)
-                    #print("formattedsummary", formattedsummary)
+                    print("\n 6. formattedsummary", formattedsummary)
                     template_path='templates/'+documenttypename+'.docx'
                     redaction_summary= documentgenerationservice().generate_pdf(formattedsummary, documenttypename,template_path)
                     divisioname = None
@@ -58,7 +59,7 @@ def processmessage(self,incomingmessage):
                         s3uricategoryfolder = category
                     s3uri = stitcheddocs3uri.split(s3uricategoryfolder+"/")[0] + s3uricategoryfolder+"/"
                     filename =self.__get_summaryfilename(message.requestnumber, category, divisioname, stitcheddocfilename) 
-                    # print("\n filename:",filename)
+                    print("\n redaction_summary.content length: {0}".format(len(redaction_summary.content)))
                     uploadobj= uploadbytes(filename,redaction_summary.content,s3uri)
                     upload_responses.append(uploadobj)
                     if uploadobj["uploadresponse"].status_code == 200:
@@ -72,6 +73,7 @@ def processmessage(self,incomingmessage):
                     summaryfilestozip.append({"filename": uploadobj["filename"], "s3uripath":uploadobj["documentpath"]})
             return summaryfilestozip
         except (Exception) as error:
+            traceback.print_exc()
             print('error occured in redaction summary service: ', error)
             pdfstitchjobactivity().recordjobstatus(message,4,"redactionsummaryfailed",str(error),"summary generation failed")
             return summaryfilestozip

From c02545ff67777cd1527e6408181e6c0f6a516783 Mon Sep 17 00:00:00 2001
From: Abin Antony <abin.antony@aot-technologies.com>
Date: Wed, 21 Aug 2024 11:46:11 -0700
Subject: [PATCH 03/18] #FOIMOD-3416 error handling for PSA-2024-41570 issue

---
 .../services/dts/redactionsummary.py          | 23 +++++++++++--------
 1 file changed, 13 insertions(+), 10 deletions(-)

diff --git a/computingservices/DocumentServices/services/dts/redactionsummary.py b/computingservices/DocumentServices/services/dts/redactionsummary.py
index 0a05f7098..7eb3cb022 100644
--- a/computingservices/DocumentServices/services/dts/redactionsummary.py
+++ b/computingservices/DocumentServices/services/dts/redactionsummary.py
@@ -42,6 +42,7 @@ def __packaggesummary(self, message, documentids, pageflags, programareas):
             print("\n_pageflags",_pageflags)
             summarydata = []
             docpageflags = documentpageflag().get_documentpageflag(message.ministryrequestid, redactionlayerid, ordereddocids)
+            print("\n docpageflags",docpageflags)
             deletedpages = self.__getdeletedpages(message.ministryrequestid, ordereddocids)
             skippages= []     
             pagecount = 0
@@ -49,16 +50,18 @@ def __packaggesummary(self, message, documentids, pageflags, programareas):
                 for docid in ordereddocids:
                     if docid in documentids:
                         docdeletedpages = deletedpages[docid] if docid in deletedpages else []
-                        docpageflag = docpageflags[docid]
-                        for pageflag in _pageflags:
-                            filteredpages = self.__get_pages_by_flagid(docpageflag["pageflag"], docdeletedpages, pagecount, pageflag["pageflagid"], message.category)
-                            if len(filteredpages) > 0:
-                                originalpagenos = [pg['originalpageno'] for pg in filteredpages]
-                                docpagesections = documentpageflag().getsections_by_documentid_pageno(redactionlayerid, docid, originalpagenos)
-                                docpageconsults = self.__get_consults_by_pageno(programareas, docpageflag["pageflag"], filteredpages)
-                                pageflag['docpageflags'] = pageflag['docpageflags'] + self.__get_pagesection_mapping(filteredpages, docpagesections, docpageconsults)
-                        skippages = self.__get_skippagenos(docpageflag['pageflag'], message.category)
-                    pagecount = (pagecount+stitchedpagedata[docid]["pagecount"])-len(skippages)
+                        if docpageflags is not None and docid in docpageflags.keys():
+                            docpageflag = docpageflags[docid]
+                            for pageflag in _pageflags:
+                                filteredpages = self.__get_pages_by_flagid(docpageflag["pageflag"], docdeletedpages, pagecount, pageflag["pageflagid"], message.category)
+                                if len(filteredpages) > 0:
+                                    originalpagenos = [pg['originalpageno'] for pg in filteredpages]
+                                    docpagesections = documentpageflag().getsections_by_documentid_pageno(redactionlayerid, docid, originalpagenos)
+                                    docpageconsults = self.__get_consults_by_pageno(programareas, docpageflag["pageflag"], filteredpages)
+                                    pageflag['docpageflags'] = pageflag['docpageflags'] + self.__get_pagesection_mapping(filteredpages, docpagesections, docpageconsults)
+                            skippages = self.__get_skippagenos(docpageflag['pageflag'], message.category)
+                    if stitchedpagedata is not None:        
+                        pagecount = (pagecount+stitchedpagedata[docid]["pagecount"])-len(skippages)
                 print("\n_pageflags1",_pageflags)
                 for pageflag in _pageflags:
                     _data = {}

From e6b4f3bccc7ad5158e90ee5012eb86c84da3d246 Mon Sep 17 00:00:00 2001
From: Richard Qi <richard.qi@aot-technologies.com>
Date: Wed, 21 Aug 2024 15:25:25 -0700
Subject: [PATCH 04/18] Fix observation #1 in FOIMOD-599

---
 web/src/components/FOI/Home/Redlining.js | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/web/src/components/FOI/Home/Redlining.js b/web/src/components/FOI/Home/Redlining.js
index 2fd39590e..72fdd0e66 100644
--- a/web/src/components/FOI/Home/Redlining.js
+++ b/web/src/components/FOI/Home/Redlining.js
@@ -1428,6 +1428,9 @@ const Redlining = React.forwardRef(
       let username = docViewer?.getAnnotationManager()?.getCurrentUser();
       for (const entry in annotData) {
         let xml = parser.parseFromString(annotData[entry]);
+        // import redactions first, free text later, so translucent redaction won't cover free text
+        let xmlAnnotsChildren_redaction = [];
+        let xmlAnnotsChildren_others = [];
         for (let annot of xml.getElementsByTagName("annots")[0].children) {
           let txt = domParser.parseFromString(
             annot.getElementsByTagName("trn-custom-data")[0].attributes.bytes,
@@ -1441,6 +1444,12 @@ const Redlining = React.forwardRef(
               (p) => p.pageNo - 1 === Number(originalPageNo)
             )?.stitchedPageNo - 1
           )?.toString();
+          if(annot.attributes.subject === "Redact") {
+            xmlAnnotsChildren_redaction.push(annot);
+          } else {
+            xmlAnnotsChildren_others.push(annot);
+          }
+          xml.getElementsByTagName("annots")[0].children = [...xmlAnnotsChildren_redaction, ...xmlAnnotsChildren_others];
         }
         xml = parser.toString(xml);
         const _annotations = await annotManager.importAnnotations(xml);

From 858b38f3a4d5491b3992386b555bac8361ab89e2 Mon Sep 17 00:00:00 2001
From: Aman-Hundal <aman.hundal@gov.bc.ca>
Date: Fri, 23 Aug 2024 13:12:58 -0700
Subject: [PATCH 05/18] Added AC 3 to ticket

---
 web/src/components/FOI/App.scss               |  1 +
 .../FOI/Home/FOIPPASectionsModal.js           | 15 ++++++-
 web/src/components/FOI/Home/Redlining.js      | 41 ++++++++++++++++++-
 3 files changed, 54 insertions(+), 3 deletions(-)

diff --git a/web/src/components/FOI/App.scss b/web/src/components/FOI/App.scss
index f518f21da..92b7f3fab 100644
--- a/web/src/components/FOI/App.scss
+++ b/web/src/components/FOI/App.scss
@@ -82,6 +82,7 @@
 
 li.modal-message-list-item {
   margin: 6px 0;
+  font-size: 15px;
 }
 
 #state-change-dialog-title .MuiIconButton-root {
diff --git a/web/src/components/FOI/Home/FOIPPASectionsModal.js b/web/src/components/FOI/Home/FOIPPASectionsModal.js
index e78d83df7..48dab0601 100644
--- a/web/src/components/FOI/Home/FOIPPASectionsModal.js
+++ b/web/src/components/FOI/Home/FOIPPASectionsModal.js
@@ -32,7 +32,9 @@ export const FOIPPASectionsModal= ({
     saveRedaction,
     defaultSections,
     saveDefaultSections,
-    clearDefaultSections
+    clearDefaultSections,
+    pageSelectionsContainNRDup,
+    setMessageModalOpen
 }) => {
 
     const [modalSortNumbered, setModalSortNumbered] = useState(false);
@@ -101,6 +103,15 @@ export const FOIPPASectionsModal= ({
           return b.count - a.count;
         }
       };
+
+      const handleSelectCodes = () => {
+        if (editRedacts) {
+          saveRedactions();
+        } else {
+          saveRedaction();
+        }
+        pageSelectionsContainNRDup ? setMessageModalOpen(true) : setMessageModalOpen(false);
+      }
       
 
     return(
@@ -187,7 +198,7 @@ export const FOIPPASectionsModal= ({
           <DialogActions className="foippa-modal-actions">
             <button
               className={`btn-bottom btn-save btn`}
-              onClick={editRedacts ? saveRedactions : saveRedaction}
+              onClick={handleSelectCodes}
               disabled={saveDisabled}
             >
               Select Code(s)
diff --git a/web/src/components/FOI/Home/Redlining.js b/web/src/components/FOI/Home/Redlining.js
index 72fdd0e66..ce1e3665d 100644
--- a/web/src/components/FOI/Home/Redlining.js
+++ b/web/src/components/FOI/Home/Redlining.js
@@ -138,6 +138,7 @@ const Redlining = React.forwardRef(
     const [pagesRemoved, setPagesRemoved] = useState([]);
     const [redlineModalOpen, setRedlineModalOpen] = useState(false);
     const [isDisableNRDuplicate, setIsDisableNRDuplicate] = useState(false);
+    const [pageSelectionsContainNRDup, setPageSelectionsContainNRDup] = useState(false);
     //xml parser
     const parser = new XMLParser();
     /**Response Package && Redline download and saving logic (react custom hooks)*/
@@ -2142,13 +2143,46 @@ const Redlining = React.forwardRef(
           </div>,
         ],
       });
-      setMessageModalOpen(true)
+      setMessageModalOpen(true);
+    }
+
+    const setMessageModalForNrDuplicatePriority = () => {
+      updateModalData({
+        modalTitle: "Selected page(s) currently have NR or Duplicate flag applied",
+        modalMessage: [
+            <ul>
+              <li className="modal-message-list-item">Please note, your redaction(s) have been applied on your selected page(s). However, to flag your selected page(s) as Withheld In Full, you must first change your selected page(s) flags to In Progress.</li>
+              <li className="modal-message-list-item">After your selected page(s) are flagged as In Progress you may proceed to mark them as Withheld in Full.</li>
+            </ul>
+        ],
+      });
     }
 
     useEffect(() => {
       if (!newRedaction) return;
       const astrType = decodeAstr(newRedaction.astr)['trn-redaction-type'] || '';
       const hasFullPageRedaction = astrType === "fullPage";
+      // logic to alert the user that a withheld in full pageflag/redaction was applied to a page with a duplicate or nr pageflag.
+      const pageFlagsMap = new Map();
+      for (let docPageFlags of pageFlags) {
+        pageFlagsMap.set(docPageFlags.documentid, docPageFlags.pageflag);
+      }
+      let NROrDuplicateFlag = false;
+      for (let pageObj of pageSelections) {
+        if (NROrDuplicateFlag) {
+          break;
+        }
+        const pageFlagList = pageFlagsMap.get(pageObj.docid);
+        if (pageFlagList) {
+          for (let flagObj of pageFlagList) {
+            if (flagObj.page === pageObj.page && (flagObj.flagid === pageFlagTypes["Not Responsive"] || flagObj.flagid === pageFlagTypes["Duplicate"])) {
+              NROrDuplicateFlag = true;
+              break;
+            }
+          }
+        }
+      }
+      setPageSelectionsContainNRDup(NROrDuplicateFlag);
 
       if (newRedaction.names?.length > REDACTION_SELECT_LIMIT) {
         setWarningModalOpen(true);
@@ -2157,6 +2191,9 @@ const Redlining = React.forwardRef(
         saveRedaction();
       } else if (defaultSections.length == 0 && !hasFullPageRedaction) {
         setModalOpen(true);
+      } else if (selectedPageFlagId === pageFlagTypes["Withheld in Full"] && NROrDuplicateFlag) {
+        setModalOpen(true);
+        setMessageModalForNrDuplicatePriority();
       } else if (selectedPageFlagId === pageFlagTypes["Withheld in Full"] && defaultSections.length > 0) {
         setMessageModalForNotResponsive();
       } else if (hasFullPageRedaction) {
@@ -2300,6 +2337,8 @@ const Redlining = React.forwardRef(
             handleSectionSelected={handleSectionSelected}
             editRedacts={editRedacts}
             saveRedactions={saveRedactions}
+            pageSelectionsContainNRDup={pageSelectionsContainNRDup}
+            setMessageModalOpen={setMessageModalOpen}
             saveDisabled={saveDisabled}
             saveRedaction={saveRedaction}
             defaultSections={defaultSections}

From 9ab348e267bb9e7a3fcd41728f3545cc04b80a7a Mon Sep 17 00:00:00 2001
From: Aman-Hundal <aman.hundal@gov.bc.ca>
Date: Fri, 23 Aug 2024 14:39:03 -0700
Subject: [PATCH 06/18] adjusted new logic to warn user of applying witheld in
 full page flag to nr or dupe when full page redactions are applied through
 specificy pages feature

---
 web/src/components/FOI/Home/Redlining.js | 36 +++++++++++++-----------
 1 file changed, 19 insertions(+), 17 deletions(-)

diff --git a/web/src/components/FOI/Home/Redlining.js b/web/src/components/FOI/Home/Redlining.js
index ce1e3665d..5216879c6 100644
--- a/web/src/components/FOI/Home/Redlining.js
+++ b/web/src/components/FOI/Home/Redlining.js
@@ -2162,27 +2162,29 @@ const Redlining = React.forwardRef(
       if (!newRedaction) return;
       const astrType = decodeAstr(newRedaction.astr)['trn-redaction-type'] || '';
       const hasFullPageRedaction = astrType === "fullPage";
-      // logic to alert the user that a withheld in full pageflag/redaction was applied to a page with a duplicate or nr pageflag.
-      const pageFlagsMap = new Map();
-      for (let docPageFlags of pageFlags) {
-        pageFlagsMap.set(docPageFlags.documentid, docPageFlags.pageflag);
-      }
-      let NROrDuplicateFlag = false;
-      for (let pageObj of pageSelections) {
-        if (NROrDuplicateFlag) {
-          break;
+      // logic to alert the user that a withheld in full pageflag/redaction was applied to a page with an existing duplicate or nr pageflag.
+      let hasNROrDuplicateFlag = false;
+      if (selectedPageFlagId === pageFlagTypes["Withheld in Full"] || hasFullPageRedaction) {
+        const pageFlagsMap = new Map();
+        for (let docPageFlags of pageFlags) {
+          pageFlagsMap.set(docPageFlags.documentid, docPageFlags.pageflag);
         }
-        const pageFlagList = pageFlagsMap.get(pageObj.docid);
-        if (pageFlagList) {
-          for (let flagObj of pageFlagList) {
-            if (flagObj.page === pageObj.page && (flagObj.flagid === pageFlagTypes["Not Responsive"] || flagObj.flagid === pageFlagTypes["Duplicate"])) {
-              NROrDuplicateFlag = true;
-              break;
+        for (let pageObj of pageSelections) {
+          if (hasNROrDuplicateFlag) {
+            break;
+          }
+          const pageFlagList = pageFlagsMap.get(pageObj.docid);
+          if (pageFlagList) {
+            for (let flagObj of pageFlagList) {
+              if (flagObj.page === pageObj.page && (flagObj.flagid === pageFlagTypes["Not Responsive"] || flagObj.flagid === pageFlagTypes["Duplicate"])) {
+                hasNROrDuplicateFlag = true;
+                break;
+              }
             }
           }
         }
       }
-      setPageSelectionsContainNRDup(NROrDuplicateFlag);
+      setPageSelectionsContainNRDup(hasNROrDuplicateFlag);
 
       if (newRedaction.names?.length > REDACTION_SELECT_LIMIT) {
         setWarningModalOpen(true);
@@ -2191,7 +2193,7 @@ const Redlining = React.forwardRef(
         saveRedaction();
       } else if (defaultSections.length == 0 && !hasFullPageRedaction) {
         setModalOpen(true);
-      } else if (selectedPageFlagId === pageFlagTypes["Withheld in Full"] && NROrDuplicateFlag) {
+      } else if (hasNROrDuplicateFlag) {
         setModalOpen(true);
         setMessageModalForNrDuplicatePriority();
       } else if (selectedPageFlagId === pageFlagTypes["Withheld in Full"] && defaultSections.length > 0) {

From 3587d63da397539585e9f65998501c0df1e39a24 Mon Sep 17 00:00:00 2001
From: Aparna <aparna.s@aot-technologies.com>
Date: Fri, 23 Aug 2024 16:38:26 -0700
Subject: [PATCH 07/18] CFD general requests - response package creation issue
 fix

---
 .../useSaveResponsePackage.js                 | 19 +++++++++++++++----
 web/src/components/FOI/Home/Redlining.js      |  4 +++-
 2 files changed, 18 insertions(+), 5 deletions(-)

diff --git a/web/src/components/FOI/Home/CreateResponsePDF/useSaveResponsePackage.js b/web/src/components/FOI/Home/CreateResponsePDF/useSaveResponsePackage.js
index 3a2a6a4c4..8bea1b63d 100644
--- a/web/src/components/FOI/Home/CreateResponsePDF/useSaveResponsePackage.js
+++ b/web/src/components/FOI/Home/CreateResponsePDF/useSaveResponsePackage.js
@@ -128,12 +128,12 @@ const useSaveResponsePackage = () => {
   //   return { sorteddocuments: sorteddocids, pkgdocuments: summarylist };
   // };
 
-  const prepareresponseredlinesummarylist = (documentlist, bcgovcode) => {
+  const prepareresponseredlinesummarylist = (documentlist, bcgovcode, requestType) => {
     let summarylist = [];
     let alldocuments = [];
     console.log("\ndocumentlist:", documentlist);
     let sorteddocids = [];
-    if (bcgovcode?.toLowerCase() === 'mcf') {
+    if (bcgovcode?.toLowerCase() === 'mcf' && requestType == "personal") {
       let labelGroups = {};
       let alldocids = [];
   
@@ -212,7 +212,8 @@ const useSaveResponsePackage = () => {
     _instance,
     documentList,
     pageMappedDocs,
-    pageFlags
+    pageFlags,
+    requestType
   ) => {
     const downloadType = "pdf";
     let zipServiceMessage = {
@@ -231,7 +232,7 @@ const useSaveResponsePackage = () => {
         const toastID = toast.loading("Start generating final package...");
         zipServiceMessage.requestnumber = res.requestnumber;
         zipServiceMessage.bcgovcode = res.bcgovcode;
-        zipServiceMessage.summarydocuments= prepareresponseredlinesummarylist(documentList,zipServiceMessage.bcgovcode)
+        zipServiceMessage.summarydocuments= prepareresponseredlinesummarylist(documentList,zipServiceMessage.bcgovcode, requestType)
         let annotList = annotationManager.getAnnotationsList();
         annotationManager.ungroupAnnotations(annotList);
         /** remove duplicate and not responsive pages */
@@ -291,6 +292,16 @@ const useSaveResponsePackage = () => {
           annotationList: filteredAnnotations,
           widgets: true,
         });
+
+        // Get the root bookmark
+        let rootBookmark = await doc.GetFirstBookmark();
+      
+        // Check if the root bookmark is valid
+        if (await rootBookmark.isValid()) {
+          await rootBookmark.delete(); // This deletes the entire bookmark tree
+        }
+
+
         /** apply redaction and save to s3 - xfdfString is needed to display
          * the freetext(section name) on downloaded file.*/
         doc
diff --git a/web/src/components/FOI/Home/Redlining.js b/web/src/components/FOI/Home/Redlining.js
index 72fdd0e66..79e4cd5e1 100644
--- a/web/src/components/FOI/Home/Redlining.js
+++ b/web/src/components/FOI/Home/Redlining.js
@@ -99,6 +99,7 @@ const Redlining = React.forwardRef(
     const currentLayer = useSelector((state) => state.documents?.currentLayer);
     const deletedDocPages = useAppSelector((state) => state.documents?.deletedDocPages);
     const validoipcreviewlayer = useAppSelector((state) => state.documents?.requestinfo?.validoipcreviewlayer);
+    const requestType = useAppSelector((state) => state.documents?.requestinfo?.requesttype);
     const viewer = useRef(null);
     const [documentList, setDocumentList] = useState([]);
     
@@ -2230,7 +2231,8 @@ const Redlining = React.forwardRef(
             docInstance,
             documentList,
             pageMappedDocs,
-            pageFlags
+            pageFlags,
+            requestType
           );
           break;
         default:

From 12b83e2f949dbebdf491cc89c53ef0e9ab1a2e43 Mon Sep 17 00:00:00 2001
From: nkan-aot2 <156717133+nkan-aot2@users.noreply.github.com>
Date: Fri, 23 Aug 2024 16:44:06 -0700
Subject: [PATCH 08/18] Update useSaveResponsePackage.js

---
 .../CreateResponsePDF/useSaveResponsePackage.js    | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/web/src/components/FOI/Home/CreateResponsePDF/useSaveResponsePackage.js b/web/src/components/FOI/Home/CreateResponsePDF/useSaveResponsePackage.js
index 3a2a6a4c4..34cb6199c 100644
--- a/web/src/components/FOI/Home/CreateResponsePDF/useSaveResponsePackage.js
+++ b/web/src/components/FOI/Home/CreateResponsePDF/useSaveResponsePackage.js
@@ -33,7 +33,10 @@ const useSaveResponsePackage = () => {
 
         let _docmain = _docViwer.getDocument();
         doc = await _docmain.getPDFDoc();
-
+        
+        let docinfo = await doc.getDocInfo();
+        docinfo.setTitle(requestnumber + " - Response Package");
+        
         // Run PDFNet methods with memory management
         await PDFNet.runWithCleanup(async () => {
           // lock the document before a write operation
@@ -261,6 +264,15 @@ const useSaveResponsePackage = () => {
         }   
         const { PDFNet } = _instance.Core;
         PDFNet.initialize();
+        
+        // remove bookmarks
+        var pdfdoc = await doc.getPDFDoc()
+        var bookmark = await pdfdoc.getFirstBookmark();
+        while (bookmark && await bookmark.isValid()) {
+          bookmark.delete();
+          bookmark = await pdfdoc.getFirstBookmark();
+        }
+        
         await stampPageNumberResponse(documentViewer, PDFNet);
         toast.update(toastID, {
           render: "Saving section stamps...",

From 877ac9c6a52875018bf4a824e894b54e2fd13f40 Mon Sep 17 00:00:00 2001
From: Aparna <aparna.s@aot-technologies.com>
Date: Fri, 23 Aug 2024 17:07:49 -0700
Subject: [PATCH 09/18] Bookmark sample code removed

---
 .../FOI/Home/CreateResponsePDF/useSaveResponsePackage.js | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/web/src/components/FOI/Home/CreateResponsePDF/useSaveResponsePackage.js b/web/src/components/FOI/Home/CreateResponsePDF/useSaveResponsePackage.js
index 8bea1b63d..3ed55cfd2 100644
--- a/web/src/components/FOI/Home/CreateResponsePDF/useSaveResponsePackage.js
+++ b/web/src/components/FOI/Home/CreateResponsePDF/useSaveResponsePackage.js
@@ -293,15 +293,6 @@ const useSaveResponsePackage = () => {
           widgets: true,
         });
 
-        // Get the root bookmark
-        let rootBookmark = await doc.GetFirstBookmark();
-      
-        // Check if the root bookmark is valid
-        if (await rootBookmark.isValid()) {
-          await rootBookmark.delete(); // This deletes the entire bookmark tree
-        }
-
-
         /** apply redaction and save to s3 - xfdfString is needed to display
          * the freetext(section name) on downloaded file.*/
         doc

From 307ce2086a4b8a3f4839d4fcfe9bf2a20b848452 Mon Sep 17 00:00:00 2001
From: nkan-aot2 <156717133+nkan-aot2@users.noreply.github.com>
Date: Fri, 23 Aug 2024 17:12:55 -0700
Subject: [PATCH 10/18] Update useSaveResponsePackage.js

---
 .../FOI/Home/CreateResponsePDF/useSaveResponsePackage.js         | 1 +
 1 file changed, 1 insertion(+)

diff --git a/web/src/components/FOI/Home/CreateResponsePDF/useSaveResponsePackage.js b/web/src/components/FOI/Home/CreateResponsePDF/useSaveResponsePackage.js
index 5904822c5..c71a9a10e 100644
--- a/web/src/components/FOI/Home/CreateResponsePDF/useSaveResponsePackage.js
+++ b/web/src/components/FOI/Home/CreateResponsePDF/useSaveResponsePackage.js
@@ -36,6 +36,7 @@ const useSaveResponsePackage = () => {
         
         let docinfo = await doc.getDocInfo();
         docinfo.setTitle(requestnumber + " - Response Package");
+        docinfo.setAuthor("");
         
         // Run PDFNet methods with memory management
         await PDFNet.runWithCleanup(async () => {

From ac05725101a4855d857c94c63f3f527b1d6e45bd Mon Sep 17 00:00:00 2001
From: nkan-aot2 <156717133+nkan-aot2@users.noreply.github.com>
Date: Fri, 23 Aug 2024 17:16:02 -0700
Subject: [PATCH 11/18] update pdf title

---
 .../FOI/Home/CreateResponsePDF/useSaveResponsePackage.js        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/web/src/components/FOI/Home/CreateResponsePDF/useSaveResponsePackage.js b/web/src/components/FOI/Home/CreateResponsePDF/useSaveResponsePackage.js
index c71a9a10e..e193f8960 100644
--- a/web/src/components/FOI/Home/CreateResponsePDF/useSaveResponsePackage.js
+++ b/web/src/components/FOI/Home/CreateResponsePDF/useSaveResponsePackage.js
@@ -35,7 +35,7 @@ const useSaveResponsePackage = () => {
         doc = await _docmain.getPDFDoc();
         
         let docinfo = await doc.getDocInfo();
-        docinfo.setTitle(requestnumber + " - Response Package");
+        docinfo.setTitle(requestnumber + ".pdf");
         docinfo.setAuthor("");
         
         // Run PDFNet methods with memory management

From b0385d3e2216175ea5ab046b712b50673211b029 Mon Sep 17 00:00:00 2001
From: unknown <nicholas.kan@aot-technologies.com>
Date: Thu, 29 Aug 2024 11:46:11 -0700
Subject: [PATCH 12/18] clear metadata during dedupe and save copy of original
 file in s3

---
 .../services/s3documentservice.py             | 25 ++++++++++++++++++-
 .../DedupeServices/utils/foidedupeconfig.py   |  1 -
 2 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/computingservices/DedupeServices/services/s3documentservice.py b/computingservices/DedupeServices/services/s3documentservice.py
index ff99843c9..95a07cca0 100644
--- a/computingservices/DedupeServices/services/s3documentservice.py
+++ b/computingservices/DedupeServices/services/s3documentservice.py
@@ -12,6 +12,8 @@
 from html import escape
 import hashlib
 import uuid
+import boto3
+from botocore.config import Config
 from re import sub
 import fitz
 from utils import (
@@ -174,8 +176,29 @@ def gets3documenthashcode(producermessage):
                     }
                 )
                 saveresponse.raise_for_status()
-        fitz_reader.close()
         
+        # clear metadata
+        writer = fitz.open()
+        writer.insert_pdf(fitz_reader)
+        client = boto3.client('s3',config=Config(signature_version='s3v4'),
+            endpoint_url='https://{0}/'.format(dedupe_s3_host),
+            aws_access_key_id= s3_access_key_id,
+            aws_secret_access_key= s3_secret_access_key,
+            region_name= dedupe_s3_region
+        )
+        response = client.copy_object(
+            CopySource="/" + "/".join(filepath.split("/")[3:]), # /Bucket-name/path/filename
+            Bucket=filepath.split("/")[3], # Destination bucket
+            Key= "/".join(filepath.split("/")[3:])[:-4] + 'ORIGINAL' + '.pdf' # Destination path/filename
+        )
+        response = requests.put(
+		 	filepath,
+		 	data=writer.tobytes(),
+		 	auth=auth
+        )
+        
+        fitz_reader.close()
+
     elif extension.lower() in file_conversion_types:
         # "Extension different {0}, so need to download pdf here for pagecount!!".format(extension))
         pdfresponseofconverted = requests.get(
diff --git a/computingservices/DedupeServices/utils/foidedupeconfig.py b/computingservices/DedupeServices/utils/foidedupeconfig.py
index 6e648af28..a51710bc2 100644
--- a/computingservices/DedupeServices/utils/foidedupeconfig.py
+++ b/computingservices/DedupeServices/utils/foidedupeconfig.py
@@ -18,7 +18,6 @@
 dedupe_db_user = os.getenv("DEDUPE_DB_USER")
 dedupe_db_password = os.getenv("DEDUPE_DB_PASSWORD")
 
-dedupe_s3_host = os.getenv("DEDUPE_S3_HOST")
 dedupe_s3_host = os.getenv("DEDUPE_S3_HOST")
 dedupe_s3_region = os.getenv("DEDUPE_S3_REGION")
 dedupe_s3_service = os.getenv("DEDUPE_S3_SERVICE")

From 31d6d07dd3954232894fff894b8f694badbce4ce Mon Sep 17 00:00:00 2001
From: unknown <nicholas.kan@aot-technologies.com>
Date: Fri, 30 Aug 2024 09:32:35 -0700
Subject: [PATCH 13/18] use pypdf instead of fitz fix hash issue

---
 .../DedupeServices/requirements.txt           | Bin 828 -> 858 bytes
 .../services/s3documentservice.py             |  53 +++++++++++-------
 2 files changed, 32 insertions(+), 21 deletions(-)

diff --git a/computingservices/DedupeServices/requirements.txt b/computingservices/DedupeServices/requirements.txt
index 8128a986fc4750d4708c541c96d43ee15247adc2..e51e508cd57d04d4fda9aae874db4291a9582b84 100644
GIT binary patch
delta 39
scmdnPc8hJpEhfPL1{Vf51|tSrAT(ysV=$O}mPwJ-1jw?S{F%uU0L2FhO8@`>

delta 21
dcmcb`wuf!QEvCsom_%5O8T1$|CJQoq0svO{23`OF

diff --git a/computingservices/DedupeServices/services/s3documentservice.py b/computingservices/DedupeServices/services/s3documentservice.py
index 95a07cca0..96110833d 100644
--- a/computingservices/DedupeServices/services/s3documentservice.py
+++ b/computingservices/DedupeServices/services/s3documentservice.py
@@ -16,6 +16,7 @@
 from botocore.config import Config
 from re import sub
 import fitz
+import PyPDF2
 from utils import (
     gets3credentialsobject,
     getdedupeproducermessage,
@@ -175,29 +176,39 @@ def gets3documenthashcode(producermessage):
                         "Content-Type": "application/json",
                     }
                 )
-                saveresponse.raise_for_status()
+                saveresponse.raise_for_status()        
+        fitz_reader.close()
         
         # clear metadata
-        writer = fitz.open()
-        writer.insert_pdf(fitz_reader)
-        client = boto3.client('s3',config=Config(signature_version='s3v4'),
-            endpoint_url='https://{0}/'.format(dedupe_s3_host),
-            aws_access_key_id= s3_access_key_id,
-            aws_secret_access_key= s3_secret_access_key,
-            region_name= dedupe_s3_region
-        )
-        response = client.copy_object(
-            CopySource="/" + "/".join(filepath.split("/")[3:]), # /Bucket-name/path/filename
-            Bucket=filepath.split("/")[3], # Destination bucket
-            Key= "/".join(filepath.split("/")[3:])[:-4] + 'ORIGINAL' + '.pdf' # Destination path/filename
-        )
-        response = requests.put(
-		 	filepath,
-		 	data=writer.tobytes(),
-		 	auth=auth
-        )
-        
-        fitz_reader.close()
+        reader2 = PyPDF2.PdfReader(BytesIO(response.content))
+        # Check if metadata exists.
+        if reader2.metadata is not None:
+            # Create a new PDF file without metadata.
+            writer = PyPDF2.PdfWriter()
+            # Copy pages from the original PDF to the new PDF.
+            for page_num in range(len(reader.pages)):
+                page = reader2.pages[page_num]                
+                writer.add_page(page)        
+            #writer.remove_links() # to remove comments.
+            buffer = BytesIO()
+            writer.write(buffer)
+            client = boto3.client('s3',config=Config(signature_version='s3v4'),
+                endpoint_url='https://{0}/'.format(dedupe_s3_host),
+                aws_access_key_id= s3_access_key_id,
+                aws_secret_access_key= s3_secret_access_key,
+                region_name= dedupe_s3_region
+            )
+            copyresponse = client.copy_object(
+                CopySource="/" + "/".join(filepath.split("/")[3:]), # /Bucket-name/path/filename
+                Bucket=filepath.split("/")[3], # Destination bucket
+                Key= "/".join(filepath.split("/")[3:])[:-4] + 'ORIGINAL' + '.pdf' # Destination path/filename
+            )
+            uploadresponse = requests.put(
+                filepath,
+                data=buffer.getvalue(),
+                auth=auth
+            )
+            uploadresponse.raise_for_status()
 
     elif extension.lower() in file_conversion_types:
         # "Extension different {0}, so need to download pdf here for pagecount!!".format(extension))

From c6d1baf4da8edec5b6c141f9cf7943a55e292e5f Mon Sep 17 00:00:00 2001
From: Aparna <aparna.s@aot-technologies.com>
Date: Fri, 30 Aug 2024 14:21:20 -0700
Subject: [PATCH 14/18] FOIMOD-3431-Blank Response package Summaries getting
 created for CFD General Requests

Fix for Blank Response package Summaries getting created for CFD General Requests
---
 api/reviewer_api/resources/redaction.py                      | 2 +-
 api/reviewer_api/schemas/finalpackage.py                     | 4 +++-
 api/reviewer_api/schemas/redline.py                          | 3 ++-
 api/reviewer_api/services/radactionservice.py                | 3 ++-
 .../rstreamio/message/schemas/redactionsummary.py            | 3 ++-
 .../DocumentServices/services/dts/redactionsummary.py        | 5 +++--
 .../DocumentServices/services/redactionsummaryservice.py     | 3 ++-
 .../DocumentServices/services/zippingservice.py              | 1 +
 .../FOI/Home/CreateResponsePDF/useSaveRedlineForSignOff.js   | 3 ++-
 .../FOI/Home/CreateResponsePDF/useSaveResponsePackage.js     | 5 +++--
 10 files changed, 21 insertions(+), 11 deletions(-)

diff --git a/api/reviewer_api/resources/redaction.py b/api/reviewer_api/resources/redaction.py
index 4b633b1f3..3552fb3d7 100644
--- a/api/reviewer_api/resources/redaction.py
+++ b/api/reviewer_api/resources/redaction.py
@@ -342,7 +342,7 @@ def post():
         try:
             requestjson = request.get_json()
             print("\nrequestjson:",requestjson)
-            if(requestjson['bcgovcode'] == "mcf"):
+            if(requestjson['bcgovcode'] == "mcf" and requestjson['requesttype'] == "personal"):
                 finalpackageschema = MCFFinalPackageSchema().load(requestjson) 
             else:
                 finalpackageschema = FinalPackageSchema().load(requestjson)
diff --git a/api/reviewer_api/schemas/finalpackage.py b/api/reviewer_api/schemas/finalpackage.py
index 94f5f0b4b..0cf5ee155 100644
--- a/api/reviewer_api/schemas/finalpackage.py
+++ b/api/reviewer_api/schemas/finalpackage.py
@@ -30,6 +30,7 @@ class FinalPackageSchema(Schema):
     )
     summarydocuments = fields.Nested(SummarySchema, allow_none=True)
     redactionlayerid = fields.Int(data_key="redactionlayerid", allow_none=False)
+    requesttype = fields.Str(data_key="requesttype", allow_none=False)
 
 class SummaryRecordSchema(Schema):
     recordname = fields.Str(data_key="recordname", allow_none=True)
@@ -53,4 +54,5 @@ class MCFFinalPackageSchema(Schema):
         AttributeSchema, many=True, required=True, allow_none=False
     )
     summarydocuments = fields.Nested(MCFSummarySchema, allow_none=True)
-    redactionlayerid = fields.Int(data_key="redactionlayerid", allow_none=False)
\ No newline at end of file
+    redactionlayerid = fields.Int(data_key="redactionlayerid", allow_none=False)
+    requesttype = fields.Str(data_key="requesttype", allow_none=False)
\ No newline at end of file
diff --git a/api/reviewer_api/schemas/redline.py b/api/reviewer_api/schemas/redline.py
index 27db2aa60..bdd484b3d 100644
--- a/api/reviewer_api/schemas/redline.py
+++ b/api/reviewer_api/schemas/redline.py
@@ -29,4 +29,5 @@ class RedlineSchema(Schema):
         AttributeSchema, many=True, required=True, allow_none=False
     )
     summarydocuments = fields.Nested(SummarySchema, allow_none=True)
-    redactionlayerid = fields.Int(data_key="redactionlayerid", allow_none=False)
\ No newline at end of file
+    redactionlayerid = fields.Int(data_key="redactionlayerid", allow_none=False)
+    requesttype = fields.Str(data_key="requesttype", allow_none=False)
\ No newline at end of file
diff --git a/api/reviewer_api/services/radactionservice.py b/api/reviewer_api/services/radactionservice.py
index f12b53969..cd63eab93 100644
--- a/api/reviewer_api/services/radactionservice.py
+++ b/api/reviewer_api/services/radactionservice.py
@@ -145,7 +145,8 @@ def __preparemessageforsummaryservice(self, messageschema, userinfo, job):
             "finaloutput": to_json(""),
             "attributes": to_json(messageschema["attributes"]),
             "summarydocuments": json.dumps(messageschema["summarydocuments"]),
-            "redactionlayerid": json.dumps(messageschema["redactionlayerid"])
+            "redactionlayerid": json.dumps(messageschema["redactionlayerid"]),
+            "requesttype": messageschema["requesttype"],
         }
         return _message
 
diff --git a/computingservices/DocumentServices/rstreamio/message/schemas/redactionsummary.py b/computingservices/DocumentServices/rstreamio/message/schemas/redactionsummary.py
index 979e4d18f..c981780ec 100644
--- a/computingservices/DocumentServices/rstreamio/message/schemas/redactionsummary.py
+++ b/computingservices/DocumentServices/rstreamio/message/schemas/redactionsummary.py
@@ -30,7 +30,7 @@ def __init__(self, sorteddocuments, pkgdocuments) -> None:
 
 class RedactionSummaryMessage(object):
     def __init__(self, jobid, requestid, ministryrequestid, category, requestnumber, 
-                 bcgovcode, createdby, filestozip, finaloutput, attributes, summarydocuments ,redactionlayerid) -> None:
+                 bcgovcode, createdby, filestozip, finaloutput, attributes, summarydocuments ,redactionlayerid,requesttype) -> None:
         self.jobid = jobid
         self.requestid = requestid
         self.ministryrequestid = ministryrequestid
@@ -43,6 +43,7 @@ def __init__(self, jobid, requestid, ministryrequestid, category, requestnumber,
         self.attributes = attributes
         self.summarydocuments = summarydocuments
         self.redactionlayerid = redactionlayerid
+        self.requesttype = requesttype
 
 
 def get_in_redactionsummary_msg(producer_json): 
diff --git a/computingservices/DocumentServices/services/dts/redactionsummary.py b/computingservices/DocumentServices/services/dts/redactionsummary.py
index 7eb3cb022..89fded2ef 100644
--- a/computingservices/DocumentServices/services/dts/redactionsummary.py
+++ b/computingservices/DocumentServices/services/dts/redactionsummary.py
@@ -7,11 +7,12 @@
 class redactionsummary():
 
     def prepareredactionsummary(self, message, documentids, pageflags, programareas):
-        if message.bcgovcode == 'mcf' and message.category == "responsepackage":
+        _ismcfpersonalrequest = True if message.bcgovcode == 'mcf' and message.requesttype == 'personal' else False
+        if _ismcfpersonalrequest and message.category == "responsepackage":
             redactionsummary = self.__packagesummaryforcfdrequests(message, documentids)
         else:
             redactionsummary = self.__packaggesummary(message, documentids, pageflags, programareas)
-        if message.category == "responsepackage" and message.bcgovcode != 'mcf':
+        if message.category == "responsepackage" and _ismcfpersonalrequest == False:
             consolidated_redactions = []
             for entry in redactionsummary['data']:
                 consolidated_redactions += entry['sections']
diff --git a/computingservices/DocumentServices/services/redactionsummaryservice.py b/computingservices/DocumentServices/services/redactionsummaryservice.py
index 7193978bb..7593ab9da 100644
--- a/computingservices/DocumentServices/services/redactionsummaryservice.py
+++ b/computingservices/DocumentServices/services/redactionsummaryservice.py
@@ -20,7 +20,8 @@ def processmessage(self,incomingmessage):
             #Condition for handling oipcredline category
             bcgovcode= message.bcgovcode
             category = message.category 
-            if bcgovcode == 'mcf' and category == 'responsepackage':
+            requesttype = message.requesttype
+            if bcgovcode == 'mcf' and requesttype == 'personal' and category == 'responsepackage':
                 documenttypename= 'CFD_responsepackage_redaction_summary'
             else:
                 documenttypename= category+"_redaction_summary" if category == 'responsepackage' else "redline_redaction_summary"
diff --git a/computingservices/DocumentServices/services/zippingservice.py b/computingservices/DocumentServices/services/zippingservice.py
index c997af363..49def91ae 100644
--- a/computingservices/DocumentServices/services/zippingservice.py
+++ b/computingservices/DocumentServices/services/zippingservice.py
@@ -11,6 +11,7 @@ def sendtozipper(self, summaryfiles, message):
     def preparemessageforzipperservice(self,summaryfiles, message):
         try:
             msgjson= json.loads(message)
+            msgjson.pop('requesttype', None)
             if summaryfiles and len(summaryfiles) > 0:                
                 filestozip_list = json.loads(msgjson['filestozip'])+summaryfiles
             else:
diff --git a/web/src/components/FOI/Home/CreateResponsePDF/useSaveRedlineForSignOff.js b/web/src/components/FOI/Home/CreateResponsePDF/useSaveRedlineForSignOff.js
index 8b81c5c90..11c575b60 100644
--- a/web/src/components/FOI/Home/CreateResponsePDF/useSaveRedlineForSignOff.js
+++ b/web/src/components/FOI/Home/CreateResponsePDF/useSaveRedlineForSignOff.js
@@ -889,7 +889,8 @@ const useSaveRedlineForSignoff = (initDocInstance, initDocViewer) => {
           requestnumber: res.requestnumber,
           bcgovcode: res.bcgovcode,
           summarydocuments: prepareredlinesummarylist(stitchDocuments),
-          redactionlayerid: currentLayer.redactionlayerid
+          redactionlayerid: currentLayer.redactionlayerid,
+          requesttype: requestType
         });
         if (res.issingleredlinepackage === "Y") {
           stitchSingleDivisionRedlineExport(
diff --git a/web/src/components/FOI/Home/CreateResponsePDF/useSaveResponsePackage.js b/web/src/components/FOI/Home/CreateResponsePDF/useSaveResponsePackage.js
index e193f8960..5d12ea826 100644
--- a/web/src/components/FOI/Home/CreateResponsePDF/useSaveResponsePackage.js
+++ b/web/src/components/FOI/Home/CreateResponsePDF/useSaveResponsePackage.js
@@ -228,6 +228,7 @@ const useSaveResponsePackage = () => {
       bcgovcode: "",
       summarydocuments: {} ,
       redactionlayerid: currentLayer.redactionlayerid,
+      requesttype: requestType
     };
     getResponsePackagePreSignedUrl(
       foiministryrequestid,
@@ -375,8 +376,8 @@ const useSaveResponsePackage = () => {
   };
   const checkSavingFinalPackage = (redlineReadyAndValid, instance) => {
     const validFinalPackageStatus = requestStatus === RequestStates["Response"];
-    setEnableSavingFinal(true)
-    //setEnableSavingFinal(redlineReadyAndValid && validFinalPackageStatus);
+    //setEnableSavingFinal(true)
+    setEnableSavingFinal(redlineReadyAndValid && validFinalPackageStatus);
     if (instance) {
       const document = instance.UI.iframeWindow.document;
       document.getElementById("final_package").disabled =

From c7dc8295222937f6d9904ddcf4e819e291b5d01d Mon Sep 17 00:00:00 2001
From: unknown <nicholas.kan@aot-technologies.com>
Date: Fri, 30 Aug 2024 14:56:10 -0700
Subject: [PATCH 15/18] fix path issue with stored original file

---
 computingservices/DedupeServices/services/s3documentservice.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/computingservices/DedupeServices/services/s3documentservice.py b/computingservices/DedupeServices/services/s3documentservice.py
index 96110833d..2129117a4 100644
--- a/computingservices/DedupeServices/services/s3documentservice.py
+++ b/computingservices/DedupeServices/services/s3documentservice.py
@@ -201,7 +201,7 @@ def gets3documenthashcode(producermessage):
             copyresponse = client.copy_object(
                 CopySource="/" + "/".join(filepath.split("/")[3:]), # /Bucket-name/path/filename
                 Bucket=filepath.split("/")[3], # Destination bucket
-                Key= "/".join(filepath.split("/")[3:])[:-4] + 'ORIGINAL' + '.pdf' # Destination path/filename
+                Key= "/".join(filepath.split("/")[4:])[:-4] + 'ORIGINAL' + '.pdf' # Destination path/filename
             )
             uploadresponse = requests.put(
                 filepath,

From b1b9e500a771098d3ea07f626fdb9ff8755d425d Mon Sep 17 00:00:00 2001
From: Abin Antony <abin.antony@aot-technologies.com>
Date: Wed, 4 Sep 2024 16:55:32 -0700
Subject: [PATCH 16/18] #FOIMOD-3439 Zipperservice update removing sensitive
 content

---
 .gitignore                                    |   2 ++
 .../ZippingServices/requirements.txt          | Bin 760 -> 790 bytes
 .../ZippingServices/services/zipperservice.py |  27 +++++++++++++++++-
 3 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index 3e5f06c34..f9395bcd2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -100,3 +100,5 @@ bld/
 MCS.FOI.S3FileConversion/MCS.FOI.S3FileConversion/QtBinariesWindows/
 computingservices/ZippingServices/env/*
 openshift/templates/zippingservice/zipper.env
+*.locenv
+
diff --git a/computingservices/ZippingServices/requirements.txt b/computingservices/ZippingServices/requirements.txt
index 6e93668753e3922077ce2411f35c45e150eb823a..a08f010df70dbc95af321746348195b932952cee 100644
GIT binary patch
delta 38
pcmeytI*o0^4<<QY1}=sGhDsoIVQ^zGVz325V+K7210XhJ006B22DktK

delta 7
OcmbQn_Jei94<-N&GXq2b

diff --git a/computingservices/ZippingServices/services/zipperservice.py b/computingservices/ZippingServices/services/zipperservice.py
index dddcc827d..8e98ff311 100644
--- a/computingservices/ZippingServices/services/zipperservice.py
+++ b/computingservices/ZippingServices/services/zipperservice.py
@@ -14,6 +14,7 @@
 from .notificationservice import notificationservice
 import json
 import traceback
+import PyPDF2
 
 
 def processmessage(message):
@@ -110,8 +111,16 @@ def __zipfilesandupload(_message, s3credentials):
                 for fileobj in _jsonfiles:
                     filename = fileobj["filename"]
                     print("\nfilename:",filename)
+
+                    _docbytes = __getdocumentbytearray(fileobj, s3credentials)
+                    _formattedbytes = None
+                    if(filename == "{0}.pdf".format(_message.requestnumber)):
+                        try:                           
+                            _formattedbytes = __removesensitivecontent(_docbytes)                           
+                        except Exception:
+                            print(traceback.format_exc())
                     zip.writestr(
-                        filename, __getdocumentbytearray(fileobj, s3credentials)
+                        filename, _docbytes if _formattedbytes is None else _formattedbytes
                     )
                     
             tp.seek(0)           
@@ -136,6 +145,22 @@ def __zipfilesandupload(_message, s3credentials):
     finally:
         zipped_bytes = None
 
+def __removesensitivecontent(documentbytes):
+    # clear metadata
+    reader2 = PyPDF2.PdfReader(BytesIO(documentbytes))
+    # Check if metadata exists.
+    if reader2.metadata is not None:
+        # Create a new PDF file without metadata.
+        writer = PyPDF2.PdfWriter()
+        # Copy pages from the original PDF to the new PDF.
+        for page_num in range(len(reader2.pages)):
+            page = reader2.pages[page_num]                
+            writer.add_page(page)        
+        #writer.remove_links() # to remove comments.
+        buffer = BytesIO()
+        writer.write(buffer)
+        return buffer.getvalue()
+
 
 def __getzipfilepath(foldername, filename):
     return (

From aa3ad6ad6ce8ec28c1badaf2e3d13de16355180f Mon Sep 17 00:00:00 2001
From: Abin Antony <abin.antony@aot-technologies.com>
Date: Thu, 5 Sep 2024 13:22:31 -0700
Subject: [PATCH 17/18] #FOIMOD-3429 - Removing sensitive info from REDLINE
 packages as well.

---
 .../ZippingServices/services/zipperservice.py          | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/computingservices/ZippingServices/services/zipperservice.py b/computingservices/ZippingServices/services/zipperservice.py
index 8e98ff311..5c343244e 100644
--- a/computingservices/ZippingServices/services/zipperservice.py
+++ b/computingservices/ZippingServices/services/zipperservice.py
@@ -114,11 +114,11 @@ def __zipfilesandupload(_message, s3credentials):
 
                     _docbytes = __getdocumentbytearray(fileobj, s3credentials)
                     _formattedbytes = None
-                    if(filename == "{0}.pdf".format(_message.requestnumber)):
-                        try:                           
-                            _formattedbytes = __removesensitivecontent(_docbytes)                           
-                        except Exception:
-                            print(traceback.format_exc())
+                    
+                    try:                           
+                        _formattedbytes = __removesensitivecontent(_docbytes)                           
+                    except Exception:
+                        print(traceback.format_exc())
                     zip.writestr(
                         filename, _docbytes if _formattedbytes is None else _formattedbytes
                     )

From a1d22b37aa9a045a22ca83bcfba9bf7ac11fc799 Mon Sep 17 00:00:00 2001
From: nkan-aot2 <156717133+nkan-aot2@users.noreply.github.com>
Date: Fri, 6 Sep 2024 13:55:46 -0700
Subject: [PATCH 18/18] Update enums.py

---
 api/reviewer_api/utils/enums.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/api/reviewer_api/utils/enums.py b/api/reviewer_api/utils/enums.py
index fec94ac70..23e99fb1a 100644
--- a/api/reviewer_api/utils/enums.py
+++ b/api/reviewer_api/utils/enums.py
@@ -35,6 +35,7 @@ class MinistryTeamWithKeycloackGroup(Enum):
     AGR = "AGR Ministry Team"
     AG = "AG Ministry Team"
     BRD = "BRD Ministry Team"
+    CAF = "CAF Ministry Team"
     CAS = "CAS Ministry Team"
     CITZ = "CITZ Ministry Team"
     CLB = "CLB Ministry Team"