From 32c8e101bf39ff86bdeaa8eea741c56788a9d627 Mon Sep 17 00:00:00 2001 From: divyav-aot Date: Wed, 14 Feb 2024 10:15:39 -0500 Subject: [PATCH 01/19] redline stitching - SetDirty() error with stampPageNumberRedline fixed --- web/src/components/FOI/Home/Redlining.js | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/web/src/components/FOI/Home/Redlining.js b/web/src/components/FOI/Home/Redlining.js index c850768bf..139714042 100644 --- a/web/src/components/FOI/Home/Redlining.js +++ b/web/src/components/FOI/Home/Redlining.js @@ -2113,15 +2113,19 @@ const Redlining = React.forwardRef( divisionsdocpages, redlineSinglePackage ) => { + let doc = await _docViwer.getPDFDoc(); + let docArr = []; for ( let pagecount = 1; pagecount <= divisionsdocpages.length; pagecount++ ) { - const doc = await _docViwer.getPDFDoc(); - // Run PDFNet methods with memory management await PDFNet.runWithCleanup(async () => { + if (docArr.length > 0) { + //updated document after each page stamping + doc = await PDFNet.PDFDoc.createFromBuffer(docArr); + } // lock the document before a write operation // runWithCleanup will auto unlock when complete doc.lock(); @@ -2152,8 +2156,15 @@ const Redlining = React.forwardRef( `${requestnumber} , Page ${pagenumber} of ${totalpagenumber}`, pgSet ); - }); + + // PDFTron might automatically handle marking the document as "dirty" during modifications. + // Make sure that you are saving the document correctly after any modifications. + const docBuf = await doc.saveMemoryBuffer(PDFNet.SDFDoc.SaveOptions.e_linearized); + docArr = new Uint8Array(docBuf.buffer); + }); } + const pageStampedDocument = await docInstance.Core.createDocument(docArr,{ extension: 'pdf' }); + return pageStampedDocument; }; const stampPageNumberResponse = async (_docViwer, PDFNet) => { @@ -3120,7 +3131,7 @@ const Redlining = React.forwardRef( redlineStitchInfo[divisionid]["documentids"] ); if(redlineCategory !== "oipcreview") { - await stampPageNumberRedline( + stitchObject = await stampPageNumberRedline( stitchObject, PDFNet, redlineStitchInfo[divisionid]["stitchpages"], @@ -3175,12 +3186,12 @@ const Redlining = React.forwardRef( const doc = await stitchObject.getPDFDoc(); await PDFNet.Redactor.redact(doc, rarr, app); } - await stampPageNumberRedline( + stitchObject = await stampPageNumberRedline( stitchObject, PDFNet, redlineStitchInfo[divisionid]["stitchpages"], redlineSinglePackage - ); + ); } //OIPC - Special Block : End From 48ef256a6afaee3f1f829f3e802c1ce2deb28546 Mon Sep 17 00:00:00 2001 From: divyav-aot Date: Wed, 14 Feb 2024 12:08:18 -0500 Subject: [PATCH 02/19] unlock before saving it --- web/src/components/FOI/Home/Redlining.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/src/components/FOI/Home/Redlining.js b/web/src/components/FOI/Home/Redlining.js index 139714042..4b8cf6a1e 100644 --- a/web/src/components/FOI/Home/Redlining.js +++ b/web/src/components/FOI/Home/Redlining.js @@ -2156,7 +2156,7 @@ const Redlining = React.forwardRef( `${requestnumber} , Page ${pagenumber} of ${totalpagenumber}`, pgSet ); - + doc.unlock(); // PDFTron might automatically handle marking the document as "dirty" during modifications. // Make sure that you are saving the document correctly after any modifications. const docBuf = await doc.saveMemoryBuffer(PDFNet.SDFDoc.SaveOptions.e_linearized); From cb49c2259d658f67020d3de099ef4c142c539518 Mon Sep 17 00:00:00 2001 From: divyav-aot Date: Thu, 15 Feb 2024 10:41:40 -0500 Subject: [PATCH 03/19] fix for memory issue --- web/src/components/FOI/Home/Redlining.js | 46 +++++++++++++++++------- 1 file changed, 33 insertions(+), 13 deletions(-) diff --git a/web/src/components/FOI/Home/Redlining.js b/web/src/components/FOI/Home/Redlining.js index 4b8cf6a1e..73b618c2e 100644 --- a/web/src/components/FOI/Home/Redlining.js +++ b/web/src/components/FOI/Home/Redlining.js @@ -2113,19 +2113,17 @@ const Redlining = React.forwardRef( divisionsdocpages, redlineSinglePackage ) => { - let doc = await _docViwer.getPDFDoc(); - let docArr = []; + // Linearize the StitchObject as it can contain different types for pages + // (ex: Normal pdf pages and scanned pages) which is causing setDirty() error while stamping the pagenumver + let doc = await linearizeStitchObject(_docViwer, PDFNet); for ( let pagecount = 1; pagecount <= divisionsdocpages.length; pagecount++ ) { + let pageIndex = 0; // Run PDFNet methods with memory management await PDFNet.runWithCleanup(async () => { - if (docArr.length > 0) { - //updated document after each page stamping - doc = await PDFNet.PDFDoc.createFromBuffer(docArr); - } // lock the document before a write operation // runWithCleanup will auto unlock when complete doc.lock(); @@ -2156,17 +2154,39 @@ const Redlining = React.forwardRef( `${requestnumber} , Page ${pagenumber} of ${totalpagenumber}`, pgSet ); - doc.unlock(); - // PDFTron might automatically handle marking the document as "dirty" during modifications. - // Make sure that you are saving the document correctly after any modifications. - const docBuf = await doc.saveMemoryBuffer(PDFNet.SDFDoc.SaveOptions.e_linearized); - docArr = new Uint8Array(docBuf.buffer); - }); + pageIndex++; + }); } - const pageStampedDocument = await docInstance.Core.createDocument(docArr,{ extension: 'pdf' }); + // After stamping the stitchObj is in PDFDoc format. + // Below method will change it back to Document format + const pageStampedDocument = convertToDocument(doc, PDFNet); return pageStampedDocument; }; + const convertToDocument = async (doc, PDFNet) => { + let pageStampedBufArr = await getDocumentArray(doc, PDFNet); + const pageStampedDocument = await docInstance.Core.createDocument(pageStampedBufArr,{ extension: 'pdf' }); + pageStampedBufArr = []; + return pageStampedDocument; + } + + const linearizeStitchObject = async (_docViwer, PDFNet) => { + let doc = await _docViwer.getPDFDoc(); + let docArr = await getDocumentArray(doc, PDFNet); + if (docArr.length > 0) { + doc = await PDFNet.PDFDoc.createFromBuffer(docArr); + docArr = [] + } + return doc; + } + + const getDocumentArray = async (doc, PDFNet) => { + let docBuf = await doc.saveMemoryBuffer(PDFNet.SDFDoc.SaveOptions.e_linearized); + let docArr = new Uint8Array(docBuf.buffer); + docBuf = null; + return docArr; + } + const stampPageNumberResponse = async (_docViwer, PDFNet) => { for ( let pagecount = 1; From 9a3f8f46396834e26ce81e3990b67f0e39773aaa Mon Sep 17 00:00:00 2001 From: divyav-aot Date: Thu, 15 Feb 2024 10:44:54 -0500 Subject: [PATCH 04/19] added comments --- web/src/components/FOI/Home/Redlining.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/web/src/components/FOI/Home/Redlining.js b/web/src/components/FOI/Home/Redlining.js index 73b618c2e..73d98c007 100644 --- a/web/src/components/FOI/Home/Redlining.js +++ b/web/src/components/FOI/Home/Redlining.js @@ -2113,8 +2113,8 @@ const Redlining = React.forwardRef( divisionsdocpages, redlineSinglePackage ) => { - // Linearize the StitchObject as it can contain different types for pages - // (ex: Normal pdf pages and scanned pages) which is causing setDirty() error while stamping the pagenumver + // Linearize the StitchObject as it can contain different types of pages + // (ex: Standard pdf and scanned pages) which is causing setDirty() error while stamping the pagenumber let doc = await linearizeStitchObject(_docViwer, PDFNet); for ( let pagecount = 1; @@ -2157,7 +2157,7 @@ const Redlining = React.forwardRef( pageIndex++; }); } - // After stamping the stitchObj is in PDFDoc format. + // The stitchObj is in PDFDoc format after the page number stamping. // Below method will change it back to Document format const pageStampedDocument = convertToDocument(doc, PDFNet); return pageStampedDocument; From 7568879a8a535844a8cf49602a231b6067a41c44 Mon Sep 17 00:00:00 2001 From: divyav-aot Date: Thu, 15 Feb 2024 16:13:34 -0500 Subject: [PATCH 05/19] reverted FE linearization --- web/src/components/FOI/Home/Redlining.js | 122 +++++++++-------------- 1 file changed, 49 insertions(+), 73 deletions(-) diff --git a/web/src/components/FOI/Home/Redlining.js b/web/src/components/FOI/Home/Redlining.js index 73d98c007..26df76b82 100644 --- a/web/src/components/FOI/Home/Redlining.js +++ b/web/src/components/FOI/Home/Redlining.js @@ -2113,79 +2113,55 @@ const Redlining = React.forwardRef( divisionsdocpages, redlineSinglePackage ) => { - // Linearize the StitchObject as it can contain different types of pages - // (ex: Standard pdf and scanned pages) which is causing setDirty() error while stamping the pagenumber - let doc = await linearizeStitchObject(_docViwer, PDFNet); - for ( - let pagecount = 1; - pagecount <= divisionsdocpages.length; - pagecount++ - ) { - let pageIndex = 0; - // Run PDFNet methods with memory management - await PDFNet.runWithCleanup(async () => { - // lock the document before a write operation - // runWithCleanup will auto unlock when complete - doc.lock(); - const s = await PDFNet.Stamper.create( - PDFNet.Stamper.SizeType.e_relative_scale, - 0.3, - 0.3 - ); + try { + for ( + let pagecount = 1; + pagecount <= divisionsdocpages.length; + pagecount++ + ) { + + let doc = await _docViwer.getPDFDoc(); + let pageIndex = 0; + // Run PDFNet methods with memory management + await PDFNet.runWithCleanup(async () => { + // lock the document before a write operation + // runWithCleanup will auto unlock when complete + doc.lock(); + const s = await PDFNet.Stamper.create( + PDFNet.Stamper.SizeType.e_relative_scale, + 0.3, + 0.3 + ); - await s.setAlignment( - PDFNet.Stamper.HorizontalAlignment.e_horizontal_center, - PDFNet.Stamper.VerticalAlignment.e_vertical_bottom - ); - const font = await PDFNet.Font.create( - doc, - PDFNet.Font.StandardType1Font.e_courier - ); - await s.setFont(font); - const redColorPt = await PDFNet.ColorPt.init(0, 0, 128, 0.5); - await s.setFontColor(redColorPt); - await s.setTextAlignment(PDFNet.Stamper.TextAlignment.e_align_right); - await s.setAsBackground(false); - const pgSet = await PDFNet.PageSet.createRange(pagecount, pagecount); - let pagenumber = redlineSinglePackage == "Y" || redlineCategory === "oipcreview" ? pagecount : divisionsdocpages[pagecount - 1]?.stitchedPageNo - let totalpagenumber = redlineCategory === "oipcreview" ? _docViwer.getPageCount() : docViewer.getPageCount() - await s.stampText( - doc, - `${requestnumber} , Page ${pagenumber} of ${totalpagenumber}`, - pgSet - ); - pageIndex++; - }); - } - // The stitchObj is in PDFDoc format after the page number stamping. - // Below method will change it back to Document format - const pageStampedDocument = convertToDocument(doc, PDFNet); - return pageStampedDocument; + await s.setAlignment( + PDFNet.Stamper.HorizontalAlignment.e_horizontal_center, + PDFNet.Stamper.VerticalAlignment.e_vertical_bottom + ); + const font = await PDFNet.Font.create( + doc, + PDFNet.Font.StandardType1Font.e_courier + ); + await s.setFont(font); + const redColorPt = await PDFNet.ColorPt.init(0, 0, 128, 0.5); + await s.setFontColor(redColorPt); + await s.setTextAlignment(PDFNet.Stamper.TextAlignment.e_align_right); + await s.setAsBackground(false); + const pgSet = await PDFNet.PageSet.createRange(pagecount, pagecount); + let pagenumber = redlineSinglePackage == "Y" || redlineCategory === "oipcreview" ? pagecount : divisionsdocpages[pagecount - 1]?.stitchedPageNo + let totalpagenumber = redlineCategory === "oipcreview" ? _docViwer.getPageCount() : docViewer.getPageCount() + await s.stampText( + doc, + `${requestnumber} , Page ${pagenumber} of ${totalpagenumber}`, + pgSet + ); + pageIndex++; + }); + } + } catch (err) { + console.log(err); + throw err; + } }; - - const convertToDocument = async (doc, PDFNet) => { - let pageStampedBufArr = await getDocumentArray(doc, PDFNet); - const pageStampedDocument = await docInstance.Core.createDocument(pageStampedBufArr,{ extension: 'pdf' }); - pageStampedBufArr = []; - return pageStampedDocument; - } - - const linearizeStitchObject = async (_docViwer, PDFNet) => { - let doc = await _docViwer.getPDFDoc(); - let docArr = await getDocumentArray(doc, PDFNet); - if (docArr.length > 0) { - doc = await PDFNet.PDFDoc.createFromBuffer(docArr); - docArr = [] - } - return doc; - } - - const getDocumentArray = async (doc, PDFNet) => { - let docBuf = await doc.saveMemoryBuffer(PDFNet.SDFDoc.SaveOptions.e_linearized); - let docArr = new Uint8Array(docBuf.buffer); - docBuf = null; - return docArr; - } const stampPageNumberResponse = async (_docViwer, PDFNet) => { for ( @@ -3151,7 +3127,7 @@ const Redlining = React.forwardRef( redlineStitchInfo[divisionid]["documentids"] ); if(redlineCategory !== "oipcreview") { - stitchObject = await stampPageNumberRedline( + await stampPageNumberRedline( stitchObject, PDFNet, redlineStitchInfo[divisionid]["stitchpages"], @@ -3206,7 +3182,7 @@ const Redlining = React.forwardRef( const doc = await stitchObject.getPDFDoc(); await PDFNet.Redactor.redact(doc, rarr, app); } - stitchObject = await stampPageNumberRedline( + await stampPageNumberRedline( stitchObject, PDFNet, redlineStitchInfo[divisionid]["stitchpages"], From 256d5e6964fb2ba2cbe9d77a94364457b4f20f63 Mon Sep 17 00:00:00 2001 From: divyav-aot Date: Fri, 16 Feb 2024 15:26:03 -0500 Subject: [PATCH 06/19] dedupe changes with pdfstitchlinearization --- .../DedupeServices/requirements.txt | Bin 794 -> 828 bytes .../services/dedupedbservice.py | 8 +++- .../DedupeServices/services/dedupeservice.py | 5 +- .../services/pdflinearizationservice.py | 32 +++++++++++++ .../services/s3documentservice.py | 43 ++++++++++++++++++ 5 files changed, 85 insertions(+), 3 deletions(-) create mode 100644 computingservices/DedupeServices/services/pdflinearizationservice.py diff --git a/computingservices/DedupeServices/requirements.txt b/computingservices/DedupeServices/requirements.txt index ca7e1b33cdcfb97590fa2882d3308273c2446589..8401abd88215e6a22eb1d28edb0e8204dbdc711a 100644 GIT binary patch delta 42 tcmbQmwufzl6tfa90~bR8LnVVRLn)AUVQ^!x1wumxJq9BNV<2h4005{)266xZ delta 7 OcmdnPHj8b86f*z{umUgu diff --git a/computingservices/DedupeServices/services/dedupedbservice.py b/computingservices/DedupeServices/services/dedupedbservice.py index 92d68febf..b009a2a34 100644 --- a/computingservices/DedupeServices/services/dedupedbservice.py +++ b/computingservices/DedupeServices/services/dedupedbservice.py @@ -3,7 +3,7 @@ from datetime import datetime import json -def savedocumentdetails(dedupeproducermessage, hashcode, pagecount = 1): +def savedocumentdetails(dedupeproducermessage, hashcode, pagecount = 1, processedfilepath = ""): conn = getdbconnection() try: cursor = conn.cursor() @@ -17,13 +17,17 @@ def savedocumentdetails(dedupeproducermessage, hashcode, pagecount = 1): conn.commit() id_of_new_row = cursor.fetchone() + documentattribute = dedupeproducermessage.attributes + if processedfilepath: + documentattribute["processedfilepath"] = processedfilepath + if (dedupeproducermessage.attributes.get('isattachment', False) and dedupeproducermessage.trigger == 'recordreplace'): documentmasterid = dedupeproducermessage.originaldocumentmasterid or dedupeproducermessage.documentmasterid else: documentmasterid = dedupeproducermessage.documentmasterid cursor.execute('''UPDATE public."DocumentAttributes" SET attributes = %s WHERE documentmasterid = %s''', - (json.dumps(dedupeproducermessage.attributes), documentmasterid)) + (json.dumps(documentattribute), documentmasterid)) conn.commit() cursor.execute('INSERT INTO public."DocumentHashCodes" (documentid, \ diff --git a/computingservices/DedupeServices/services/dedupeservice.py b/computingservices/DedupeServices/services/dedupeservice.py index d434c737b..9b3e6ac6b 100644 --- a/computingservices/DedupeServices/services/dedupeservice.py +++ b/computingservices/DedupeServices/services/dedupeservice.py @@ -1,6 +1,7 @@ from .s3documentservice import gets3documenthashcode from .dedupedbservice import savedocumentdetails, recordjobstart, recordjobend, updateredactionstatus +from .pdflinearizationservice import pdflinearizationservice import traceback @@ -8,7 +9,9 @@ def processmessage(message): recordjobstart(message) try: hashcode, _pagecount = gets3documenthashcode(message) - savedocumentdetails(message, hashcode, _pagecount) + linearizedfilepath = pdflinearizationservice().linearizepdf(message) + print(f'linearizedfilepath = {linearizedfilepath}') + savedocumentdetails(message, hashcode, _pagecount, linearizedfilepath) recordjobend(message, False) updateredactionstatus(message) except(Exception) as error: diff --git a/computingservices/DedupeServices/services/pdflinearizationservice.py b/computingservices/DedupeServices/services/pdflinearizationservice.py new file mode 100644 index 000000000..4f65788ae --- /dev/null +++ b/computingservices/DedupeServices/services/pdflinearizationservice.py @@ -0,0 +1,32 @@ +from os import path +from io import BytesIO +import fitz +from utils import ( + file_conversion_types +) +from .s3documentservice import gets3documentbytearray, uploaddocument + +class pdflinearizationservice: + + def linearizepdf(self, producermessage): + try: + _filename, extension = path.splitext(producermessage.filename) + s3uripath = "" + if extension.lower() in [".pdf"] or extension.lower() in file_conversion_types: + pdfbytearray = gets3documentbytearray(producermessage) + if pdfbytearray: + _bytes = BytesIO(pdfbytearray) + pdf_doc_in = fitz.open(stream=_bytes) + bytes_stream = BytesIO() + if pdf_doc_in: + pdf_doc_in.save(bytes_stream, linear=True) + s3uripath = path.splitext(producermessage.s3filepath)[0] + "_linearized" + extension + uploaddocument(producermessage.bcgovcode, s3uripath, bytes_stream.getvalue()) + pdf_doc_in.close() + del pdf_doc_in + bytes_stream.close() + return s3uripath + except Exception as ex: + print("Error in Linearizing PDF") + print(ex) + raise diff --git a/computingservices/DedupeServices/services/s3documentservice.py b/computingservices/DedupeServices/services/s3documentservice.py index 6f4d46047..aa1e82da4 100644 --- a/computingservices/DedupeServices/services/s3documentservice.py +++ b/computingservices/DedupeServices/services/s3documentservice.py @@ -136,3 +136,46 @@ def gets3documenthashcode(producermessage): sig.update(line) return (sig.hexdigest(), pagecount) + + +def gets3documentbytearray(producermessage): + filepath = producermessage.s3filepath + try: + s3credentials = __getcredentialsbybcgovcode(producermessage.bcgovcode) + s3_access_key_id = s3credentials.s3accesskey + s3_secret_access_key = s3credentials.s3secretkey + + auth = AWSRequestsAuth( + aws_access_key=s3_access_key_id, + aws_secret_access_key=s3_secret_access_key, + aws_host=dedupe_s3_host, + aws_region=dedupe_s3_region, + aws_service=dedupe_s3_service, + ) + response= requests.get(filepath, auth=auth,stream=True) + return response.content + except Exception as ex: + print("Error in connecting S3.") + print(ex) + + + +def uploaddocument(bcgovcode, s3uripath, filebytes): + try: + s3credentials = __getcredentialsbybcgovcode(bcgovcode) + s3_access_key_id = s3credentials.s3accesskey + s3_secret_access_key = s3credentials.s3secretkey + auth = AWSRequestsAuth( + aws_access_key=s3_access_key_id, + aws_secret_access_key=s3_secret_access_key, + aws_host=dedupe_s3_host, + aws_region=dedupe_s3_region, + aws_service=dedupe_s3_service, + ) + + uploadresponse = requests.put(s3uripath, data=filebytes, auth=auth) + uploadresponse.raise_for_status() + + except Exception as ex: + print("Error in writing to S3.") + print(ex) \ No newline at end of file From cb519864541088538346eb14dbf19ea88cc8839e Mon Sep 17 00:00:00 2001 From: divyav-aot Date: Tue, 20 Feb 2024 12:20:58 -0500 Subject: [PATCH 07/19] reviewer-api changes --- .../resources/foiflowmasterdata.py | 19 +++++++++++-------- api/reviewer_api/services/documentservice.py | 4 ++++ 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/api/reviewer_api/resources/foiflowmasterdata.py b/api/reviewer_api/resources/foiflowmasterdata.py index 2b553984e..a12073ee1 100644 --- a/api/reviewer_api/resources/foiflowmasterdata.py +++ b/api/reviewer_api/resources/foiflowmasterdata.py @@ -129,11 +129,15 @@ def post(): ) documentobjs = [] - documentids = [documentinfo["file"]["documentid"] for documentinfo in data["documentobjs"]] - documents = documentservice().getdocumentbyids(documentids) + # documentids = [documentinfo["file"]["documentid"] for documentinfo in data["documentobjs"]] + # documents = documentservice().getdocumentbyids(documentids) for documentinfo in data["documentobjs"]: - filepath = "/".join(documents[documentinfo["file"]["documentid"]].split("/")[4:]) + filepath = "/".join(documentinfo["file"]["filepath"].split("/")[4:]) + if documentinfo["file"]["processedfilepath"]: + filepath = "/".join(documentinfo["file"]["processedfilepath"].split("/")[4:]) filename, file_extension = os.path.splitext(filepath) + if file_extension in FILE_CONVERSION_FILE_TYPES: + filepath = filename + ".pdf" documentinfo["s3url"] = s3client.generate_presigned_url( ClientMethod="get_object", Params={ @@ -217,13 +221,12 @@ def post(ministryrequestid, redactionlayer="redline", layertype="redline"): # for save/put - stitch by division div["s3path_save"] = s3path_save for doc in div["documentlist"]: - realfilepath = documentservice().getfilepathbydocumentid(doc["documentid"]) - # filepathlist = doc["filepath"].split("/")[4:] - filepathlist = realfilepath.split("/")[4:] - + filepathlist = doc["filepath"].split("/")[4:] + if doc["processedfilepath"]: + filepathlist = doc["processedfilepath"].split("/")[4:] # for load/get filepath_get = "/".join(filepathlist) - + filename_get, file_extension_get = os.path.splitext( filepath_get ) diff --git a/api/reviewer_api/services/documentservice.py b/api/reviewer_api/services/documentservice.py index 1dbfaa9c0..9baac7eb8 100644 --- a/api/reviewer_api/services/documentservice.py +++ b/api/reviewer_api/services/documentservice.py @@ -404,6 +404,8 @@ def updatedocumentattributes(self, payload, userid): return DocumentAttributes.update(newRows, oldRows) + def __getprocessedfilepath(self, attributes): + return attributes.get("processedfilepath", None) def getdocuments(self, requestid,bcgovcode): divisions_data = requests.request( @@ -455,6 +457,8 @@ def getdocuments(self, requestid,bcgovcode): for documentid in documents: document = documents[documentid] + documentattributes = document["attributes"] + document["processedfilepath"] = self.__getprocessedfilepath(documentattributes) documentdivisions = set( map(lambda d: d["divisionid"], document["attributes"]["divisions"]) ) From dd0794cf47878b06eb465e6a92ef78535cb85d13 Mon Sep 17 00:00:00 2001 From: divyav-aot Date: Tue, 20 Feb 2024 13:53:39 -0500 Subject: [PATCH 08/19] added comments --- api/reviewer_api/resources/foiflowmasterdata.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/reviewer_api/resources/foiflowmasterdata.py b/api/reviewer_api/resources/foiflowmasterdata.py index a12073ee1..b3d15db7c 100644 --- a/api/reviewer_api/resources/foiflowmasterdata.py +++ b/api/reviewer_api/resources/foiflowmasterdata.py @@ -129,10 +129,9 @@ def post(): ) documentobjs = [] - # documentids = [documentinfo["file"]["documentid"] for documentinfo in data["documentobjs"]] - # documents = documentservice().getdocumentbyids(documentids) for documentinfo in data["documentobjs"]: filepath = "/".join(documentinfo["file"]["filepath"].split("/")[4:]) + # get the Linearized document if documentinfo["file"]["processedfilepath"]: filepath = "/".join(documentinfo["file"]["processedfilepath"].split("/")[4:]) filename, file_extension = os.path.splitext(filepath) @@ -222,6 +221,7 @@ def post(ministryrequestid, redactionlayer="redline", layertype="redline"): div["s3path_save"] = s3path_save for doc in div["documentlist"]: filepathlist = doc["filepath"].split("/")[4:] + # get the Linearized document if doc["processedfilepath"]: filepathlist = doc["processedfilepath"].split("/")[4:] # for load/get From 1cc4b549026e11e145d3243b5ba079ea0c38fc8c Mon Sep 17 00:00:00 2001 From: divyav-aot Date: Tue, 20 Feb 2024 14:07:45 -0500 Subject: [PATCH 09/19] removed unwanted code --- web/src/components/FOI/Home/Redlining.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/web/src/components/FOI/Home/Redlining.js b/web/src/components/FOI/Home/Redlining.js index 26df76b82..e76a5b84b 100644 --- a/web/src/components/FOI/Home/Redlining.js +++ b/web/src/components/FOI/Home/Redlining.js @@ -2120,8 +2120,7 @@ const Redlining = React.forwardRef( pagecount++ ) { - let doc = await _docViwer.getPDFDoc(); - let pageIndex = 0; + const doc = await _docViwer.getPDFDoc(); // Run PDFNet methods with memory management await PDFNet.runWithCleanup(async () => { // lock the document before a write operation @@ -2154,7 +2153,6 @@ const Redlining = React.forwardRef( `${requestnumber} , Page ${pagenumber} of ${totalpagenumber}`, pgSet ); - pageIndex++; }); } } catch (err) { From 845bf40c9d9bd3337d47cc4f0a8f63fb2b0e46e0 Mon Sep 17 00:00:00 2001 From: divyav-aot Date: Wed, 21 Feb 2024 14:13:37 -0500 Subject: [PATCH 10/19] logs added --- web/src/components/FOI/Home/Redlining.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/web/src/components/FOI/Home/Redlining.js b/web/src/components/FOI/Home/Redlining.js index e76a5b84b..143c15e13 100644 --- a/web/src/components/FOI/Home/Redlining.js +++ b/web/src/components/FOI/Home/Redlining.js @@ -689,6 +689,7 @@ const Redlining = React.forwardRef( set ) => { slicedsetofdoclist.forEach(async (filerow) => { + console.log(`filename = ${filerow.file.filename}`) await createDocument(filerow.s3url).then(async (newDoc) => { setpdftronDocObjects((_arr) => [ ..._arr, @@ -1287,10 +1288,12 @@ const Redlining = React.forwardRef( ); if (_exists?.length === 0) { let index = filerow.stitchIndex; + console.log(`index = ${index}, filerow.pages = ${filerow.pages}`) _doc .insertPages(filerow.pdftronobject, filerow.pages, index) .then(() => { const pageCount = docViewer.getPageCount(); + console.log(`pageCount = ${pageCount}, totalPageCount = ${docsForStitcing.totalPageCount}`) if (pageCount === docsForStitcing.totalPageCount) { setStitchPageCount(pageCount); } From 046db9e9199ecd5efa82d1a801ab43d0ddde357d Mon Sep 17 00:00:00 2001 From: divyav-aot Date: Wed, 21 Feb 2024 14:22:00 -0500 Subject: [PATCH 11/19] logs added --- web/src/components/FOI/Home/Redlining.js | 1 + 1 file changed, 1 insertion(+) diff --git a/web/src/components/FOI/Home/Redlining.js b/web/src/components/FOI/Home/Redlining.js index 143c15e13..d7a33dc64 100644 --- a/web/src/components/FOI/Home/Redlining.js +++ b/web/src/components/FOI/Home/Redlining.js @@ -691,6 +691,7 @@ const Redlining = React.forwardRef( slicedsetofdoclist.forEach(async (filerow) => { console.log(`filename = ${filerow.file.filename}`) await createDocument(filerow.s3url).then(async (newDoc) => { + console.log(`createDocument filename = ${filerow.file.filename}`) setpdftronDocObjects((_arr) => [ ..._arr, { From 982e132f05513a26b19d67b0eef848f6ecf18f19 Mon Sep 17 00:00:00 2001 From: divyav-aot Date: Thu, 22 Feb 2024 09:59:15 -0500 Subject: [PATCH 12/19] disabled linearization and added useDownloader: false for the createDocument --- .../DedupeServices/services/dedupeservice.py | 7 ++--- web/src/components/FOI/Home/Redlining.js | 26 ++++++++++--------- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/computingservices/DedupeServices/services/dedupeservice.py b/computingservices/DedupeServices/services/dedupeservice.py index 9b3e6ac6b..323cd6e26 100644 --- a/computingservices/DedupeServices/services/dedupeservice.py +++ b/computingservices/DedupeServices/services/dedupeservice.py @@ -9,9 +9,10 @@ def processmessage(message): recordjobstart(message) try: hashcode, _pagecount = gets3documenthashcode(message) - linearizedfilepath = pdflinearizationservice().linearizepdf(message) - print(f'linearizedfilepath = {linearizedfilepath}') - savedocumentdetails(message, hashcode, _pagecount, linearizedfilepath) + # linearizedfilepath = pdflinearizationservice().linearizepdf(message) + # print(f'linearizedfilepath = {linearizedfilepath}') + # savedocumentdetails(message, hashcode, _pagecount, linearizedfilepath) + savedocumentdetails(message, hashcode, _pagecount) recordjobend(message, False) updateredactionstatus(message) except(Exception) as error: diff --git a/web/src/components/FOI/Home/Redlining.js b/web/src/components/FOI/Home/Redlining.js index d7a33dc64..d31f3ac7d 100644 --- a/web/src/components/FOI/Home/Redlining.js +++ b/web/src/components/FOI/Home/Redlining.js @@ -689,9 +689,7 @@ const Redlining = React.forwardRef( set ) => { slicedsetofdoclist.forEach(async (filerow) => { - console.log(`filename = ${filerow.file.filename}`) - await createDocument(filerow.s3url).then(async (newDoc) => { - console.log(`createDocument filename = ${filerow.file.filename}`) + await createDocument(filerow.s3url, { useDownloader: false }).then(async (newDoc) => { setpdftronDocObjects((_arr) => [ ..._arr, { @@ -1289,12 +1287,10 @@ const Redlining = React.forwardRef( ); if (_exists?.length === 0) { let index = filerow.stitchIndex; - console.log(`index = ${index}, filerow.pages = ${filerow.pages}`) _doc .insertPages(filerow.pdftronobject, filerow.pages, index) .then(() => { const pageCount = docViewer.getPageCount(); - console.log(`pageCount = ${pageCount}, totalPageCount = ${docsForStitcing.totalPageCount}`) if (pageCount === docsForStitcing.totalPageCount) { setStitchPageCount(pageCount); } @@ -2896,6 +2892,7 @@ const Redlining = React.forwardRef( for (let doc of documentlist) { await _instance.Core.createDocument(doc.s3path_load, { loadAsPDF: true, + useDownloader: false, }).then(async (docObj) => { //if (isIgnoredDocument(doc, docObj.getPageCount(), divisionDocuments) == false) { docCount++; @@ -3002,10 +2999,9 @@ const Redlining = React.forwardRef( docCount, stitchedDocObj, ) => { - try { - //console.log("\nsliceDoclist:", sliceDoclist.length); for (const filerow of sliceDoclist) { - await createDocument(filerow.s3path_load).then(async (newDoc) => { + try { + await createDocument(filerow.s3path_load, { useDownloader: false }).then(async (newDoc) => { docCount++; setredlineDocCount(docCount); if (isIgnoredDocument(filerow, newDoc, divisionDocuments) === false) { @@ -3028,10 +3024,12 @@ const Redlining = React.forwardRef( } } }); + } catch (error) { + console.error("An error occurred during create document:", error); + // Handle any errors that occurred during the asynchronous operations } - } catch (error) { - // Handle any errors that occurred during the asynchronous operations - } + } + }; useEffect(() => { @@ -3086,7 +3084,10 @@ const Redlining = React.forwardRef( if (_exists?.length === 0) { let index = filerow.stitchIndex; try { - stichedfilesForRedline?.insertPages(filerow.pdftronobject, filerow.pages, index); + stichedfilesForRedline?.insertPages(filerow.pdftronobject, filerow.pages, index).then(() => { + }).catch((error) => { + console.error("An error occurred during page insertion:", error); + }) ; setAlreadyStitchedList((_arr) => [..._arr, filerow]); setstichedfilesForRedline(stichedfilesForRedline) } catch (error) { @@ -3200,6 +3201,7 @@ const Redlining = React.forwardRef( // saves the document with annotations in it xfdfString: xfdfString, downloadType: downloadType, + // flags: docInstance.Core.SaveOptions.LINEARIZED, //flatten: true, //commented this as part of #4862 }) .then(async (_data) => { From 2ee29823aed4c38e3213a71b75dc2ef3bf808941 Mon Sep 17 00:00:00 2001 From: divyav-aot Date: Thu, 22 Feb 2024 13:03:32 -0500 Subject: [PATCH 13/19] revert linearization changes --- .../resources/foiflowmasterdata.py | 21 ++++----- api/reviewer_api/services/documentservice.py | 12 +---- .../services/dedupedbservice.py | 8 +--- .../DedupeServices/services/dedupeservice.py | 4 -- .../services/pdflinearizationservice.py | 32 ------------- .../services/s3documentservice.py | 45 +------------------ 6 files changed, 13 insertions(+), 109 deletions(-) delete mode 100644 computingservices/DedupeServices/services/pdflinearizationservice.py diff --git a/api/reviewer_api/resources/foiflowmasterdata.py b/api/reviewer_api/resources/foiflowmasterdata.py index b3d15db7c..c35f9c4c7 100644 --- a/api/reviewer_api/resources/foiflowmasterdata.py +++ b/api/reviewer_api/resources/foiflowmasterdata.py @@ -129,14 +129,11 @@ def post(): ) documentobjs = [] + documentids = [documentinfo["file"]["documentid"] for documentinfo in data["documentobjs"]] + documents = documentservice().getdocumentbyids(documentids) for documentinfo in data["documentobjs"]: - filepath = "/".join(documentinfo["file"]["filepath"].split("/")[4:]) - # get the Linearized document - if documentinfo["file"]["processedfilepath"]: - filepath = "/".join(documentinfo["file"]["processedfilepath"].split("/")[4:]) + filepath = "/".join(documents[documentinfo["file"]["documentid"]].split("/")[4:]) filename, file_extension = os.path.splitext(filepath) - if file_extension in FILE_CONVERSION_FILE_TYPES: - filepath = filename + ".pdf" documentinfo["s3url"] = s3client.generate_presigned_url( ClientMethod="get_object", Params={ @@ -220,13 +217,13 @@ def post(ministryrequestid, redactionlayer="redline", layertype="redline"): # for save/put - stitch by division div["s3path_save"] = s3path_save for doc in div["documentlist"]: - filepathlist = doc["filepath"].split("/")[4:] - # get the Linearized document - if doc["processedfilepath"]: - filepathlist = doc["processedfilepath"].split("/")[4:] + realfilepath = documentservice().getfilepathbydocumentid(doc["documentid"]) + # filepathlist = doc["filepath"].split("/")[4:] + filepathlist = realfilepath.split("/")[4:] + # for load/get filepath_get = "/".join(filepathlist) - + filename_get, file_extension_get = os.path.splitext( filepath_get ) @@ -373,4 +370,4 @@ def get(): try: return json.dumps({"license": webviewerlicense}), 200 except BusinessException as exception: - return {"status": exception.status_code, "message": exception.message}, 500 + return {"status": exception.status_code, "message": exception.message}, 500 \ No newline at end of file diff --git a/api/reviewer_api/services/documentservice.py b/api/reviewer_api/services/documentservice.py index 9baac7eb8..3088c723a 100644 --- a/api/reviewer_api/services/documentservice.py +++ b/api/reviewer_api/services/documentservice.py @@ -404,8 +404,6 @@ def updatedocumentattributes(self, payload, userid): return DocumentAttributes.update(newRows, oldRows) - def __getprocessedfilepath(self, attributes): - return attributes.get("processedfilepath", None) def getdocuments(self, requestid,bcgovcode): divisions_data = requests.request( @@ -424,12 +422,6 @@ def getdocuments(self, requestid,bcgovcode): for documentid in documents: _attachments = documents[documentid].pop("attachments", []) for attachment in _attachments: - attachment["attributes"]["attachmentlastmodified"] = attachment[ - "attributes" - ]["lastmodified"] - attachment["attributes"]["lastmodified"] = documents[documentid][ - "attributes" - ]["lastmodified"] attachments.append(attachment) for attachment in attachments: @@ -457,8 +449,6 @@ def getdocuments(self, requestid,bcgovcode): for documentid in documents: document = documents[documentid] - documentattributes = document["attributes"] - document["processedfilepath"] = self.__getprocessedfilepath(documentattributes) documentdivisions = set( map(lambda d: d["divisionid"], document["attributes"]["divisions"]) ) @@ -510,4 +500,4 @@ def __get_close_datetime(self, request_json): if state['status'] == StateName.response.value and isresponsephasecompleted == False: isresponsephasecompleted = True return generatedbefore if isresponsephasecompleted == True else None - + \ No newline at end of file diff --git a/computingservices/DedupeServices/services/dedupedbservice.py b/computingservices/DedupeServices/services/dedupedbservice.py index b009a2a34..92d68febf 100644 --- a/computingservices/DedupeServices/services/dedupedbservice.py +++ b/computingservices/DedupeServices/services/dedupedbservice.py @@ -3,7 +3,7 @@ from datetime import datetime import json -def savedocumentdetails(dedupeproducermessage, hashcode, pagecount = 1, processedfilepath = ""): +def savedocumentdetails(dedupeproducermessage, hashcode, pagecount = 1): conn = getdbconnection() try: cursor = conn.cursor() @@ -17,17 +17,13 @@ def savedocumentdetails(dedupeproducermessage, hashcode, pagecount = 1, processe conn.commit() id_of_new_row = cursor.fetchone() - documentattribute = dedupeproducermessage.attributes - if processedfilepath: - documentattribute["processedfilepath"] = processedfilepath - if (dedupeproducermessage.attributes.get('isattachment', False) and dedupeproducermessage.trigger == 'recordreplace'): documentmasterid = dedupeproducermessage.originaldocumentmasterid or dedupeproducermessage.documentmasterid else: documentmasterid = dedupeproducermessage.documentmasterid cursor.execute('''UPDATE public."DocumentAttributes" SET attributes = %s WHERE documentmasterid = %s''', - (json.dumps(documentattribute), documentmasterid)) + (json.dumps(dedupeproducermessage.attributes), documentmasterid)) conn.commit() cursor.execute('INSERT INTO public."DocumentHashCodes" (documentid, \ diff --git a/computingservices/DedupeServices/services/dedupeservice.py b/computingservices/DedupeServices/services/dedupeservice.py index 323cd6e26..d434c737b 100644 --- a/computingservices/DedupeServices/services/dedupeservice.py +++ b/computingservices/DedupeServices/services/dedupeservice.py @@ -1,7 +1,6 @@ from .s3documentservice import gets3documenthashcode from .dedupedbservice import savedocumentdetails, recordjobstart, recordjobend, updateredactionstatus -from .pdflinearizationservice import pdflinearizationservice import traceback @@ -9,9 +8,6 @@ def processmessage(message): recordjobstart(message) try: hashcode, _pagecount = gets3documenthashcode(message) - # linearizedfilepath = pdflinearizationservice().linearizepdf(message) - # print(f'linearizedfilepath = {linearizedfilepath}') - # savedocumentdetails(message, hashcode, _pagecount, linearizedfilepath) savedocumentdetails(message, hashcode, _pagecount) recordjobend(message, False) updateredactionstatus(message) diff --git a/computingservices/DedupeServices/services/pdflinearizationservice.py b/computingservices/DedupeServices/services/pdflinearizationservice.py deleted file mode 100644 index 4f65788ae..000000000 --- a/computingservices/DedupeServices/services/pdflinearizationservice.py +++ /dev/null @@ -1,32 +0,0 @@ -from os import path -from io import BytesIO -import fitz -from utils import ( - file_conversion_types -) -from .s3documentservice import gets3documentbytearray, uploaddocument - -class pdflinearizationservice: - - def linearizepdf(self, producermessage): - try: - _filename, extension = path.splitext(producermessage.filename) - s3uripath = "" - if extension.lower() in [".pdf"] or extension.lower() in file_conversion_types: - pdfbytearray = gets3documentbytearray(producermessage) - if pdfbytearray: - _bytes = BytesIO(pdfbytearray) - pdf_doc_in = fitz.open(stream=_bytes) - bytes_stream = BytesIO() - if pdf_doc_in: - pdf_doc_in.save(bytes_stream, linear=True) - s3uripath = path.splitext(producermessage.s3filepath)[0] + "_linearized" + extension - uploaddocument(producermessage.bcgovcode, s3uripath, bytes_stream.getvalue()) - pdf_doc_in.close() - del pdf_doc_in - bytes_stream.close() - return s3uripath - except Exception as ex: - print("Error in Linearizing PDF") - print(ex) - raise diff --git a/computingservices/DedupeServices/services/s3documentservice.py b/computingservices/DedupeServices/services/s3documentservice.py index aa1e82da4..073afaf9f 100644 --- a/computingservices/DedupeServices/services/s3documentservice.py +++ b/computingservices/DedupeServices/services/s3documentservice.py @@ -135,47 +135,4 @@ def gets3documenthashcode(producermessage): for line in response.iter_lines(): sig.update(line) - return (sig.hexdigest(), pagecount) - - -def gets3documentbytearray(producermessage): - filepath = producermessage.s3filepath - try: - s3credentials = __getcredentialsbybcgovcode(producermessage.bcgovcode) - s3_access_key_id = s3credentials.s3accesskey - s3_secret_access_key = s3credentials.s3secretkey - - auth = AWSRequestsAuth( - aws_access_key=s3_access_key_id, - aws_secret_access_key=s3_secret_access_key, - aws_host=dedupe_s3_host, - aws_region=dedupe_s3_region, - aws_service=dedupe_s3_service, - ) - response= requests.get(filepath, auth=auth,stream=True) - return response.content - except Exception as ex: - print("Error in connecting S3.") - print(ex) - - - -def uploaddocument(bcgovcode, s3uripath, filebytes): - try: - s3credentials = __getcredentialsbybcgovcode(bcgovcode) - s3_access_key_id = s3credentials.s3accesskey - s3_secret_access_key = s3credentials.s3secretkey - auth = AWSRequestsAuth( - aws_access_key=s3_access_key_id, - aws_secret_access_key=s3_secret_access_key, - aws_host=dedupe_s3_host, - aws_region=dedupe_s3_region, - aws_service=dedupe_s3_service, - ) - - uploadresponse = requests.put(s3uripath, data=filebytes, auth=auth) - uploadresponse.raise_for_status() - - except Exception as ex: - print("Error in writing to S3.") - print(ex) \ No newline at end of file + return (sig.hexdigest(), pagecount) \ No newline at end of file From 732e1ca6b2838700ffdad15efb0a783861df1c24 Mon Sep 17 00:00:00 2001 From: divyav-aot Date: Thu, 22 Feb 2024 13:03:45 -0500 Subject: [PATCH 14/19] revert linearization changes --- .../DedupeServices/requirements.txt | Bin 828 -> 794 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/computingservices/DedupeServices/requirements.txt b/computingservices/DedupeServices/requirements.txt index 8401abd88215e6a22eb1d28edb0e8204dbdc711a..ca7e1b33cdcfb97590fa2882d3308273c2446589 100644 GIT binary patch delta 7 OcmdnPHj8b86f*z{umUgu delta 42 tcmbQmwufzl6tfa90~bR8LnVVRLn)AUVQ^!x1wumxJq9BNV<2h4005{)266xZ From 13c13cf1ad1a3ad4a13432dee3a8b5ac9e36b0e1 Mon Sep 17 00:00:00 2001 From: divyav-aot Date: Fri, 23 Feb 2024 09:34:02 -0500 Subject: [PATCH 15/19] code cleanup --- web/src/components/FOI/Home/Redlining.js | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/web/src/components/FOI/Home/Redlining.js b/web/src/components/FOI/Home/Redlining.js index d31f3ac7d..9ac2d1d5d 100644 --- a/web/src/components/FOI/Home/Redlining.js +++ b/web/src/components/FOI/Home/Redlining.js @@ -689,7 +689,9 @@ const Redlining = React.forwardRef( set ) => { slicedsetofdoclist.forEach(async (filerow) => { - await createDocument(filerow.s3url, { useDownloader: false }).then(async (newDoc) => { + await createDocument(filerow.s3url, + { useDownloader: false } // Added to fix BLANK page issue + ).then(async (newDoc) => { setpdftronDocObjects((_arr) => [ ..._arr, { @@ -2892,7 +2894,7 @@ const Redlining = React.forwardRef( for (let doc of documentlist) { await _instance.Core.createDocument(doc.s3path_load, { loadAsPDF: true, - useDownloader: false, + useDownloader: false, // Added to fix BLANK page issue }).then(async (docObj) => { //if (isIgnoredDocument(doc, docObj.getPageCount(), divisionDocuments) == false) { docCount++; @@ -3001,7 +3003,9 @@ const Redlining = React.forwardRef( ) => { for (const filerow of sliceDoclist) { try { - await createDocument(filerow.s3path_load, { useDownloader: false }).then(async (newDoc) => { + await createDocument(filerow.s3path_load, + { useDownloader: false } // Added to fix BLANK page issue + ).then(async (newDoc) => { docCount++; setredlineDocCount(docCount); if (isIgnoredDocument(filerow, newDoc, divisionDocuments) === false) { @@ -3201,7 +3205,6 @@ const Redlining = React.forwardRef( // saves the document with annotations in it xfdfString: xfdfString, downloadType: downloadType, - // flags: docInstance.Core.SaveOptions.LINEARIZED, //flatten: true, //commented this as part of #4862 }) .then(async (_data) => { From 62d69be8c2ca5f8a243c9d6253d54b800d5408e4 Mon Sep 17 00:00:00 2001 From: divyav-aot Date: Fri, 23 Feb 2024 10:27:54 -0500 Subject: [PATCH 16/19] code cleanup --- api/reviewer_api/resources/foiflowmasterdata.py | 2 +- computingservices/DedupeServices/services/s3documentservice.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/reviewer_api/resources/foiflowmasterdata.py b/api/reviewer_api/resources/foiflowmasterdata.py index c35f9c4c7..2b553984e 100644 --- a/api/reviewer_api/resources/foiflowmasterdata.py +++ b/api/reviewer_api/resources/foiflowmasterdata.py @@ -370,4 +370,4 @@ def get(): try: return json.dumps({"license": webviewerlicense}), 200 except BusinessException as exception: - return {"status": exception.status_code, "message": exception.message}, 500 \ No newline at end of file + return {"status": exception.status_code, "message": exception.message}, 500 diff --git a/computingservices/DedupeServices/services/s3documentservice.py b/computingservices/DedupeServices/services/s3documentservice.py index 073afaf9f..6f4d46047 100644 --- a/computingservices/DedupeServices/services/s3documentservice.py +++ b/computingservices/DedupeServices/services/s3documentservice.py @@ -135,4 +135,4 @@ def gets3documenthashcode(producermessage): for line in response.iter_lines(): sig.update(line) - return (sig.hexdigest(), pagecount) \ No newline at end of file + return (sig.hexdigest(), pagecount) From b3655239ba95c6b385c3ca1f3707cb871ba4860e Mon Sep 17 00:00:00 2001 From: divyav-aot Date: Fri, 23 Feb 2024 10:29:26 -0500 Subject: [PATCH 17/19] code cleanup --- api/reviewer_api/services/documentservice.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/reviewer_api/services/documentservice.py b/api/reviewer_api/services/documentservice.py index 3088c723a..d66f2f5bc 100644 --- a/api/reviewer_api/services/documentservice.py +++ b/api/reviewer_api/services/documentservice.py @@ -500,4 +500,4 @@ def __get_close_datetime(self, request_json): if state['status'] == StateName.response.value and isresponsephasecompleted == False: isresponsephasecompleted = True return generatedbefore if isresponsephasecompleted == True else None - \ No newline at end of file + \ No newline at end of file From edaa41c0ffb25639c6b91c88455b2a7b57613648 Mon Sep 17 00:00:00 2001 From: divyav-aot Date: Fri, 23 Feb 2024 10:29:55 -0500 Subject: [PATCH 18/19] code cleanup --- api/reviewer_api/services/documentservice.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/api/reviewer_api/services/documentservice.py b/api/reviewer_api/services/documentservice.py index d66f2f5bc..754e5f0c0 100644 --- a/api/reviewer_api/services/documentservice.py +++ b/api/reviewer_api/services/documentservice.py @@ -499,5 +499,4 @@ def __get_close_datetime(self, request_json): generatedbefore = state['created_at'] if state['status'] == StateName.response.value and isresponsephasecompleted == False: isresponsephasecompleted = True - return generatedbefore if isresponsephasecompleted == True else None - \ No newline at end of file + return generatedbefore if isresponsephasecompleted == True else None \ No newline at end of file From ff5b1e8ef1f04a4ccd3f8541df644f28bbaf09e4 Mon Sep 17 00:00:00 2001 From: divyav-aot Date: Fri, 23 Feb 2024 10:31:17 -0500 Subject: [PATCH 19/19] code cleanup --- api/reviewer_api/services/documentservice.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/reviewer_api/services/documentservice.py b/api/reviewer_api/services/documentservice.py index 754e5f0c0..65b72b55c 100644 --- a/api/reviewer_api/services/documentservice.py +++ b/api/reviewer_api/services/documentservice.py @@ -499,4 +499,5 @@ def __get_close_datetime(self, request_json): generatedbefore = state['created_at'] if state['status'] == StateName.response.value and isresponsephasecompleted == False: isresponsephasecompleted = True - return generatedbefore if isresponsephasecompleted == True else None \ No newline at end of file + return generatedbefore if isresponsephasecompleted == True else None +