Merge pull request #514 from bcgov/dev-marshal

dev-marshal changes to test-marshal
bcgov · Oct 5, 2023 · eebd405 · eebd405
2 parents f0ffb5b + c620170
commit eebd405
Show file tree

Hide file tree

Showing 19 changed files with 381 additions and 109 deletions.
diff --git a/MCS.FOI.S3FileConversion/MCS.FOI.MSGToPDF/MCS.FOI.MSGToPDF.csproj b/MCS.FOI.S3FileConversion/MCS.FOI.MSGToPDF/MCS.FOI.MSGToPDF.csproj
@@ -12,7 +12,7 @@
 		<PackageReference Include="Serilog.Extensions.Logging" Version="3.0.1" />
 		<PackageReference Include="SkiaSharp" Version="2.88.3" />
 		<PackageReference Include="SkiaSharp.NativeAssets.Linux" Version="2.88.3" />
-		<PackageReference Include="MsgReader" Version="5.1.0" />
+		<PackageReference Include="MsgReader" Version="4.5.0" />
 		<PackageReference Include="Syncfusion.DocIO.Net.Core" Version="22.1.37" />
 		<PackageReference Include="Syncfusion.DocIORenderer.Net.Core" Version="22.1.37" />
 		<PackageReference Include="Syncfusion.HtmlToPdfConverter.QtWebKit.Net.Core" Version="20.3.0.56" />		

diff --git a/MCS.FOI.S3FileConversion/MCS.FOI.MSGToPDF/MSGFileProcessor.cs b/MCS.FOI.S3FileConversion/MCS.FOI.MSGToPDF/MSGFileProcessor.cs
@@ -12,6 +12,9 @@
 using System.Collections.Generic;
 using Syncfusion.Pdf.HtmlToPdf;
 using System.ComponentModel.DataAnnotations;
+using System;
+using RtfPipe.Tokens;
+using System.IO;
 
 namespace MCS.FOI.MSGToPDF
 {
@@ -96,45 +99,23 @@ public MSGFileProcessor(Stream sourceStream)
 
                                             filename = Path.GetFileNameWithoutExtension(filename) + '1' + extension;
                                         }
-                                        Console.WriteLine("attachmentname: " + _attachment.FileName);
-                                        Console.WriteLine("attachmentpos: " + _attachment.RenderingPosition);
-                                        Console.WriteLine("attachmentmime: " + extension);
                                         fileNameHash.Add(filename, true);
                                         attachmentInfo.Add("filename", _attachment.FileName);
                                         attachmentInfo.Add("s3filename", filename);
+                                        attachmentInfo.Add("cid", _attachment.ContentId);
                                         attachmentInfo.Add("size", _attachment.Data.Length.ToString());
                                         attachmentInfo.Add("lastmodified", _attachment.LastModificationTime.ToString());
                                         attachmentInfo.Add("created", _attachment.CreationTime.ToString());
                                         attachmentsObj.Add(attachmentStream, attachmentInfo);
                                     }
                                 }
                             }
-
-                            //msg.CharsetDetectionEncodingConfidenceLevel = 0.7f;
                             if (msg.BodyRtf != null)
                             {
-                                //using var fs0 = new FileStream("C:\\folder\\log.txt", FileMode.Create, FileAccess.Write);
                                 var msgReader = new Reader();
 
                                 var inputStream = new MemoryStream();
-                                //SourceStream.CopyTo(inputStream);
-                                //List<MemoryStream> result = msgReader.ExtractToStream(inputStream, true);
-
-
-                                //using var htmlfs = new FileStream("C:\\folder\\test.txt", FileMode.Create, FileAccess.Write);
-                                //result[0].WriteTo(htmlfs);
-                                //string bin = BitConverter.ToString(((Storage.Attachment)msg.Attachments[0]).Data).Replace("-", string.Empty);
-                                //Console.WriteLine(bin);
                                 string body = msgReader.ExtractMsgEmailBody(SourceStream, ReaderHyperLinks.Both, "text/html; charset=utf-8", false);
-                                //var ms = new MemoryStream();
-                                //var success = false;
-                                //(ms, success) = ConvertHTMLtoPDF(body, ms);
-
-                                //using var fs1 = new FileStream("C:\\folder\\blinkconverted.pdf", FileMode.Create, FileAccess.Write);
-                                using var fs2 = new FileStream("C:\\folder\\syncfusionconverted.pdf", FileMode.Create, FileAccess.Write);
-                                //using var fs3 = new FileStream("C:\\folder\\syncfusionconverted.docx", FileMode.Create, FileAccess.Write);
-                                //using var fs4 = new FileStream("C:\\folder\\test.txt", FileMode.Create, FileAccess.Write);
-                                //ms.WriteTo(fs1);
                                 var bodyreplaced = Regex.Replace(Regex.Replace(body.Replace("<br>", "<br/>").Replace("<![if !supportAnnotations]>", "").Replace("<![endif]>", ""), "=(?<tagname>(?!utf-8)[\\w|-]+)", "=\"${tagname}\""), "<meta .*?>", "");
                                 const string rtfInlineObject = "[*[RTFINLINEOBJECT]*]";
                                 const string imgString = "<img";
@@ -148,68 +129,86 @@ public MSGFileProcessor(Stream sourceStream)
                                         if (!attachment.GetType().FullName.ToLower().Contains("message"))
                                         {
                                             var _attachment = (Storage.Attachment)attachment;
-                                            if (htmlInline && !String.IsNullOrEmpty(_attachment.ContentId) && bodyreplaced.Contains(_attachment.ContentId))
+                                            if (htmlInline)
                                             {
-                                                inlineAttachments.Add(_attachment);
-                                                //attachmentStream = new();
-                                                //attachmentStream.Write(_attachment.Data, 0, _attachment.Data.Length);
-                                                //attachmentsObj.Remove(attachmentStream);
+                                                if (!String.IsNullOrEmpty(_attachment.ContentId) && bodyreplaced.Contains(_attachment.ContentId))
+                                                {
+                                                    inlineAttachments.Add(_attachment);
+                                                }
                                             }
                                             else if (rtfInline)
                                             {
                                                 inlineAttachments.Add(_attachment);
-                                                //attachmentStream = new();
-                                                //attachmentStream.Write(_attachment.Data, 0, _attachment.Data.Length);
-                                                //attachmentsObj.Remove(attachmentStream); 
                                             }
                                         }
                                     }
                                     foreach (var inlineAttachment in inlineAttachments.OrderBy(m => m.RenderingPosition))
                                     {
                                         if (htmlInline)
                                         {
-                                            var match = Regex.Match(bodyreplaced, "<img.*cid:" + inlineAttachment.ContentId + ".*?>");
-                                            var width = float.Parse(Regex.Match(match.Value, "(?<=width=\")\\d+").Value);
-                                            var height = float.Parse(Regex.Match(match.Value, "(?<=height=\")\\d+").Value);
-                                            const float maxSize = 750;
-                                            if (width > maxSize)
-                                            {
-                                                float scale = maxSize / width;
-                                                width = maxSize;
-                                                height = scale * height;
-                                            }
-                                            else if (height > maxSize)
+                                            bodyreplaced = Regex.Replace(bodyreplaced, "<img.*cid:" + inlineAttachment.ContentId + ".*?>", "<img src=\"data:" + inlineAttachment.MimeType + ";base64," + Convert.ToBase64String(inlineAttachment.Data) + "\"/>");
+                                            foreach (KeyValuePair<MemoryStream, Dictionary<string, string>> attachment in attachmentsObj)
                                             {
-                                                float scale = maxSize / height;
-                                                height = maxSize;
-                                                width = scale * width;
+                                                if (attachment.Value["cid"] == inlineAttachment.ContentId)
+                                                {
+                                                    attachmentsObj.Remove(attachment.Key);
+                                                }
                                             }
-                                            bodyreplaced = Regex.Replace(bodyreplaced, "<img.*cid:" + inlineAttachment.ContentId + ".*?>", "<img width='" + width.ToString() + "' height ='" + height.ToString() + "' src=\"data:" + inlineAttachment.MimeType + ";base64," + Convert.ToBase64String(inlineAttachment.Data) + "\"/>");
                                         }
                                         else if (rtfInline)
                                         {
-                                            //using var fs5 = new FileStream("C:\\folder\\image.png", FileMode.Create, FileAccess.Write);
-                                            //fs5.Write(inlineAttachment.Data, 0, inlineAttachment.Data.Length);
-                                            bodyreplaced = ReplaceFirstOccurrence(bodyreplaced, rtfInlineObject, "<img src=\"data:image/" + Path.GetExtension(inlineAttachment.FileName) + ";base64," + Convert.ToBase64String(inlineAttachment.Data) + "\"/>");
-                                            //var string1 = Convert.ToBase64String(inlineAttachment.Data);
-                                            //var string2 = BitConverter.ToString(inlineAttachment.Data).Replace("-", string.Empty);
+                                            if (inlineAttachment.OleAttachment)
+                                            {
+                                                bodyreplaced = ReplaceFirstOccurrence(bodyreplaced, rtfInlineObject, "<img src=\"data:image/" + Path.GetExtension(inlineAttachment.FileName) + ";base64," + Convert.ToBase64String(inlineAttachment.Data) + "\"/>");
+                                                foreach (KeyValuePair<MemoryStream, Dictionary<string, string>> attachment in attachmentsObj)
+                                                {
+                                                    if (attachment.Value["filename"] == inlineAttachment.FileName)
+                                                    {
+                                                        attachmentsObj.Remove(attachment.Key);
+                                                    }
+                                                }
+                                            }
+                                            else
+                                            {
+                                                bodyreplaced = ReplaceFirstOccurrence(bodyreplaced, rtfInlineObject, " <b>[**Inline Attachment - " + inlineAttachment.FileName + "**]</b>");
+                                            }
                                         }
                                     }
                                 }
 
-
                                 byte[] byteArray = Encoding.ASCII.GetBytes(bodyreplaced);
                                 using (MemoryStream messageStream = new MemoryStream(byteArray))
-                                {
-                                    //messageStream.WriteTo(fs4);
+                                {                                    
                                     using (WordDocument rtfDoc = new WordDocument(messageStream, Syncfusion.DocIO.FormatType.Html))
                                     {
-                                        //rtfDoc.Save(fs3, FormatType.Docx);
                                         // Replace leading tabs, issue with syncfusion
                                         rtfDoc.ReplaceFirst = true;
                                         var regex = new Regex(@"(\r)*(\n)*(\t)+", RegexOptions.Multiline);
                                         var occurences = rtfDoc.Replace(regex, "\r\n");
 
+                                        List<Entity> pictures = rtfDoc.FindAllItemsByProperty(EntityType.Picture, "", "");
+
+                                        if (pictures != null)
+                                        {
+                                            foreach (WPicture picture in pictures)
+                                            {
+                                                picture.LockAspectRatio = true;
+                                                const float maxSize = 500;
+                                                if (picture.Height > maxSize && picture.Height >= picture.Width)
+                                                {
+                                                    var scale = (maxSize / picture.Height) * 100;
+                                                    picture.HeightScale = scale;
+                                                    picture.WidthScale = scale;
+                                                }
+                                                if (picture.Width > maxSize)
+                                                {
+                                                    var scale = (maxSize / picture.Width) * 100;
+                                                    picture.HeightScale = scale;
+                                                    picture.WidthScale = scale;
+                                                }
+                                            }
+                                        }
+
                                         //Gets all the hyperlink fields in the document
 
                                         List<Entity> fields = rtfDoc.FindAllItemsByProperty(EntityType.Field, "FieldType", FieldType.FieldHyperlink.ToString());
@@ -311,6 +310,8 @@ public MSGFileProcessor(Stream sourceStream)
                             //bool isConverted;
                             //(output, isConverted) = ConvertHTMLtoPDF(htmlString, output);
 
+
+
 
                             break;
                         }

diff --git a/api/dockerfile b/api/dockerfile
@@ -2,7 +2,7 @@
 # FROM python:3.8
 
 # Necessary to pull images from bcgov and not hit Dockerhub quotas.
-FROM artifacts.developer.gov.bc.ca/docker-remote/python:3.8.5-buster
+FROM artifacts.developer.gov.bc.ca/docker-remote/python:3.10.8-buster
 EXPOSE 6402
 
 # Keeps Python from generating .pyc files in the container

diff --git a/api/dockerfile.local b/api/dockerfile.local
@@ -1,5 +1,5 @@
 # For more information, please refer to https://aka.ms/vscode-docker-python
-FROM python:3.8
+FROM python:3.10.8
 EXPOSE 6402
 
 # Keeps Python from generating .pyc files in the container

diff --git a/api/migrations/versions/13766f2831b6_.py b/api/migrations/versions/13766f2831b6_.py
@@ -0,0 +1,31 @@
+"""empty message
+
+Revision ID: 13766f2831b6
+Revises: 36e7b771fbb5
+Create Date: 2023-10-03 13:53:19.344756
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "13766f2831b6"
+down_revision = "36e7b771fbb5"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.execute(
+        'ALTER TABLE public."Annotations" DROP CONSTRAINT "annotation_version_key";'
+    )
+    op.execute(
+        'ALTER TABLE public."Annotations" ADD CONSTRAINT annotation_version_redactionlayerid_key UNIQUE (annotationname, version, redactionlayerid);'
+    )
+
+
+def downgrade():
+    op.execute(
+        'ALTER TABLE public."Annotations" DROP CONSTRAINT "annotation_version_redactionlayerid_key";'
+    )
diff --git a/api/requirements.txt b/api/requirements.txt
diff --git a/api/reviewer_api/models/Annotations.py b/api/reviewer_api/models/Annotations.py
@@ -322,7 +322,11 @@ def __newannotation(cls, annot, redactionlayerid, userinfo) -> DefaultMethodResu
             ]
             insertstmt = insert(Annotation).values(values)
             upsertstmt = insertstmt.on_conflict_do_update(
-                index_elements=[Annotation.annotationname, Annotation.version],
+                index_elements=[
+                    Annotation.annotationname,
+                    Annotation.version,
+                    Annotation.redactionlayerid,
+                ],
                 set_={
                     "isactive": False,
                     "updatedby": userinfo,

diff --git a/api/reviewer_api/resources/document.py b/api/reviewer_api/resources/document.py
@@ -99,7 +99,7 @@ def get(requestid):
             jsonObj = response.json()
 
             result = documentservice().getdocuments(requestid)
-            return json.dumps({"requeststatusid": jsonObj["requeststatusid"], "documents": result}), 200
+            return json.dumps({"requeststatusid": jsonObj["requeststatusid"], "documents": result,"requestnumber":jsonObj["axisRequestId"]}), 200
         except KeyError as err:
             return {'status': False, 'message':err.messages}, 400
         except BusinessException as exception:

diff --git a/api/reviewer_api/services/radactionservice.py b/api/reviewer_api/services/radactionservice.py
@@ -48,15 +48,22 @@ def saveannotation(self, annotationschema, userinfo):
         result = annotationservice().saveannotation(annotationschema, userinfo)
         if (
             result.success == True
-            and "foiministryrequestid" in annotationschema
             and "pageflags" in annotationschema
             and annotationschema["pageflags"] is not None
         ):
-            documentpageflagservice().bulksavepageflag(
-                annotationschema["foiministryrequestid"],
-                annotationschema["pageflags"],
-                userinfo,
-            )
+            foiministryrequestid = None
+            if "foiministryrequestid" in annotationschema:
+                foiministryrequestid = annotationschema["foiministryrequestid"]
+            elif "sections" in annotationschema:
+                foiministryrequestid = annotationschema["sections"][
+                    "foiministryrequestid"
+                ]
+            if foiministryrequestid:
+                documentpageflagservice().bulksavepageflag(
+                    foiministryrequestid,
+                    annotationschema["pageflags"],
+                    userinfo,
+                )
         return result
 
     def deactivateannotation(

diff --git a/computingservices/DedupeServices/DockerFile.bcgov b/computingservices/DedupeServices/DockerFile.bcgov
@@ -1,4 +1,4 @@
-FROM artifacts.developer.gov.bc.ca/docker-remote/python:3.10-slim
+FROM artifacts.developer.gov.bc.ca/docker-remote/python:3.10.8-buster
 
 # Keeps Python from generating .pyc files in the container
 ENV PYTHONDONTWRITEBYTECODE=1

diff --git a/computingservices/DedupeServices/Dockerfile.local b/computingservices/DedupeServices/Dockerfile.local
@@ -1,4 +1,4 @@
-FROM python:3.10-slim
+FROM python:3.10.8
 
 # Keeps Python from generating .pyc files in the container
 ENV PYTHONDONTWRITEBYTECODE=1

diff --git a/computingservices/PDFStitchServices/Dockerfile.local b/computingservices/PDFStitchServices/Dockerfile.local
@@ -1,4 +1,4 @@
-FROM python:3.10-slim
+FROM python:3.10.8
 
 # Keeps Python from generating .pyc files in the container
 ENV PYTHONDONTWRITEBYTECODE=1

diff --git a/computingservices/ZippingServices/Dockerfile.local b/computingservices/ZippingServices/Dockerfile.local
@@ -1,4 +1,4 @@
-FROM python:3.10-slim
+FROM python:3.10.8
 
 # Keeps Python from generating .pyc files in the container
 ENV PYTHONDONTWRITEBYTECODE=1

diff --git a/web/src/actions/actionConstants.ts b/web/src/actions/actionConstants.ts
@@ -16,7 +16,8 @@ const ACTION_CONSTANTS = {
   SET_REQUEST_STATUS: "SET_REQUEST_STATUS",
   SET_REDACTION_LAYERS: "SET_REDACTION_LAYERS",
   SET_CURRENT_LAYER: "SET_CURRENT_LAYER",
-  INC_REDACTION_LAYER: "INC_REDACTION_LAYER"
+  INC_REDACTION_LAYER: "INC_REDACTION_LAYER",
+  SET_REQUEST_NUMBER:"SET_REQUEST_NUMBER"
 };
 
 export default ACTION_CONSTANTS;
diff --git a/web/src/actions/documentActions.ts b/web/src/actions/documentActions.ts
@@ -51,6 +51,13 @@ export const setRequestStatus = (data: any) => (dispatch:any) =>{
     })
 }
 
+export const setRequestNumber = (data: any) => (dispatch:any) =>{
+    dispatch({
+        type:ACTION_CONSTANTS.SET_REQUEST_NUMBER,
+        payload:data
+    })
+}
+
 export const setRedactionLayers = (data: any) => (dispatch:any) =>{
     dispatch({
         type:ACTION_CONSTANTS.SET_REDACTION_LAYERS,