Merge pull request #1085 from bcgov/test-marshal-AH-FOIMOD-3385-testrook

Test-Rook <> Test-Marshal Merge August 12 2024

.sonarcloud.properties

@@ -1,7 +1,7 @@
 sonar.projectName=foi-docreviewer
 
 # Path to sources
-sonar.sources=api,web/src,computingservices/DedupeServices,computingservices/PDFStitchServices,computingservices/ZippingServices,MCS.FOI.S3FileConversion/MCS.FOI.S3FileConversion
+#sonar.sources=api,web/src,computingservices/DedupeServices,computingservices/PDFStitchServices,computingservices/ZippingServices,MCS.FOI.S3FileConversion/MCS.FOI.S3FileConversion
 sonar.exclusions=**/*.test.js,**/*.test.tsx,api/migrations/versions,api/tests
 #sonar.inclusions=
 

MCS.FOI.S3FileConversion/Dockerfile.local

@@ -13,6 +13,7 @@ COPY MCS.FOI.CalendarToPDFUnitTests/*.csproj ./MCS.FOI.CalendarToPDFUnitTests/
 COPY MCS.FOI.MSGToPDFUnitTests/*.csproj ./MCS.FOI.MSGToPDFUnitTests/
 COPY MCS.FOI.DocToPDFUnitTests/*.csproj ./MCS.FOI.DocToPDFUnitTests/
 COPY MCS.FOI.DocToPDF/*.csproj ./MCS.FOI.DocToPDF/
+COPY MCS.FOI.PPTToPDFUnitTests/*.csproj ./MCS.FOI.PPTToPDFUnitTests/
 
 # COPY NuGet.Config /
 # RUN ["cp", "/NuGet.Config", "/root/.nuget/NuGet/NuGet.Config"]

MCS.FOI.S3FileConversion/MCS.FOI.CalendarToPDF/MCS.FOI.CalendarToPDF.csproj

@@ -11,7 +11,7 @@
 		<PackageReference Include="SkiaSharp" Version="2.88.6" />
 		<PackageReference Include="SkiaSharp.NativeAssets.Linux" Version="2.88.6" />
 		<PackageReference Include="Syncfusion.HtmlToPdfConverter.QtWebKit.Net.Core" Version="19.1.0.63" />
-		<PackageReference Include="Syncfusion.Pdf.Net.Core" Version="23.2.7" />
+		<PackageReference Include="Syncfusion.Pdf.Net.Core" Version="25.2.7" />
 	</ItemGroup>
 
 </Project>

MCS.FOI.S3FileConversion/MCS.FOI.CalendarToPDFUnitTests/CalendarFileProcessorTest.cs

@@ -39,7 +39,7 @@ public void ProcessSimpleCalendarFilesTest()
         {
             bool isProcessed;
             string message = string.Empty;
-            Dictionary<MemoryStream, string> attachments = new Dictionary<MemoryStream, string>();
+            Dictionary<MemoryStream, Dictionary<string,string>> attachments = new Dictionary<MemoryStream, Dictionary<string, string>>();
             string rootFolder = getSourceFolder();
             Stream output = new MemoryStream();
             Stream testFile = new FileStream(Path.Combine(getSourceFolder(), "test-cal.ics"), FileMode.Open, FileAccess.Read);
@@ -57,7 +57,7 @@ public void ProcessCalendarFileWithAttachmentsTest()
         {
             bool isProcessed;
             string message = string.Empty;
-            Dictionary<MemoryStream, string> attachments = new Dictionary<MemoryStream, string>();
+            Dictionary<MemoryStream, Dictionary<string, string>> attachments = new Dictionary<MemoryStream, Dictionary<string, string>>();
             string rootFolder = getSourceFolder();
             Stream output = new MemoryStream();
             Stream testFile = new FileStream(Path.Combine(getSourceFolder(), "test-with-attachments.ics"), FileMode.Open, FileAccess.Read);
@@ -78,7 +78,7 @@ public void ProcessComplexCalendarFilesTest()
         {
             bool isProcessed;
             string message = string.Empty;
-            Dictionary<MemoryStream, string> attachments = new Dictionary<MemoryStream, string>();
+            Dictionary<MemoryStream, Dictionary<string, string>> attachments = new Dictionary<MemoryStream, Dictionary<string, string>>();
             string rootFolder = getSourceFolder();
             Stream output = new MemoryStream();
             Stream testFile = new FileStream(Path.Combine(getSourceFolder(), "test-problematic-calendar.ics"), FileMode.Open, FileAccess.Read);

MCS.FOI.S3FileConversion/MCS.FOI.DocToPDF/DocFileProcessor.cs

@@ -62,7 +62,7 @@ public DocFileProcessor(Stream SourceStream)
                                     }
 
                                     List<Entity> datefields = wordDocument.FindAllItemsByProperty(EntityType.Field, "FieldType", FieldType.FieldDate.ToString());
-                                    if (datefields != null)
+                                    if (datefields != null && originalDates?.Count > 0 && datefields?.Count == originalDates?.Count)
                                     {
                                         foreach (var (datefield, i) in datefields.Select((datefield, i) => (datefield, i)))
                                         {
@@ -86,6 +86,9 @@ public DocFileProcessor(Stream SourceStream)
                             wordDocument.RevisionOptions.CommentColor = RevisionColor.Blue;
                             wordDocument.RevisionOptions.ShowMarkup = RevisionType.Deletions | RevisionType.Insertions;
 
+
+
+
                             using (DocIORenderer renderer = new DocIORenderer())
                             {
                                 using PdfDocument pdfDocument = renderer.ConvertToPDF(wordDocument);

MCS.FOI.S3FileConversion/MCS.FOI.DocToPDF/MCS.FOI.DocToPDF.csproj

@@ -7,16 +7,16 @@
   </PropertyGroup>
 
 	<ItemGroup>
-		<PackageReference Include="Syncfusion.DocIO.Net.Core" Version="23.2.7" />
+		<PackageReference Include="Syncfusion.DocIO.Net.Core" Version="25.2.7" />
 		<PackageReference Include="SkiaSharp" Version="2.88.6" />
 		<PackageReference Include="SkiaSharp.NativeAssets.Linux" Version="2.88.6" />
 
 		<PackageReference Include="Serilog" Version="2.10.0" />
 		<PackageReference Include="Serilog.Sinks.File" Version="4.1.0" />
 		<PackageReference Include="Serilog.Extensions.Logging" Version="3.0.1" />
 		<!--<PackageReference Include="System.Drawing.Common" Version="5.0.2" />-->
-		<PackageReference Include="Syncfusion.DocIORenderer.Net.Core" Version="23.2.7" />
-		<PackageReference Include="Syncfusion.Pdf.Net.Core" Version="23.2.7" />
+		<PackageReference Include="Syncfusion.DocIORenderer.Net.Core" Version="25.2.7" />
+		<PackageReference Include="Syncfusion.Pdf.Net.Core" Version="25.2.7" />
 		<PackageReference Include="DocumentFormat.OpenXml" Version="3.0.1" />
 	</ItemGroup>
 

MCS.FOI.S3FileConversion/MCS.FOI.DocToPDFUnitTests/DocToPDFTests.cs

@@ -28,19 +28,36 @@ public void DOCConvertToPDFTest()
             bool converted;
             string message = string.Empty;
             Stream output = new MemoryStream();
-            Stream testFile = new FileStream(Path.Combine(getDOCRootFolder(), "simple-test-doc.docx"), FileMode.Open, FileAccess.Read);
+            Stream testFile = new FileStream(Path.Combine(getDOCRootFolder(), "Ministers Housing Weekly 2023 12 19 DRAFT - TEST.docx"), FileMode.Open, FileAccess.Read);
             DocFileProcessor DocFileProcessor = new DocFileProcessor();
 
             DocFileProcessor docFileProcessor = new DocFileProcessor(testFile);
             docFileProcessor.IsSinglePDFOutput = false;
             docFileProcessor.WaitTimeinMilliSeconds = 5000;
             docFileProcessor.FailureAttemptCount = 10;
-            (converted, output) = docFileProcessor.ConvertToPDF(); 
+            (converted, output) = docFileProcessor.ConvertToPDF();
+
+
+            SaveStreamAsFile(getDOCRootFolder(), output, "result.pdf");
 
             Assert.IsTrue(converted == true, $"DOC to PDF Conversion failed for {testFile}");
             Assert.IsTrue(output.Length > 0, $"Conversion failed: output file size is zero");
         }
 
+        public static void SaveStreamAsFile(string filePath, Stream stream, string fileName)
+        {
+            stream.Position = 0;
+             var path = Path.Combine(filePath, fileName);
+            var bytesInStream = new byte[stream.Length];
+
+            stream.Read(bytesInStream, 0, (int)bytesInStream.Length);
+
+            using (var outputFileStream = new FileStream(path, FileMode.Create))
+            {
+                outputFileStream.Write(bytesInStream, 0, bytesInStream.Length);
+            }
+        }
+
 
         [TestMethod]
         public void ProblematicDOCXConvertToPDFTest()
@@ -58,6 +75,8 @@ public void ProblematicDOCXConvertToPDFTest()
 
             (converted, output) = DocFileProcessor.ConvertToPDF();
 
+            SaveStreamAsFile(getDOCRootFolder(), output, "result.pdf");
+
             Assert.IsTrue(converted == true, $"Doc to PDF Conversion failed for {testFile}");
             Assert.IsTrue(output.Length > 0, $"Conversion failed: output file size is zero");
         }
@@ -66,7 +85,7 @@ public void ProblematicDOCXConvertToPDFTest()
 
         private string getDOCRootFolder()
         {
-            return "C:\\Projects\\foi-docreviewer\\MCS.FOI.S3FileConversion\\MCS.FOI.DocToPDFUnitTests\\SourceFiles";
+            return "C:\\AOT\\FOI\\Source\\foi-docreviewer\\foi-docreviewer\\MCS.FOI.S3FileConversion\\MCS.FOI.DocToPDFUnitTests\\SourceFiles";
         }
     }
 }

MCS.FOI.S3FileConversion/MCS.FOI.EMLToPDF/MCS.FOI.EMLToPDF.csproj

@@ -12,7 +12,7 @@
 		<PackageReference Include="Serilog.Extensions.Logging" Version="3.0.1" />
 		<PackageReference Include="SkiaSharp" Version="2.88.6" />
 		<PackageReference Include="SkiaSharp.NativeAssets.Linux" Version="2.88.6" />
-		<PackageReference Include="Syncfusion.Pdf.Net.Core" Version="23.1.40" />
+		<PackageReference Include="Syncfusion.Pdf.Net.Core" Version="25.2.7" />
 		<PackageReference Include="MsgReader" Version="4.2.1" />
 		<PackageReference Include="Syncfusion.HtmlToPdfConverter.QtWebKit.Net.Core" Version="19.3.0.43" />
 		<PackageReference Include="System.Drawing.Common" Version="7.0.0" />

MCS.FOI.S3FileConversion/MCS.FOI.ExcelToPDF/MCS.FOI.ExcelToPDF.csproj

@@ -6,9 +6,9 @@
     <Nullable>enable</Nullable>
   </PropertyGroup>
 	<ItemGroup>
-		<PackageReference Include="Syncfusion.XlsIO.Net.Core" Version="23.2.7" />
-		<PackageReference Include="Syncfusion.XlsIORenderer.Net.Core" Version="23.2.7" />
-		<PackageReference Include="Syncfusion.Pdf.Net.Core" Version="23.2.7" />
+		<PackageReference Include="Syncfusion.XlsIO.Net.Core" Version="25.2.7" />
+		<PackageReference Include="Syncfusion.XlsIORenderer.Net.Core" Version="25.2.7" />
+		<PackageReference Include="Syncfusion.Pdf.Net.Core" Version="25.2.7" />
 		<PackageReference Include="SkiaSharp" Version="2.88.6" />
 		<PackageReference Include="SkiaSharp.NativeAssets.Linux" Version="2.88.6" />
 

MCS.FOI.S3FileConversion/MCS.FOI.MSGToPDF/MCS.FOI.MSGToPDF.csproj

@@ -14,7 +14,7 @@
 		<PackageReference Include="SkiaSharp.NativeAssets.Linux" Version="2.88.3" />
 		<PackageReference Include="MsgReader" Version="4.5.0" />
 		<PackageReference Include="Syncfusion.HtmlToPdfConverter.QtWebKit.Net.Core" Version="20.3.0.56" />
-		<PackageReference Include="Syncfusion.Pdf.Net.Core" Version="23.2.7" />
+		<PackageReference Include="Syncfusion.Pdf.Net.Core" Version="25.2.7" />
 		<PackageReference Include="System.Text.Encoding.CodePages" Version="5.0.0" />
 	</ItemGroup>
 

MCS.FOI.S3FileConversion/MCS.FOI.MSGToPDF/MSGFileProcessor.cs

@@ -1,11 +1,12 @@
 using MsgReader.Outlook;
 using MsgReader;
-using Serilog;
+using SerilogNS = Serilog;
 using Syncfusion.HtmlConverter;
 using Syncfusion.Pdf;
 using System.Text;
 using System.Text.RegularExpressions;
 using System.Linq;
+using static MsgReader.Outlook.Storage;
 
 namespace MCS.FOI.MSGToPDF
 {
@@ -77,10 +78,20 @@ public MSGFileProcessor(Stream sourceStream)
                                             filename = newFilename;
                                         }
                                         fileNameHash.Add(filename, true);
+
+                                        var lastModified = _attachment.LastModificationTime.ToString();
+                                        var sentOn = _attachment.SentOn.ToString();
+                                        if (!string.IsNullOrEmpty(sentOn))
+                                            lastModified = sentOn;
+
+                                        var attachmentSize = attachmentStream.Length.ToString();
+                                        if (string.IsNullOrEmpty(attachmentSize))
+                                            attachmentSize = attachmentStream.Capacity.ToString();
+
                                         attachmentInfo.Add("filename", _attachment.FileName);
                                         attachmentInfo.Add("s3filename", filename);
-                                        attachmentInfo.Add("size", attachmentStream.Capacity.ToString());
-                                        attachmentInfo.Add("lastmodified", _attachment.LastModificationTime.ToString());
+                                        attachmentInfo.Add("size", attachmentSize);
+                                        attachmentInfo.Add("lastmodified", lastModified);
                                         attachmentInfo.Add("created", _attachment.CreationTime.ToString());
                                         attachmentsObj.Add(attachmentStream, attachmentInfo);
                                     }
@@ -160,8 +171,22 @@ public MSGFileProcessor(Stream sourceStream)
                                     }
                                 }
                                 var startAt = 0;
-                                foreach (var inlineAttachment in inlineAttachments.OrderBy(m => m.GetType().GetProperty("RenderingPosition").GetValue(m, null)))
+                                foreach (var inlineAttachment in inlineAttachments.OrderBy(m =>
                                 {
+                                    int pos = (int) m.GetType().GetProperty("RenderingPosition").GetValue(m, null);
+                                    if (pos > -1)
+                                    {
+                                        return pos;
+                                    }
+                                    else
+                                    {
+                                        var _inlineAttachment = (Storage.Attachment)m;
+                                        Regex regex = new Regex(@"<img(.|\\n)*cid:" + _inlineAttachment.ContentId + "(.|\\n)*?>");
+                                        Match match = regex.Match(bodyreplaced, startAt);
+                                        return match.Index;
+                                    }
+                                }))
+                                {                                    
                                     if (rtfInline)
                                     {
                                         if (!inlineAttachment.GetType().FullName.ToLower().Contains("message"))
@@ -192,7 +217,7 @@ public MSGFileProcessor(Stream sourceStream)
                                     else if (htmlInline)
                                     {
                                         var _inlineAttachment = (Storage.Attachment)inlineAttachment;
-                                        Regex regex = new Regex("<img(.|\\n)*cid:" + _inlineAttachment.ContentId + "(.|\\n)*?>");
+                                        Regex regex = new Regex(@"<img(.|\\n)*cid:" + _inlineAttachment.ContentId + "(.|\\n)*?>");
                                         Match match = regex.Match(bodyreplaced, startAt);
                                         if (match.Success)
                                         {
@@ -225,7 +250,7 @@ public MSGFileProcessor(Stream sourceStream)
                                                 heightString = " height =\"" + height + "\"";
                                             }
                                             string imgReplacementString = "<img "+ widthString + heightString + " style =\"margin: 1px;\" src=\"data:" + _inlineAttachment.MimeType + ";base64," + Convert.ToBase64String(_inlineAttachment.Data) + "\"/>";
-                                            bodyreplaced = regex.Replace(bodyreplaced, imgReplacementString, 1, startAt);
+                                            bodyreplaced = regex.Replace(bodyreplaced, imgReplacementString, Int32.MaxValue, startAt);
                                             startAt = match.Index + imgReplacementString.Length;
                                         }
                                         foreach (KeyValuePair<MemoryStream, Dictionary<string, string>> attachment in attachmentsObj)
@@ -407,7 +432,7 @@ public MSGFileProcessor(Stream sourceStream)
                         {
                             string errorMessage = $"Exception occured while coverting a message file, exception :  {e.Message}";
                             message = $"Exception happened while accessing the File, re-attempting count : {attempt} , Error Message : {e.Message} , Stack trace : {e.StackTrace}";
-                            Log.Error(message);
+                            SerilogNS.Log.Error(message);
                             Console.WriteLine(message);
                             if (attempt == FailureAttemptCount)
                             {
@@ -420,12 +445,12 @@ public MSGFileProcessor(Stream sourceStream)
                 else
                 {
                     message = $"SourceStream does not exist!";
-                    Log.Error(message);
+                    SerilogNS.Log.Error(message);
                 }
             }
             catch (Exception ex)
             {
-                Log.Error($"Error happened while moving attachments on MSG File, Exception message : {ex.Message} , details : {ex.StackTrace}");
+                SerilogNS.Log.Error($"Error happened while moving attachments on MSG File, Exception message : {ex.Message} , details : {ex.StackTrace}");
                 message = $"Error happened while moving attachments on MSG File, Exception message : {ex.Message} , details : {ex.StackTrace}";
                 moved = false;
                 throw;
@@ -460,20 +485,31 @@ private string GenerateHtmlfromMsg(Storage.Message msg)
                             <td>" + sender + "</td></tr>");
                 //Recipient Name and Email
                 string recipientName = "";
-                foreach (var recipient in msg.GetEmailRecipients(RecipientType.To, false, false))
+                msg.Recipients?.Where(t => t.Type == RecipientType.To).ToList()?.ForEach(toreceipient =>
                 {
-                    recipientName += recipient;
-                }
+
+                    recipientName += String.Format("{0}, {1}", toreceipient.Email, toreceipient.DisplayName);
+
+                });
+
+
                 string recipientCCName = "";
-                foreach (var recipient in msg.GetEmailRecipients(RecipientType.Cc, false, false))
+                msg.Recipients?.Where(t => t.Type == RecipientType.Cc).ToList()?.ForEach(ccreceipient =>
                 {
-                    recipientCCName += recipient;
-                }
+
+                    recipientCCName += String.Format("{0}, {1}", ccreceipient.Email, ccreceipient.DisplayName);
+
+                });
+
                 string recipientBccName = "";
-                foreach (var recipient in msg.GetEmailRecipients(RecipientType.Bcc, false, false))
+
+                msg.Recipients?.Where(t => t.Type == RecipientType.Bcc).ToList()?.ForEach(bccreceipient =>
                 {
-                    recipientBccName += recipient;
-                }
+
+                    recipientBccName += String.Format("{0}, {1}", bccreceipient.Email, bccreceipient.DisplayName);
+
+                });
+
                 if (!string.IsNullOrEmpty(recipientName))
                 {
                     htmlString.Append(@"<tr>

MCS.FOI.S3FileConversion/MCS.FOI.MSGToPDFUnitTests/MCS - Backup.FOI.MSGToPDFUnitTests.csproj

@@ -0,0 +1,31 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+	<ItemGroup>
+		<Compile Remove="Input\**" />
+		<EmbeddedResource Remove="Input\**" />
+		<None Remove="Input\**" />
+	</ItemGroup>
+
+	<ItemGroup>
+		<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.7.1" />
+		<PackageReference Include="MSTest.TestAdapter" Version="2.1.1" />
+		<PackageReference Include="MSTest.TestFramework" Version="2.1.1" />
+		<PackageReference Include="coverlet.collector" Version="1.3.0" />
+		<PackageReference Include="Syncfusion.HtmlToPdfConverter.Net.Windows" Version="23.1.40" />
+	</ItemGroup>
+
+	<ItemGroup>
+		<ProjectReference Include="..\MCS.FOI.MSGToPDF\MCS.FOI.MSGToPDF.csproj" />
+	</ItemGroup>
+
+	<ItemGroup>
+		<Folder Include="Properties\" />
+	</ItemGroup>
+
+</Project>