Skip to content

Commit

Permalink
Merge pull request #853 from bcgov/dev
Browse files Browse the repository at this point in the history
Release 9.5.9 changes to Main
  • Loading branch information
divyav-aot authored Mar 8, 2024
2 parents 952ff89 + a975d91 commit 8ece791
Show file tree
Hide file tree
Showing 13 changed files with 214 additions and 145 deletions.
257 changes: 146 additions & 111 deletions MCS.FOI.S3FileConversion/MCS.FOI.DocToPDF/DocFileProcessor.cs
Original file line number Diff line number Diff line change
@@ -1,111 +1,146 @@

using Serilog;
using Syncfusion.DocIO.DLS;
using Syncfusion.DocIORenderer;
using Syncfusion.Pdf;



namespace MCS.FOI.DocToPDF
{
public class DocFileProcessor : IDocFileProcessor, IDisposable
{


public DocFileProcessor() { }

public DocFileProcessor(Stream SourceStream)
{
this.SourceStream = SourceStream;
}

public Stream SourceStream { get; set; }

public int FailureAttemptCount { get; set; }

public int WaitTimeinMilliSeconds { get; set; }

public bool IsSinglePDFOutput { get; set; }


private MemoryStream? output = null;
public (bool, Stream) ConvertToPDF()
{
bool converted = false;
string message = string.Empty;
bool _isSinglePDFOutput = IsSinglePDFOutput;
output = new MemoryStream();
try
{
for (int attempt = 1; attempt <= FailureAttemptCount && !converted; attempt++)
{
try
{
using (WordDocument wordDocument = new WordDocument(SourceStream, Syncfusion.DocIO.FormatType.Automatic))
{

wordDocument.RevisionOptions.CommentDisplayMode = CommentDisplayMode.ShowInBalloons;
wordDocument.RevisionOptions.CommentColor = RevisionColor.Blue;
wordDocument.RevisionOptions.ShowMarkup = RevisionType.Deletions | RevisionType.Insertions;

using (DocIORenderer renderer = new DocIORenderer())
{
using PdfDocument pdfDocument = renderer.ConvertToPDF(wordDocument);
//Save the PDF file
//Close the instance of document objects
pdfDocument.Save(output);
pdfDocument.Close(true);
converted = true;

}

}
}
catch (Exception e)
{
string errorMessage = $"Exception occured while coverting a document file, exception : {e.Message}";
message = $"Exception happened while accessing File, re-attempting count : {attempt} , Error Message : {e.Message} , Stack trace : {e.StackTrace}";
Log.Error(message);
Console.WriteLine(message);
if (attempt == FailureAttemptCount)
{
throw new Exception(errorMessage);
}
Thread.Sleep(WaitTimeinMilliSeconds);
}
}
}
catch (Exception ex)
{
converted = false;
string error = $"Exception occured while coverting Doc file, exception : {ex.Message} , stacktrace : {ex.StackTrace}";
Log.Error(error);
Console.WriteLine(error);
throw;
}
return (converted, output);
}

public void Dispose()
{
Dispose(true);
GC.SuppressFinalize(this);
}

protected virtual void Dispose(bool disposing)
{
if (disposing)
{
if (this.SourceStream != null)
{
this.SourceStream.Close();
this.SourceStream.Dispose();
}

if (output != null) output.Dispose();
// free managed resources
}

}
}
}

using Serilog;
using Syncfusion.DocIO.DLS;
using Syncfusion.DocIO;
using Syncfusion.DocIORenderer;
using Syncfusion.Pdf;
using DocumentFormat.OpenXml.Packaging;
using DocumentFormat.OpenXml.Wordprocessing;



namespace MCS.FOI.DocToPDF
{
public class DocFileProcessor : IDocFileProcessor, IDisposable
{


public DocFileProcessor() { }

public DocFileProcessor(Stream SourceStream)
{
this.SourceStream = SourceStream;
}

public Stream SourceStream { get; set; }

public int FailureAttemptCount { get; set; }

public int WaitTimeinMilliSeconds { get; set; }

public bool IsSinglePDFOutput { get; set; }


private MemoryStream? output = null;
public (bool, Stream) ConvertToPDF()
{
bool converted = false;
string message = string.Empty;
bool _isSinglePDFOutput = IsSinglePDFOutput;
output = new MemoryStream();
try
{
for (int attempt = 1; attempt <= FailureAttemptCount && !converted; attempt++)
{
try
{
using (WordDocument wordDocument = new WordDocument(SourceStream, Syncfusion.DocIO.FormatType.Automatic))
{
SourceStream.Position = 0;

using (var docXML = WordprocessingDocument.Open(SourceStream, false))
{

DocumentFormat.OpenXml.Wordprocessing.Body body = docXML.MainDocumentPart.Document.Body;
List<String> originalDates = new List<String>();
foreach (var textItem in body.Descendants<FieldCode>().Where(textItem => textItem.Text.Contains("DATE")))
{
var datetext = textItem.Parent.NextSibling().NextSibling();
originalDates.Add(datetext.InnerText);
}

List<Entity> datefields = wordDocument.FindAllItemsByProperty(EntityType.Field, "FieldType", FieldType.FieldDate.ToString());
if (datefields != null)
{
foreach (var (datefield, i) in datefields.Select((datefield, i) => (datefield, i)))
{
var dateField = datefield as WField;
//Takes the owner paragraph.
WParagraph ownerPara = dateField.OwnerParagraph;
int dateFieldIndex = ownerPara.ChildEntities.IndexOf(dateField);
//Removes the date field.
ownerPara.ChildEntities.Remove(dateField);
//Creating a new text range with required date.
WTextRange textRange = new WTextRange(ownerPara.Document);
textRange.Text = originalDates[i];//"February 12, 2023";
//Inserting the date field with the created text range.
ownerPara.ChildEntities.Insert(dateFieldIndex, textRange);
}
}
}

wordDocument.RevisionOptions.CommentDisplayMode = CommentDisplayMode.ShowInBalloons;
wordDocument.RevisionOptions.CommentColor = RevisionColor.Blue;
wordDocument.RevisionOptions.ShowMarkup = RevisionType.Deletions | RevisionType.Insertions;

using (DocIORenderer renderer = new DocIORenderer())
{
using PdfDocument pdfDocument = renderer.ConvertToPDF(wordDocument);
//Save the PDF file
//Close the instance of document objects
pdfDocument.Save(output);
pdfDocument.Close(true);
converted = true;

}

}
}
catch (Exception e)
{
string errorMessage = $"Exception occured while coverting a document file, exception : {e.Message}";
message = $"Exception happened while accessing File, re-attempting count : {attempt} , Error Message : {e.Message} , Stack trace : {e.StackTrace}";
Log.Error(message);
Console.WriteLine(message);
if (attempt == FailureAttemptCount)
{
throw new Exception(errorMessage);
}
Thread.Sleep(WaitTimeinMilliSeconds);
}
}
}
catch (Exception ex)
{
converted = false;
string error = $"Exception occured while coverting Doc file, exception : {ex.Message} , stacktrace : {ex.StackTrace}";
Log.Error(error);
Console.WriteLine(error);
throw;
}
return (converted, output);
}

public void Dispose()
{
Dispose(true);
GC.SuppressFinalize(this);
}

protected virtual void Dispose(bool disposing)
{
if (disposing)
{
if (this.SourceStream != null)
{
this.SourceStream.Close();
this.SourceStream.Dispose();
}

if (output != null) output.Dispose();
// free managed resources
}

}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
<!--<PackageReference Include="System.Drawing.Common" Version="5.0.2" />-->
<PackageReference Include="Syncfusion.DocIORenderer.Net.Core" Version="23.2.7" />
<PackageReference Include="Syncfusion.Pdf.Net.Core" Version="23.2.7" />
<PackageReference Include="DocumentFormat.OpenXml" Version="3.0.1" />
</ItemGroup>

</Project>
39 changes: 26 additions & 13 deletions api/reviewer_api/models/AnnotationSections.py
Original file line number Diff line number Diff line change
Expand Up @@ -229,13 +229,9 @@ def __bulknewsections(cls, annots, _pkvannots, redactionlayerid, _foiministryreq
"section": annot["sectionsschema"],
"createdby": userinfo,
"isactive": True,
"version": pkkey["version"] + 1,
"redactionlayerid": redactionlayerid
if pkkey is not None and "version" in pkkey
else 1,
"id": pkkey["id"]
if pkkey is not None and "id" in pkkey
else None,
"redactionlayerid": redactionlayerid,
"version": pkkey["version"] + 1 if pkkey is not None and "version" in pkkey else 1,
"id": pkkey["id"] if pkkey is not None and "id" in pkkey else None
}
)
idxannots.append(annot["name"])
Expand Down Expand Up @@ -401,7 +397,20 @@ def get_by_ministryid(cls, ministryrequestid, redactionlayerid):
@classmethod
def getredactedsectionsbyrequest(cls, ministryrequestid, redactionlayerid):
try:
sql = """select section from public."Sections" where sectionid in
sql = """
select unnest(xpath('//contents/text()', annotation::xml))::text as section
from "Annotations" a
join public."Documents" d on d.documentid = a.documentid and d.foiministryrequestid = :ministryrequestid
join public."DocumentMaster" dm on dm.documentmasterid = d.documentmasterid and dm.ministryrequestid = :ministryrequestid
left join public."DocumentDeleted" dd on dm.filepath ilike dd.filepath || '%' and dd.ministryrequestid = :ministryrequestid
where a.annotation like '%%freetext%%'
and a.redactionlayerid = :redactionlayerid
and (dd.deleted is null or dd.deleted is false)
and a.isactive = true;
"""
"""
sql = select section from public."Sections" where sectionid in
(select distinct (json_array_elements((as1.section::json->>'ids')::json)->>'id')::integer
from public."AnnotationSections" as1
join public."Annotations" a on a.annotationname = as1.annotationname
Expand All @@ -414,13 +423,17 @@ def getredactedsectionsbyrequest(cls, ministryrequestid, redactionlayerid):
and (dd.deleted is null or dd.deleted is false)
and a.isactive = true)
and sectionid != 25
order by sortorder"""
order by sortorder
"""
rs = db.session.execute(text(sql), {"ministryrequestid": ministryrequestid, "redactionlayerid": redactionlayerid})
sectionstring = ""
sections = []
for row in rs:
sectionstring = sectionstring + row["section"] + ", "
sectionstring = sectionstring[:-2]
return sectionstring
sections += [x.strip() for x in row['section'].split(",")]
if len(sections) > 0:
distinctsections = list(set(sections))
distinctsections.sort()
return ", ".join(distinctsections)
return None
except Exception as ex:
logging.error(ex)
finally:
Expand Down
8 changes: 2 additions & 6 deletions api/reviewer_api/models/Annotations.py
Original file line number Diff line number Diff line change
Expand Up @@ -375,12 +375,8 @@ def __bulknewannotations(cls, annots, _pkvannots, redactionlayerid, userinfo):
"createdby": userinfo,
"isactive": True,
"redactionlayerid": redactionlayerid,
"version": pkkey["version"] + 1
if pkkey is not None and "version" in pkkey
else 1,
"annotationid": pkkey["annotationid"]
if pkkey is not None and "annotationid" in pkkey
else None,
"version": pkkey["version"] + 1 if pkkey is not None and "version" in pkkey else 1,
"annotationid": pkkey["annotationid"] if pkkey is not None and "annotationid" in pkkey else None
}
)
idxannots.append(annot["name"])
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,10 +33,7 @@ def start(consumer_id: str, start_from: StartFrom = StartFrom.latest):
print(f"Starting from {start_from.name}")

while True:
print("Reading stream...")
messages = stream.read(last_id=last_id, block=BLOCK_TIME)
print("*********** Messages ***********")
print(messages)
if messages:
for _message in messages:
# message_id is the random id created to identify the message
Expand All @@ -58,4 +55,4 @@ def start(consumer_id: str, start_from: StartFrom = StartFrom.latest):
rdb.set(LAST_ID_KEY.format(consumer_id=consumer_id), last_id)
print(f"finished processing {message_id}")
else:
print(f"No new messages after ID: {last_id}")
logging.info(f"No new messages after ID: {last_id}")
1 change: 1 addition & 0 deletions computingservices/PDFStitchServices/.sampleenv
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ PDFSTITCH_S3_SERVICE=

DIVISION_PDF_STITCH_STREAM_KEY=DIVISION-PDF-STITCH
DIVISION_BLOB_STITCH_STREAM_KEY=DIVISION-PDF-STITCH
DIVISION_STITCH_FOLDER_PATH=Clean/divisionname

ZIPPER_REDIS_HOST=
ZIPPER_REDIS_PASSWORD=
Expand Down
5 changes: 3 additions & 2 deletions computingservices/PDFStitchServices/models/zipperproducer.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

class zipperproducer(object):

def __init__(self,jobid,requestid,category,requestnumber,bcgovcode,createdby,ministryrequestid,filestozip,finaloutput,attributes) -> None:
def __init__(self,jobid,requestid,category,requestnumber,bcgovcode,createdby,ministryrequestid,filestozip,finaloutput,attributes,foldername) -> None:
self.jobid = jobid
self.requestid = requestid
self.category=category
Expand All @@ -12,4 +12,5 @@ def __init__(self,jobid,requestid,category,requestnumber,bcgovcode,createdby,min
self.ministryrequestid = ministryrequestid
self.filestozip = filestozip
self.finaloutput = finaloutput
self.attributes = attributes
self.attributes = attributes
self.foldername = foldername
Loading

0 comments on commit 8ece791

Please sign in to comment.