Merge pull request #495 from bcgov/dev-marshal

webviewer upgrade and other changes
bcgov · Sep 28, 2023 · 32dc882 · 32dc882
2 parents 6c5766c + ffc7ee3
commit 32dc882
Show file tree

Hide file tree

Showing 510 changed files with 8,413 additions and 7,421 deletions.
diff --git a/api/reviewer_api/resources/foiflowmasterdata.py b/api/reviewer_api/resources/foiflowmasterdata.py
@@ -43,6 +43,7 @@
 
 s3host = os.getenv("OSS_S3_HOST")
 s3region = os.getenv("OSS_S3_REGION")
+webviewerlicense = os.getenv("PDFTRON_WEBVIEWER_LICENSE")
 
 imageextensions = [".png", ".jpg", ".jpeg", ".gif"]
 
@@ -256,3 +257,17 @@ def post(ministryrequestid):
             return json.dumps(data), 200
         except BusinessException as exception:
             return {"status": exception.status_code, "message": exception.message}, 500
+
+@cors_preflight("POST,OPTIONS")
+@API.route("/foiflow/webviewerlicense")
+class WebveiwerLicense(Resource):
+    @staticmethod
+    @TRACER.trace()
+    @cross_origin(origins=allowedorigins())
+    @auth.require
+    @auth.hasusertype('iao')
+    def get():
+        try:
+            return json.dumps({"license": webviewerlicense}), 200
+        except BusinessException as exception:
+            return {"status": exception.status_code, "message": exception.message}, 500
diff --git a/computingservices/ZippingServices/services/foirediszipperconsumer.py b/computingservices/ZippingServices/services/foirediszipperconsumer.py
@@ -29,51 +29,54 @@ class StartFrom(str, Enum):
 
 @app.command()
 def start(consumer_id: str, start_from: StartFrom = StartFrom.latest):
-    rdb = redisstreamdb
-    stream = rdb.Stream(STREAM_KEY)
+    try:
+        rdb = redisstreamdb
+        stream = rdb.Stream(STREAM_KEY)
 
-    last_id = rdb.get(LAST_ID_KEY.format(consumer_id=consumer_id))
-    if last_id:
-        print(f"Resume from ID: {last_id}")
-    else:
-        last_id = start_from.value
-        print(f"Starting from {start_from.name}")
+        last_id = rdb.get(LAST_ID_KEY.format(consumer_id=consumer_id))
+        if last_id:
+            print(f"Resume from ID: {last_id}")
+        else:
+            last_id = start_from.value
+            print(f"Starting from {start_from.name}")
 
-    while True:
-        # print("Reading stream...")
-        messages = stream.read(last_id=last_id, block=BLOCK_TIME)
-        if messages:
-            for message_id, message in messages:
-                print(f"processing {message_id}::{message}")
-                if message is not None:
-                    readyfornotification = False
-                    producermessage = None
-                    try:
-                        _message = json.dumps(
-                            {
-                                key.decode("utf-8"): value.decode("utf-8")
-                                for (key, value) in message.items()
-                            }
-                        )
-                        producermessage = jsonmessageparser.getzipperproducermessage(
-                            _message
-                        )
-                        print(producermessage)
-                        processmessage(producermessage)
-                        readyfornotification = True
-                        print(
-                            "Processing is completed for Job ID {0} for category {1}".format(
-                                producermessage.jobid, producermessage.category
+        while True:
+            # print("Reading stream...")
+            messages = stream.read(last_id=last_id, block=BLOCK_TIME)
+            if messages:
+                for message_id, message in messages:
+                    print(f"processing {message_id}::{message}")
+                    if message is not None:
+                        readyfornotification = False
+                        producermessage = None
+                        try:
+                            _message = json.dumps(
+                                {
+                                    key.decode("utf-8"): value.decode("utf-8")
+                                    for (key, value) in message.items()
+                                }
                             )
-                        )
-                    except Exception as error:
-                        print(
-                            "Exception while processing redis message, func start(p1), Error : {0} ".format(
-                                error
+                            producermessage = jsonmessageparser.getzipperproducermessage(
+                                _message
                             )
-                        )
-                    sendnotification(readyfornotification, producermessage)
-                last_id = message_id
-                rdb.set(LAST_ID_KEY.format(consumer_id=consumer_id), last_id)
-        # else:
-        # print(f"No new messages after ID: {last_id}")
+                            print(producermessage)
+                            processmessage(producermessage)
+                            readyfornotification = True
+                            print(
+                                "Processing is completed for Job ID {0} for category {1}".format(
+                                    producermessage.jobid, producermessage.category
+                                )
+                            )
+                            sendnotification(readyfornotification, producermessage)
+                        except Exception as error:
+                            print(
+                                "Exception while processing redis message, func start(p1), Error : {0} ".format(
+                                    str(error)
+                                )
+                            )
+                        finally:
+                            last_id = message_id
+                            rdb.set(LAST_ID_KEY.format(consumer_id=consumer_id), last_id)
+                            stream.delete(message_id)                                        
+    except Exception as ex:
+       print("Exception happened at the top level of zipper service: ", str(ex))
diff --git a/computingservices/ZippingServices/services/zipperservice.py b/computingservices/ZippingServices/services/zipperservice.py
@@ -76,9 +76,8 @@ def processmessage(message):
             isziping=False,
             status="error",
             finalmessage=message,
-            message=traceback.format_exc(),
+            message='Error while processing zipper message - ' + str(ex)
         )
-        raise
 
 
 def sendnotification(readyfornotification, producermessage):

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -62,6 +62,7 @@ services:
       - OSS_S3_HOST=${OSS_S3_HOST}
       - OSS_S3_REGION=${OSS_S3_REGION}
       - OSS_S3_SERVICE=${OSS_S3_SERVICE}
+      - PDFTRON_WEBVIEWER_LICENSE=${PDFTRON_WEBVIEWER_LICENSE}
       - DEDUPE_S3_ENV=${DEDUPE_S3_ENV}
       - DEDUPE_STREAM_KEY=${DEDUPE_STREAM_KEY}
       - FILE_CONVERSION_STREAM_KEY=${FILE_CONVERSION_STREAM_KEY}