Merge pull request #1092 from bcgov/dev

Merge CFD features from DEV to TEST env

api/reviewer_api/models/DocumentMaster.py

@@ -48,7 +48,7 @@ def getdocumentmaster(cls, ministryrequestid):
                     and dm.documentmasterid not in (select distinct d.documentmasterid
                         from "DocumentMaster" d , "DocumentDeleted" dd where  d.filepath like dd.filepath||'%'
                         and d.ministryrequestid = dd.ministryrequestid and d.ministryrequestid =:ministryrequestid)
-                    order by da.attributes->>'lastmodified' DESC"""
+                    order by da.attributes->>'lastmodified' DESC, da.attributeid ASC"""
             rs = db.session.execute(text(sql), {'ministryrequestid': ministryrequestid})
             for row in rs:
                 # if row["documentmasterid"] not in deleted:

api/reviewer_api/resources/document.py

@@ -22,7 +22,7 @@
 from reviewer_api.tracer import Tracer
 from reviewer_api.utils.util import  cors_preflight, allowedorigins, getrequiredmemberships
 from reviewer_api.exceptions import BusinessException
-from reviewer_api.schemas.document import FOIRequestDeleteRecordsSchema, FOIRequestUpdateRecordsSchema, DocumentDeletedPage
+from reviewer_api.schemas.document import FOIRequestDeleteRecordsSchema, FOIRequestUpdateRecordsSchema, DocumentDeletedPage, FOIRequestUpdateRecordPersonalAttributesSchema
 import json
 import requests
 import logging
@@ -70,6 +70,7 @@ class UpdateDocumentAttributes(Resource):
     def post():
         try:
             payload = request.get_json()
+            # print("payload: ", payload)
             payload = FOIRequestUpdateRecordsSchema().load(payload)
             result = documentservice().updatedocumentattributes(payload, AuthHelper.getuserid())
             return {'status': result.success, 'message':result.message,'id':result.identifier} , 200
@@ -78,6 +79,28 @@ def post():
         except BusinessException as exception:
             return {'status': exception.status_code, 'message':exception.message}, 500
 
+@cors_preflight('POST,OPTIONS')
+@API.route('/document/update/personal')
+class UpdateDocumentPersonalAttributes(Resource):
+    """Add document to deleted list.
+    """
+    @staticmethod
+    @TRACER.trace()
+    @cross_origin(origins=allowedorigins())
+    @auth.require
+    @auth.ismemberofgroups(getrequiredmemberships())
+    def post():
+        try:
+            payload = request.get_json()
+            # print("payload personal: ", payload)
+            payload = FOIRequestUpdateRecordPersonalAttributesSchema().load(payload)
+            result = documentservice().updatedocumentpersonalattributes(payload, AuthHelper.getuserid())
+            return {'status': result.success, 'message':result.message,'id':result.identifier} , 200
+        except KeyError as error:
+            return {'status': False, 'message': CUSTOM_KEYERROR_MESSAGE + str(error)}, 400
+        except BusinessException as exception:
+            return {'status': exception.status_code, 'message':exception.message}, 500
+
 @cors_preflight('GET,OPTIONS')
 @API.route('/document/<requestid>')
 class GetDocuments(Resource):

api/reviewer_api/resources/foiflowmasterdata.py

@@ -38,17 +38,23 @@
 from reviewer_api.services.documentservice import documentservice
 from reviewer_api.utils.constants import FILE_CONVERSION_FILE_TYPES
 
+import requests
+import logging
+
 API = Namespace(
     "FOI Flow Master Data", description="Endpoints for FOI Flow master data"
 )
 TRACER = Tracer.get_instance()
+CUSTOM_KEYERROR_MESSAGE = "Key error has occured: "
 
 s3host = os.getenv("OSS_S3_HOST")
 s3region = os.getenv("OSS_S3_REGION")
 webviewerlicense = os.getenv("PDFTRON_WEBVIEWER_LICENSE")
 
 imageextensions = [".png", ".jpg", ".jpeg", ".gif"]
 
+requestapiurl = os.getenv("FOI_REQ_MANAGEMENT_API_URL")
+requestapitimeout = os.getenv("FOI_REQ_MANAGEMENT_API_TIMEOUT")
 
 @cors_preflight("GET,OPTIONS")
 @API.route("/foiflow/oss/presigned/<documentid>")
@@ -166,6 +172,8 @@ class FOIFlowS3PresignedRedline(Resource):
     def post(ministryrequestid, redactionlayer="redline", layertype="redline"):
         try:            
             data = request.get_json()
+            # print("data!:",data)
+            requesttype = data["requestType"]
             documentmapper = redactionservice().getdocumentmapper(
                 data["divdocumentList"][0]["documentlist"][0]["filepath"].split("/")[3]
             )
@@ -192,11 +200,16 @@ def post(ministryrequestid, redactionlayer="redline", layertype="redline"):
             packagetype = "redline"
             if redactionlayer == "oipc":
                 packagetype = "oipcreview" if layertype == "oipcreview" else "oipcredline"
-
+
+            #check if is single redline package
+            is_single_redline = is_single_redline_package(_bcgovcode, packagetype, requesttype)
+            # print("is_single_redline:",is_single_redline)
+            #print("divdocumentList:",data["divdocumentList"])
             for div in data["divdocumentList"]:
                 if len(div["documentlist"]) > 0:
+                    # print("filepathlist:" , div["documentlist"][0]["filepath"])
                     filepathlist = div["documentlist"][0]["filepath"].split("/")[4:]
-                    if is_single_redline_package(_bcgovcode, packagetype) == False:
+                    if is_single_redline == False:
                         division_name = div["divisionname"]
                         # generate save url for stitched file
                         filepath_put = "{0}/{2}/{1}/{0} - {2} - {1}.pdf".format(
@@ -254,14 +267,17 @@ def post(ministryrequestid, redactionlayer="redline", layertype="redline"):
                                     )
                 elif len(div["incompatableList"]) > 0:
                     filepathlist = div["incompatableList"][0]["filepath"].split("/")[4:]
-                if is_single_redline_package(_bcgovcode, packagetype) and singlepkgpath is None :
+                if is_single_redline and singlepkgpath is None :
                     if len(div["documentlist"]) > 0 or len(div["incompatableList"]) > 0:
                         div = data["divdocumentList"][0] 
                         filepathlist = div["documentlist"][0]["filepath"].split("/")[4:]
+                        # print("filepathlist:",filepathlist)
                         filename = filepathlist[0]
+                        # print("filename1:",filename)
                         filepath_put = "{0}/{2}/{1}-Redline.pdf".format(
                             filepathlist[0],filename, packagetype
                         )
+                        # print("filepath_put:",filepath_put)
                         s3path_save = s3client.generate_presigned_url(
                             ClientMethod="get_object",
                             Params={
@@ -272,10 +288,11 @@ def post(ministryrequestid, redactionlayer="redline", layertype="redline"):
                             ExpiresIn=3600,
                             HttpMethod="PUT",
                         )
+                        # print("s3path_save:",s3path_save)
                         singlepkgpath = s3path_save
                         data["s3path_save"] = s3path_save
 
-            if is_single_redline_package(_bcgovcode, packagetype):
+            if is_single_redline:
                 for div in data["divdocumentList"]:
                     if len(div["documentlist"]) > 0:
                         documentlist_copy = div["documentlist"][:]
@@ -294,7 +311,7 @@ def post(ministryrequestid, redactionlayer="redline", layertype="redline"):
 
             data["requestnumber"] = filepathlist[0]
             data["bcgovcode"] = _bcgovcode
-            data["issingleredlinepackage"] = "Y" if is_single_redline_package(_bcgovcode, packagetype) else "N"
+            data["issingleredlinepackage"] = "Y" if is_single_redline else "N"
             return json.dumps(data), 200
         except BusinessException as exception:
             return {"status": exception.status_code, "message": exception.message}, 500
@@ -373,3 +390,40 @@ def get():
             return json.dumps({"license": webviewerlicense}), 200
         except BusinessException as exception:
             return {"status": exception.status_code, "message": exception.message}, 500
+
+
+@cors_preflight('GET,OPTIONS')
+@API.route('/foiflow/personalattributes/<bcgovcode>')
+class GetPersonalTags(Resource):
+    """Get document list.
+    """
+    @staticmethod
+    @TRACER.trace()
+    @cross_origin(origins=allowedorigins())
+    @auth.require
+    @auth.ismemberofgroups(getrequiredmemberships())
+    def get(bcgovcode):
+        try:
+            attributes = ["people", "filetypes", "volumes", "personaltag"]
+            personalattributes = {}
+
+            for attribute in attributes:
+                response = requests.request(
+                    method='GET',
+                    url= requestapiurl + "/api/foiflow/divisions/" + bcgovcode + "/true/" + attribute,
+                    headers={'Authorization': AuthHelper.getauthtoken(), 'Content-Type': 'application/json'},
+                    timeout=float(requestapitimeout)
+                )
+                response.raise_for_status()
+                # get request status
+                jsonobj = response.json()
+                personalattributes.update(jsonobj)
+
+            return json.dumps(personalattributes), 200
+        except KeyError as error:
+            return {'status': False, 'message': CUSTOM_KEYERROR_MESSAGE + str(error)}, 400
+        except BusinessException as exception:
+            return {'status': exception.status_code, 'message':exception.message}, 500
+        except requests.exceptions.HTTPError as err:
+            logging.error("Request Management API returned the following message: {0} - {1}".format(err.response.status_code, err.response.text))
+            return {'status': False, 'message': err.response.text}, err.response.status_code

api/reviewer_api/resources/redaction.py

@@ -39,7 +39,7 @@
     SectionSchema,
 )
 from reviewer_api.schemas.redline import RedlineSchema
-from reviewer_api.schemas.finalpackage import FinalPackageSchema
+from reviewer_api.schemas.finalpackage import FinalPackageSchema, MCFFinalPackageSchema
 
 API = Namespace(
     "Document and annotations",
@@ -341,7 +341,12 @@ class SaveFinalPackage(Resource):
     def post():
         try:
             requestjson = request.get_json()
-            finalpackageschema = FinalPackageSchema().load(requestjson)
+            print("\nrequestjson:",requestjson)
+            if(requestjson['bcgovcode'] == "mcf"):
+                finalpackageschema = MCFFinalPackageSchema().load(requestjson) 
+            else:
+                finalpackageschema = FinalPackageSchema().load(requestjson)
+            print("\nfinalpackageschema:",finalpackageschema)
             result = redactionservice().triggerdownloadredlinefinalpackage(
                 finalpackageschema, AuthHelper.getuserinfo()
             )

api/reviewer_api/schemas/document.py

@@ -24,6 +24,17 @@ class Meta:  # pylint: disable=too-few-public-methods
         unknown = EXCLUDE 
     divisionid = fields.Int(data_key="divisionid",allow_none=False)
 
+class PersonalAttributesSchema(Schema):
+    class Meta:  # pylint: disable=too-few-public-methods
+        """Exclude unknown fields in the deserialized output."""
+
+        unknown = EXCLUDE 
+    person = fields.Str(data_key="person",allow_none=True)
+    filetype = fields.Str(data_key="filetype",allow_none=True)
+    volume = fields.Str(data_key="volume",allow_none=True)
+    trackingid = fields.Str(data_key="trackingid",allow_none=True)
+    personaltag = fields.Str(data_key="personaltag",allow_none=True)
+
 class FOIRequestUpdateRecordsSchema(Schema):
     class Meta:  # pylint: disable=too-few-public-methods
         """Exclude unknown fields in the deserialized output."""
@@ -34,6 +45,14 @@ class Meta:  # pylint: disable=too-few-public-methods
     divisions = fields.Nested(DivisionSchema,many=True,validate=validate.Length(min=1),allow_none=True,required=False)
     rotatedpages = fields.Dict(keys=fields.Str,values=fields.Int,allow_none=True,required=False) # {page: rotation in degrees}
 
+class FOIRequestUpdateRecordPersonalAttributesSchema(Schema):
+    class Meta:  # pylint: disable=too-few-public-methods
+        """Exclude unknown fields in the deserialized output."""
+
+        unknown = EXCLUDE
+    documentmasterids = fields.List(fields.Integer(),data_key="documentmasterids",allow_none=False)
+    ministryrequestid = fields.Int(data_key="ministryrequestid",allow_none=True)
+    personalattributes = fields.Nested(PersonalAttributesSchema,required=False,allow_none=False)
 
 class DocumentPage(Schema):
     class Meta:  # pylint: disable=too-few-public-methods

api/reviewer_api/schemas/finalpackage.py

@@ -4,15 +4,17 @@
 class FileSchema(Schema):
     s3uripath = fields.Str(data_key="s3uripath", allow_none=False)
     filename = fields.Str(data_key="filename", allow_none=False)
-
+    recordname = fields.Str(data_key="recordname", allow_none=False)
+    documentid = fields.Int(data_key="documentid", allow_none=True)
 
 class AttributeSchema(Schema):
     files = fields.Nested(FileSchema, many=True, required=True, allow_none=False)
 
 
 class SummaryPkgSchema(Schema):
     divisionid = fields.Int(data_key="divisionid", allow_none=True)
-    documentids = fields.List(fields.Int())
+    recordname = fields.Str(data_key="recordname", allow_none=True)
+    documentids = fields.List(fields.Int(), allow_none=True)
 
 class SummarySchema(Schema):
     pkgdocuments = fields.List(fields.Nested(SummaryPkgSchema, allow_none=True))
@@ -27,4 +29,28 @@ class FinalPackageSchema(Schema):
         AttributeSchema, many=True, required=True, allow_none=False
     )
     summarydocuments = fields.Nested(SummarySchema, allow_none=True)
+    redactionlayerid = fields.Int(data_key="redactionlayerid", allow_none=False)
+
+class SummaryRecordSchema(Schema):
+    recordname = fields.Str(data_key="recordname", allow_none=True)
+    documentids = fields.List(fields.Int(), allow_none=True)
+
+class MCFSummaryPkgSchema(Schema):
+    divisionid = fields.Int(data_key="divisionid", allow_none=True)
+    documentids = fields.List(fields.Int(), allow_none=True)
+    records = fields.List(fields.Nested(SummaryRecordSchema), allow_none=True)
+
+class MCFSummarySchema(Schema):
+    pkgdocuments = fields.List(fields.Nested(MCFSummaryPkgSchema, allow_none=True))
+    sorteddocuments = fields.List(fields.Int())
+
+class MCFFinalPackageSchema(Schema):
+    ministryrequestid = fields.Str(data_key="ministryrequestid", allow_none=False)
+    category = fields.Str(data_key="category", allow_none=False)
+    requestnumber = fields.Str(data_key="requestnumber", allow_none=False)
+    bcgovcode = fields.Str(data_key="bcgovcode", allow_none=False)
+    attributes = fields.Nested(
+        AttributeSchema, many=True, required=True, allow_none=False
+    )
+    summarydocuments = fields.Nested(MCFSummarySchema, allow_none=True)
     redactionlayerid = fields.Int(data_key="redactionlayerid", allow_none=False)

api/reviewer_api/services/documentservice.py

@@ -430,6 +430,52 @@ def updatedocumentattributes(self, payload, userid):
             )
 
         return DocumentAttributes.update(newRows, oldRows)
+
+    def updatedocumentpersonalattributes(self, payload, userid):
+        """update document attributes"""
+
+        docattributeslist = DocumentAttributes.getdocumentattributesbyid(
+            payload["documentmasterids"]
+        )
+        oldRows = []
+        newRows = []
+        for docattributes in docattributeslist:
+            oldRows.append(
+                {
+                    "attributeid": docattributes["attributeid"],
+                    "version": docattributes["version"],
+                    "documentmasterid": docattributes["documentmasterid"],
+                    "attributes": docattributes["attributes"],
+                    "createdby": docattributes["createdby"],
+                    "created_at": docattributes["created_at"],
+                    "updatedby": userid,
+                    "updated_at": datetime2.now(),
+                    "isactive": False,
+                }
+            )
+            newdocattributes = json.loads(json.dumps(docattributes["attributes"]))
+            if payload["personalattributes"] is not None:
+                #apply change to all
+                if(len(payload["documentmasterids"]) > 1):
+                    if(payload["personalattributes"]["person"] is not None):
+                        newdocattributes["personalattributes"]["person"]=payload["personalattributes"]["person"]
+                    if(payload["personalattributes"]["filetype"] is not None):
+                        newdocattributes["personalattributes"]["filetype"]=payload["personalattributes"]["filetype"]
+                #apply change to individual
+                else:
+                    newdocattributes["personalattributes"] = payload["personalattributes"]
+            newRows.append(
+                DocumentAttributes(
+                    version=docattributes["version"] + 1,
+                    documentmasterid=docattributes["documentmasterid"],
+                    attributes=newdocattributes,
+                    createdby=docattributes["createdby"],
+                    created_at=docattributes["created_at"],
+                    isactive=True,
+                )
+            )
+
+        return DocumentAttributes.update(newRows, oldRows)
 
 
     def getdocuments(self, requestid,bcgovcode):

api/reviewer_api/utils/constants.py

@@ -22,6 +22,7 @@
 BLANK_EXCEPTION_MESSAGE = 'Field cannot be blank'
 MAX_EXCEPTION_MESSAGE = 'Field exceeds the size limit'
 REDLINE_SINGLE_PKG_MINISTRIES = getenv('REDLINE_SINGLE_PKG_MINISTRIES','NA')
+REDLINE_SINGLE_PKG_MINISTRIES_PERSONAL = getenv('REDLINE_SINGLE_PKG_MINISTRIES_PERSONAL','NA')
 try:
     response = requests.request(
         method='GET',

api/reviewer_api/utils/util.py

@@ -34,7 +34,7 @@
     ProcessingTeamWithKeycloackGroup,
 )
 import maya
-from reviewer_api.utils.constants import REDLINE_SINGLE_PKG_MINISTRIES
+from reviewer_api.utils.constants import REDLINE_SINGLE_PKG_MINISTRIES, REDLINE_SINGLE_PKG_MINISTRIES_PERSONAL
 
 def cors_preflight(methods):
     # Render an option method on the class.
@@ -136,11 +136,15 @@ def getbatchconfig():
     _limit = _batchconfig["limit"] if "limit" in _batchconfig else 250    
     return _begin, _size, _limit
 
-def is_single_redline_package(bcgovcode, packagetype):
+def is_single_redline_package(bcgovcode, packagetype, requesttype):
     if (packagetype == "oipcreview"):
         return True
     if REDLINE_SINGLE_PKG_MINISTRIES not in (None, ""):
         _pkg_ministries = REDLINE_SINGLE_PKG_MINISTRIES.replace(" ", "").split(',')
         if bcgovcode.upper() in _pkg_ministries:
             return True
+    if REDLINE_SINGLE_PKG_MINISTRIES_PERSONAL not in (None, ""):
+        _pkg_ministries_personal = REDLINE_SINGLE_PKG_MINISTRIES_PERSONAL.replace(" ", "").split(',')
+        if bcgovcode.upper() in _pkg_ministries_personal and requesttype.upper() == "PERSONAL":
+            return True
     return False