Engagement/jessica/15406 cli download report (#15675)

* Created an API that downloads files from a specified environment and optionally (unless it is prod) removes PII

prime-router/docs/api/reports.yml

@@ -118,6 +118,36 @@ paths:
                   $ref: '#/components/schemas/Report'
         '500':
           description: Internal Server Error
+  /reports/download:
+    get:
+      summary: Downloads a message based on the report id
+      security:
+        - OAuth2: [ system_admin ]
+      parameters:
+        - in: query
+          name: reportId
+          description: The report id to look for to download.
+          schema:
+            type: string
+          required: true
+          example: e491f4fb-f2c5-4473-8db2-206ea04991e8
+        - in: query
+          name: removePII
+          description: Boolean that determines if PII will be removed from the message. If missing will default to true.
+            Required to be true if prod env.
+          required: false
+          schema:
+            type: boolean
+          example: true
+      responses:
+        '200':
+          description: OK
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Report'
+        '500':
+          description: Internal Server Error
 # Building
 components:
   schemas:

prime-router/src/main/kotlin/azure/ReportFunction.kt

@@ -18,6 +18,7 @@ import gov.cdc.prime.router.ActionLogLevel
 import gov.cdc.prime.router.InvalidParamMessage
 import gov.cdc.prime.router.InvalidReportMessage
 import gov.cdc.prime.router.Options
+import gov.cdc.prime.router.ReportId
 import gov.cdc.prime.router.Sender
 import gov.cdc.prime.router.Sender.ProcessingType
 import gov.cdc.prime.router.SubmissionReceiver
@@ -29,14 +30,18 @@ import gov.cdc.prime.router.azure.observability.event.IReportStreamEventService
 import gov.cdc.prime.router.azure.observability.event.ReportStreamEventName
 import gov.cdc.prime.router.azure.observability.event.ReportStreamEventProperties
 import gov.cdc.prime.router.azure.observability.event.ReportStreamEventService
+import gov.cdc.prime.router.cli.PIIRemovalCommands
 import gov.cdc.prime.router.common.AzureHttpUtils.getSenderIP
+import gov.cdc.prime.router.common.Environment
 import gov.cdc.prime.router.common.JacksonMapperUtilities
+import gov.cdc.prime.router.fhirengine.utils.FhirTranscoder
 import gov.cdc.prime.router.history.azure.SubmissionsFacade
 import gov.cdc.prime.router.tokens.AuthenticatedClaims
 import gov.cdc.prime.router.tokens.Scope
 import gov.cdc.prime.router.tokens.authenticationFailure
 import gov.cdc.prime.router.tokens.authorizationFailure
 import org.apache.logging.log4j.kotlin.Logging
+import java.util.UUID
 
 private const val PROCESSING_TYPE_PARAMETER = "processing"
 
@@ -155,6 +160,70 @@ class ReportFunction(
         var reportBody: String,
     )
 
+    /**
+     * GET report to download
+     *
+     * @see ../../../docs/api/reports.yml
+     */
+    @FunctionName("downloadReport")
+    fun downloadReport(
+        @HttpTrigger(
+            name = "downloadReport",
+            methods = [HttpMethod.GET],
+            authLevel = AuthorizationLevel.FUNCTION,
+            route = "reports/download"
+        ) request: HttpRequestMessage<String?>,
+    ): HttpResponseMessage {
+        val reportId = request.queryParameters[REPORT_ID_PARAMETER]
+        val removePIIRaw = request.queryParameters[REMOVE_PII]
+        var removePII = false
+        if (removePIIRaw.isNullOrBlank() || removePIIRaw.toBoolean()) {
+            removePII = true
+        }
+        if (reportId.isNullOrBlank()) {
+            return HttpUtilities.badRequestResponse(request, "Must provide a reportId.")
+        }
+        return processDownloadReport(
+            request,
+            ReportId.fromString(reportId),
+            removePII,
+            Environment.get().envName
+        )
+    }
+
+    fun processDownloadReport(
+        request: HttpRequestMessage<String?>,
+        reportId: UUID,
+        removePII: Boolean?,
+        envName: String,
+        databaseAccess: DatabaseAccess = DatabaseAccess(),
+        piiRemovalCommands: PIIRemovalCommands = PIIRemovalCommands(),
+    ): HttpResponseMessage {
+        val requestedReport = databaseAccess.fetchReportFile(reportId)
+
+        return if (requestedReport.bodyUrl != null && requestedReport.bodyUrl.toString().lowercase().endsWith("fhir")) {
+            val contents = BlobAccess.downloadBlobAsByteArray(requestedReport.bodyUrl)
+
+            val content = if (removePII == null || removePII) {
+                piiRemovalCommands.removePii(FhirTranscoder.decode(contents.toString(Charsets.UTF_8)))
+            } else {
+                if (envName == "prod") {
+                    return HttpUtilities.badRequestResponse(request, "Must remove PII for messages from prod.")
+                }
+
+                val jsonObject = JacksonMapperUtilities.defaultMapper
+                    .readValue(contents.toString(Charsets.UTF_8), Any::class.java)
+                JacksonMapperUtilities.defaultMapper.writeValueAsString(jsonObject)
+            }
+
+            HttpUtilities.okJSONResponse(request, content)
+        } else if (requestedReport.bodyUrl == null) {
+            HttpUtilities.badRequestResponse(request, "The requested report does not exist.")
+        } else {
+            HttpUtilities.badRequestResponse(request, "The requested report is not fhir.")
+        }
+    }
+
     /**
      * The Waters API, in memory of Dr. Michael Waters
      * (The older version of this API is "/api/reports")

prime-router/src/main/kotlin/azure/RequestFunction.kt

@@ -23,6 +23,8 @@ const val ALLOW_DUPLICATES_PARAMETER = "allowDuplicate"
 const val TOPIC_PARAMETER = "topic"
 const val SCHEMA_PARAMETER = "schema"
 const val FORMAT_PARAMETER = "format"
+const val REPORT_ID_PARAMETER = "reportId"
+const val REMOVE_PII = "removePII"
 
 /**
  * Base class for ReportFunction and ValidateFunction

prime-router/src/main/kotlin/cli/PIIRemovalCommands.kt

@@ -56,8 +56,16 @@ class PIIRemovalCommands : CliktCommand(
         if (inputFile.extension.uppercase() != "FHIR") {
             throw CliktError("File ${inputFile.absolutePath} is not a FHIR file.")
         }
-        var bundle = FhirTranscoder.decode(contents)
+        val bundle = FhirTranscoder.decode(contents)
 
+        // Write the output to the screen or a file.
+        if (outputFile != null) {
+            outputFile!!.writeText(removePii(bundle), Charsets.UTF_8)
+        }
+        echo("Wrote output to ${outputFile!!.absolutePath}")
+    }
+
+    fun removePii(bundle: Bundle): String {
         bundle.entry.map { it.resource }.filterIsInstance<Patient>()
             .forEach { patient ->
                 patient.name.forEach { name ->
@@ -76,16 +84,16 @@ class PIIRemovalCommands : CliktCommand(
 
         bundle.entry.map { it.resource }.filterIsInstance<Organization>()
             .forEach { organization ->
-            organization.address.forEach { address ->
-                address.line = mutableListOf(StringType(getFakeValueForElementCall("STREET")))
-            }
-            organization.telecom.forEach { telecom ->
-                handleTelecom(telecom)
-            }
-            organization.contact.forEach { contact ->
-               handleOrganizationalContact(contact)
+                organization.address.forEach { address ->
+                    address.line = mutableListOf(StringType(getFakeValueForElementCall("STREET")))
+                }
+                organization.telecom.forEach { telecom ->
+                    handleTelecom(telecom)
+                }
+                organization.contact.forEach { contact ->
+                    handleOrganizationalContact(contact)
+                }
             }
-        }
 
         bundle.entry.map { it.resource }.filterIsInstance<Practitioner>()
             .forEach { practitioner ->
@@ -103,18 +111,14 @@ class PIIRemovalCommands : CliktCommand(
                 }
             }
 
-        bundle = FhirTransformer("classpath:/metadata/fhir_transforms/common/remove-pii-enrichment.yml").process(bundle)
+        val bundleAfterTransform = FhirTransformer(
+            "classpath:/metadata/fhir_transforms/common/remove-pii-enrichment.yml"
+        ).process(bundle)
 
         val jsonObject = JacksonMapperUtilities.defaultMapper
-            .readValue(FhirTranscoder.encode(bundle), Any::class.java)
-        var prettyText = JacksonMapperUtilities.defaultMapper.writeValueAsString(jsonObject)
-        prettyText = replaceIds(bundle, prettyText)
-
-        // Write the output to the screen or a file.
-        if (outputFile != null) {
-            outputFile!!.writeText(prettyText, Charsets.UTF_8)
-        }
-        echo("Wrote output to ${outputFile!!.absolutePath}")
+            .readValue(FhirTranscoder.encode(bundleAfterTransform), Any::class.java)
+        val prettyText = JacksonMapperUtilities.defaultMapper.writeValueAsString(jsonObject)
+        return replaceIds(bundleAfterTransform, prettyText)
     }
 
     /**
@@ -185,8 +189,10 @@ class PIIRemovalCommands : CliktCommand(
             bundle,
             path
         ).forEach { resourceId ->
-            val newIdentifier = getFakeValueForElementCall("UUID")
-            return prettyText.replace(resourceId.primitiveValue(), newIdentifier, true)
+            if (resourceId.primitiveValue() != null) {
+                val newIdentifier = getFakeValueForElementCall("UUID")
+                return prettyText.replace(resourceId.primitiveValue(), newIdentifier, true)
+            }
         }
         return prettyText
     }

prime-router/src/main/resources/metadata/fhir_transforms/common/remove-pii-address.yml

@@ -2,8 +2,9 @@ elements:
   # removing the street address is more complicated because it is a list so we will do this in code
 
   - name: pii-removal-street-address2
+    condition: '%resource.extension("https://reportstream.cdc.gov/fhir/StructureDefinition/xad-address").extension.where(url = "XAD.2")'
     value: [ 'getFakeValueForElement("STREET_ADDRESS_2")' ]
-    bundleProperty: '%resource.extension(%`rsext-xad-address`).extension.where(url = "XAD.2").value'
+    bundleProperty: '%resource.extension("https://reportstream.cdc.gov/fhir/StructureDefinition/xad-address").extension.where(url = "XAD.2").value'
 
   - name: pii-removal-city
     value: [ 'getFakeValueForElement("CITY",%resource.state)' ]

prime-router/src/main/resources/metadata/fhir_transforms/common/remove-pii-name.yml

@@ -6,5 +6,6 @@ elements:
   #  removing a given name is more complicated because it is a list so we will do this in code
 
   - name: pii-removal-middle-name
+    condition: '%resource.extension("https://reportstream.cdc.gov/fhir/StructureDefinition/xpn-human-name").exists()'
     value: [ 'getFakeValueForElement("PERSON_GIVEN_NAME")' ]
-    bundleProperty: '%resource.extension(%`rsext-xpn-human-name`).extension.where(url="XPN.3").value'
+    bundleProperty: '%resource.extension("https://reportstream.cdc.gov/fhir/StructureDefinition/xpn-human-name").extension("XPN.2").value[x]'

prime-router/src/main/resources/metadata/fhir_transforms/common/remove-pii-telecom.yml

@@ -2,7 +2,7 @@ elements:
   - name: pii-removal-phone-area-code
     condition: "%resource.where(system = 'phone')"
     value: [ 'getFakeValueForElement("TELEPHONE").substring(0,3)' ]
-    bundleProperty: '%resource.extension(%`ext-contactpoint-area`).value'
+    bundleProperty: '%resource.extension(`https://reportstream.cdc.gov/fhir/StructureDefinition/contactpoint-area`).value[x]'
 
   - name: pii-removal-local-phone
     condition: "%resource.where(system = 'phone')"