feat: CE-1135-webeoc-polling (#798)

backend/src/v1/complaint/complaint.controller.ts

@@ -141,15 +141,15 @@ export class ComplaintController {
   @Public()
   @Post("/staging/action-taken")
   @UseGuards(ApiKeyGuard)
-  stageActionTaken(@Body() action: ActionTaken) {
-    this.stagingService.stageObject("ACTION-TAKEN", action);
+  async stageActionTaken(@Body() action: ActionTaken) {
+    return await this.stagingService.stageObject("ACTION-TAKEN", action);
   }
 
   @Public()
   @Post("/staging/action-taken-update")
   @UseGuards(ApiKeyGuard)
-  stageActionTakenUpdate(@Body() action: ActionTaken) {
-    this.stagingService.stageObject("ACTION-TAKEN-UPDATE", action);
+  async stageActionTakenUpdate(@Body() action: ActionTaken) {
+    return await this.stagingService.stageObject("ACTION-TAKEN-UPDATE", action);
   }
 
   @Public()

charts/app/templates/secret.yaml

@@ -31,8 +31,8 @@
 {{- $webeocPosition := (get $secretData "webeocPosition" | b64dec | default "") }}
 {{- $webeocIncident := (get $secretData "webeocIncident" | b64dec | default "") }}
 {{- $webeocUrl := (get $secretData "webeocUrl" | b64dec | default "") }}
-{{- $webeocComplaintHistorySeconds := (get $secretData "webeocComplaintHistorySeconds" | b64dec | default "") }}
 {{- $webeocCronExpression := (get $secretData "webeocCronExpression" | b64dec | default "") }}
+{{- $webeocLogPath := (get $secretData "webeocLogPath" | b64dec | default "") }}
 {{- $backupDir := (get $secretData "backupDir" | b64dec | default "") }}
 {{- $backupStrategy := (get $secretData "backupStrategy" | b64dec | default "") }}
 {{- $numBackups := (get $secretData "numBackups" | b64dec | default "") }}
@@ -120,8 +120,8 @@ data:
   WEBEOC_POSITION: {{ $webeocPosition | b64enc | quote }}
   WEBEOC_INCIDENT: {{ $webeocIncident | b64enc | quote }}
   WEBEOC_URL: {{ $webeocUrl | b64enc | quote }}
-  WEBEOC_COMPLAINT_HISTORY_SECONDS: {{ $webeocComplaintHistorySeconds | b64enc | quote }}
   WEBEOC_CRON_EXPRESSION: {{ $webeocCronExpression | b64enc | quote }}
+  WEBEOC_LOG_PATH: {{ $webeocLogPath | b64enc | quote }}
   COMPLAINTS_API_KEY: {{ $caseManagementApiKey | b64enc | quote }}
 {{- end }}
 {{- if not (lookup "v1" "Secret" .Release.Namespace (printf "%s-flyway" .Release.Name)) }}

charts/app/templates/webeoc/templates/deployment.yaml

@@ -22,6 +22,10 @@ spec:
       labels:
         {{- include "webeoc.labels" . | nindent 8 }}
     spec:
+      volumes:
+        - name: {{ include "webeoc.fullname" . }}
+          persistentVolumeClaim:
+            claimName: {{ include "webeoc.fullname" . }}
       automountServiceAccountToken: false
       {{- if .Values.webeoc.podSecurityContext }}
       securityContext:
@@ -33,20 +37,23 @@ spec:
           securityContext:
             {{- toYaml .Values.webeoc.securityContext | nindent 12 }}
           {{- end }}
-          image: "{{.Values.global.registry}}/{{.Values.global.repository}}/webeoc:{{ .Values.global.tag | default .Chart.AppVersion }}"
+          image: "{{ .Values.global.registry }}/{{ .Values.global.repository }}/webeoc:{{ .Values.global.tag | default .Chart.AppVersion }}"
           imagePullPolicy: {{ default "Always" .Values.webeoc.imagePullPolicy }}
           envFrom:
             - secretRef:
-                name: {{.Release.Name}}-webeoc
+                name: {{ .Release.Name }}-webeoc
           env:
             - name: LOG_LEVEL
               value: info
             - name: NODE_TLS_REJECT_UNAUTHORIZED
               value: "0"
             - name: NATS_HOST
-              value: nats://{{.Release.Name}}-nats:4222
+              value: nats://{{ .Release.Name }}-nats:4222
             - name: COMPLAINTS_MANAGEMENT_API_URL
-              value: https://{{.Release.Name}}-frontend.apps.silver.devops.gov.bc.ca/api/v1
+              value: https://{{ .Release.Name }}-frontend.apps.silver.devops.gov.bc.ca/api/v1
+          volumeMounts:
+            - name: {{ include "webeoc.fullname" . }}
+              mountPath: /mnt/data
           ports:
             - name: http
               containerPort: {{ .Values.webeoc.service.targetPort }}
@@ -69,9 +76,11 @@ spec:
             timeoutSeconds: 5
           resources: # this is optional
             limits:
+              ephemeral-storage: "25Mi"
               cpu: 80m
               memory: 150Mi
             requests:
+              ephemeral-storage: "15Mi"
               cpu: 40m
               memory: 75Mi
       {{- with .Values.webeoc.nodeSelector }}

charts/app/templates/webeoc/templates/pvc.yaml

@@ -0,0 +1,15 @@
+{{- if .Values.webeoc.enabled }}
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: {{ include "webeoc.fullname" . }}
+  labels:
+    {{- include "webeoc.labels" . | nindent 4 }}
+spec:
+    accessModes:
+      - ReadWriteMany
+    resources:
+      requests:
+        storage: "25Mi"
+    storageClassName: netapp-file-standard
+{{- end }}

charts/app/values.yaml

@@ -34,8 +34,8 @@ global:
     webeocPosition: ~
     webeocIncident: ~
     webeocUrl: ~
-    webeocComplaintHistorySeconds: ~
     webeocCronExpression: ~
+    webeocLogPath: ~
     backupDir: ~
     backupStrategy: ~
     numBackups: ~

docker-compose.yml

@@ -117,5 +117,8 @@ services:
     volumes:
       - ./webeoc:/app:z
       - /app/node_modules
+      - ./webeoc/logs:/mnt/data  # this is just for the developer webeoc logging
     user: root
     working_dir: "/app"
+
+

terraform/alerts_prod.tf

@@ -256,4 +256,20 @@ resource "sysdig_monitor_alert_v2_prometheus" "nr_database_prod_storage_usage" {
     service = "NatCom Database"
     app = "NatCom"
   }
+}
+resource "sysdig_monitor_alert_v2_prometheus" "nr_webeoc_prod_storage_usage" {
+  name = "Prod Webeoc Custom Log Storage Alert"
+  description = "Alert when the PVC storage usage is too high"
+  severity = "high"
+  query = "sysdig_fs_used_percent{kube_cluster_name=\"silver\",kube_namespace_name=\"c1c7ed-prod\",kube_deployment_name=\"nr-compliance-enforcement-prod-webeoc\"} > 70"
+  enabled = true
+  duration_seconds = 600
+  notification_channels {
+    id = sysdig_monitor_notification_channel_email.prod_environment_alerts.id
+    renotify_every_minutes = 120
+  }
+  labels = {
+    service = "NatCom Webeoc"
+    app = "NatCom"
+  }
 }

terraform/alerts_test.tf

@@ -256,4 +256,20 @@ resource "sysdig_monitor_alert_v2_prometheus" "nr_database_test_storage_usage" {
     service = "NatCom Database"
     app = "NatCom"
   }
+}
+resource "sysdig_monitor_alert_v2_prometheus" "nr_webeoc_test_storage_usage" {
+  name = "Test Webeoc Custom Log Storage Alert"
+  description = "Alert when the PVC storage usage is too high"
+  severity = "high"
+  query = "sysdig_fs_used_percent{kube_cluster_name=\"silver\",kube_namespace_name=\"c1c7ed-test\",kube_deployment_name=\"nr-compliance-enforcement-test-webeoc\"} > 70"
+  enabled = true
+  duration_seconds = 600
+  notification_channels {
+    id = sysdig_monitor_notification_channel_email.test_environment_alerts.id
+    renotify_every_minutes = 120
+  }
+  labels = {
+    service = "NatCom Webeoc"
+    app = "NatCom"
+  }
 }

terraform/channels.tf

@@ -1,6 +1,6 @@
 resource "sysdig_monitor_notification_channel_email" "test_environment_alerts" {
     name                    = "Team Wolverine - Test Environment Alerts"
-    recipients              = ["[email protected]"]
+    recipients              = ["[email protected]", "[email protected]", "[email protected]", "[email protected]", "[email protected]", "[email protected]"]
     enabled                 = true
     notify_when_ok          = true
     notify_when_resolved    = true
@@ -9,7 +9,7 @@ resource "sysdig_monitor_notification_channel_email" "test_environment_alerts" {
 }
 resource "sysdig_monitor_notification_channel_email" "prod_environment_alerts" {
     name                    = "Team Wolverine - Prod Environment Alerts"
-    recipients              = ["[email protected]"]
+    recipients              = ["[email protected]", "[email protected]", "[email protected]", "[email protected]", "[email protected]", "[email protected]"]
     enabled                 = true
     notify_when_ok          = true
     notify_when_resolved    = true

webeoc/.gitignore

@@ -3,14 +3,16 @@
 /node_modules
 
 # Logs
-logs
-*.log
 npm-debug.log*
 pnpm-debug.log*
 yarn-debug.log*
 yarn-error.log*
 lerna-debug.log*
 
+# Include the log directory for local development but ignore all the logs inside of it
+!/logs/
+logs/*.log
+
 # OS
 .DS_Store
 

webeoc/src/common/constants.ts

@@ -7,6 +7,13 @@ export const STREAMS = {
   ACTIONS_TAKEN: "actions",
 };
 
+export const OPERATIONS = {
+  COMPLAINT: "Complaint(s)",
+  COMPLAINT_UPDATE: "Complaint Update(s)",
+  ACTION_TAKEN: "Action(s) Taken",
+  ACTION_TAKEN_UPDATE: "Action Taken Update(s)",
+};
+
 export const STREAM_TOPICS = {
   COMPLAINTS: "new_complaints",
   STAGING_COMPLAINTS: "new_staging_complaints",

webeoc/src/complaint-api-service/complaint-api.service.ts

@@ -19,7 +19,7 @@ export class ComplaintApiService {
   stageActionTaken = async (record: ActionTakenDto) => {
     try {
       const url = `${process.env.COMPLAINTS_MANAGEMENT_API_URL}/${STAGING_APIS.ACTION_TAKEN}`;
-      this.logger.debug(`Posting action-taken to staging. API URL: ${url}`);
+      this.logger.debug(`Posting action-taken for ${record.actionTakenId} to staging. API URL: ${url}`);
 
       await axios.post(url, record, this._apiConfig);
     } catch (error) {
@@ -33,7 +33,7 @@ export class ComplaintApiService {
   stageActionTakenUpdate = async (record: ActionTakenDto) => {
     try {
       const url = `${process.env.COMPLAINTS_MANAGEMENT_API_URL}/${STAGING_APIS.UPDATE_ACTION_TAKEN}`;
-      this.logger.debug(`Posting action-taken-update to staging. API URL: ${url}`);
+      this.logger.debug(`Posting action-taken-update ${record.actionTakenId} to staging. API URL: ${url}`);
 
       await axios.post(url, record, this._apiConfig);
     } catch (error) {

webeoc/src/publishers/actions-taken-publisher.service.ts

@@ -45,11 +45,15 @@ export class ActionsTakenPublisherService {
       const ack = await this.jsClient.publish(STREAM_TOPICS.STAGE_ACTION_TAKEN, msg, { headers: natsHeaders });
       if (!ack.duplicate) {
         this.logger.debug(
-          `Processing message: ${this._generateHeader("stage-new-action-taken", action, "ACTION-TAKEN")}`,
+          `Publishing new action taken for staging: ${this._generateHeader(
+            "stage-new-action-taken",
+            action,
+            "ACTION-TAKEN",
+          )}`,
         );
       } else {
         this.logger.debug(
-          `Message processed: ${this._generateHeader("stage-new-action-taken", action, "ACTION-TAKEN")}`,
+          `Action taken already published: ${this._generateHeader("stage-new-action-taken", action, "ACTION-TAKEN")}`,
         );
       }
     } catch (error) {
@@ -72,11 +76,19 @@ export class ActionsTakenPublisherService {
 
       if (!ack.duplicate) {
         this.logger.debug(
-          `Processing message: ${this._generateHeader("stage-new-action-taken-update", action, "ACTION-TAKEN-UPDATE")}`,
+          `Publishing new action taken update for staging: ${this._generateHeader(
+            "stage-new-action-taken-update",
+            action,
+            "ACTION-TAKEN-UPDATE",
+          )}`,
         );
       } else {
         this.logger.debug(
-          `Message processed: ${this._generateHeader("stage-new-action-taken-update", action, "ACTION-TAKEN-UPDATE")}`,
+          `Action taken already published: ${this._generateHeader(
+            "stage-new-action-taken-update",
+            action,
+            "ACTION-TAKEN-UPDATE",
+          )}`,
         );
       }
     } catch (error) {
@@ -109,10 +121,20 @@ export class ActionsTakenPublisherService {
 
       if (!ack.duplicate) {
         this.logger.debug(
-          `Processing message: ${this._generateHeader("promote-action-taken", action, "ACTION-TAKEN")}`,
+          `Action taken ready to be moved to operational tables: ${this._generateHeader(
+            "promote-action-taken",
+            action,
+            "ACTION-TAKEN",
+          )}`,
         );
       } else {
-        this.logger.debug(`Message processed: ${this._generateHeader("promote-action-taken", action, "ACTION-TAKEN")}`);
+        this.logger.debug(
+          `Action taken already moved to operational tables: ${this._generateHeader(
+            "promote-action-taken",
+            action,
+            "ACTION-TAKEN",
+          )}`,
+        );
       }
     } catch (error) {
       this.logger.error(`Unable to process request: ${error.message}`, error.stack);
@@ -140,11 +162,19 @@ export class ActionsTakenPublisherService {
 
       if (!ack.duplicate) {
         this.logger.debug(
-          `Processing message: ${this._generateHeader("promote-action-taken-update", action, "ACTION-TAKEN-UPDATE")}`,
+          `Action taken update ready to be moved to operational tables: ${this._generateHeader(
+            "promote-action-taken-update",
+            action,
+            "ACTION-TAKEN-UPDATE",
+          )}`,
         );
       } else {
         this.logger.debug(
-          `Message processed: ${this._generateHeader("promote-action-taken-update", action, "ACTION-TAKEN-UPDATE")}`,
+          `Action taken already moved to operational tables: ${this._generateHeader(
+            "promote-action-taken-update",
+            action,
+            "ACTION-TAKEN-UPDATE",
+          )}`,
         );
       }
     } catch (error) {

webeoc/src/publishers/complaints-publisher.service.ts

@@ -33,7 +33,7 @@ export class ComplaintsPublisherService {
       natsHeaders.set("Nats-Msg-Id", `staged-${complaint.incident_number}-${complaint.created_by_datetime}`);
       const ack = await this.jsClient.publish(STREAM_TOPICS.COMPLAINTS, msg, { headers: natsHeaders });
       if (!ack.duplicate) {
-        this.logger.debug(`New complaint: ${complaint.incident_number}`);
+        this.logger.debug(`Publishing new complaint for staging: ${complaint.incident_number}`);
       } else {
         this.logger.debug(`Complaint already published: ${complaint.incident_number}`);
       }
@@ -59,12 +59,12 @@ export class ComplaintsPublisherService {
       );
       const ack = await this.jsClient.publish(STREAM_TOPICS.COMPLAINT_UPDATE, jsonData, { headers: natsHeaders });
       if (!ack.duplicate) {
-        this.logger.debug(`Complaint update: ${incidentNumber} ${updateNumber}`);
+        this.logger.debug(`Publishing new complaint update for staging: ${incidentNumber} ${updateNumber}`);
       } else {
         this.logger.debug(`Complaint update already published: ${incidentNumber}`);
       }
     } catch (error) {
-      this.logger.error(`Error publishing complaint: ${error.message}`, error.stack);
+      this.logger.error(`Error publishing complaint update: ${error.message}`, error.stack);
       throw error;
     }
   }
@@ -84,7 +84,7 @@ export class ComplaintsPublisherService {
       if (!ack?.duplicate) {
         this.logger.debug(`Complaint ready to be moved to operational tables: ${incident_number}`);
       } else {
-        this.logger.debug(`Complaint already moved to operational: ${incident_number}`);
+        this.logger.debug(`Complaint already moved to operational tables: ${incident_number}`);
       }
     } catch (error) {
       this.logger.error(`Error saving complaint to staging: ${error.message}`, error.stack);
@@ -119,7 +119,7 @@ export class ComplaintsPublisherService {
           `Complaint update ready to be moved to operational tables: ${incidentNumber} ${updateNumber}`,
         );
       } else {
-        this.logger.debug(`Complaint update already moved to operational: ${incidentNumber} ${updateNumber}`);
+        this.logger.debug(`Complaint update already moved to operational tables: ${incidentNumber} ${updateNumber}`);
       }
     } catch (error) {
       this.logger.error(`Error saving complaint update to staging: ${error.message}`, error.stack);