Deployment: siembol-monitoring (#675)

deployment/helm-k8s/Chart.yaml

@@ -3,7 +3,7 @@ apiVersion: v2
 description: Siembol is an open-source, real-time Security Information & Event Management tool based on big data technologies, providing a scalable, advanced security analytics framework
 name: siembol
 type: application
-version: 0.1.7
+version: 0.1.8
 keywords:
 - kafka
 - zookeeper

deployment/helm-k8s/README.md

@@ -32,7 +32,7 @@ The command removes all the Kubernetes components associated with the chart and
 | ---------------------------| -------------------------| ------------------- |
 | `namespace`                | Name of namespace        | siembol           |
 | `environment`              | Environment used         | dev               |
-| `enabled_apps`             | List of apps to deploy  | [ui, rest, manager, dep_ingresses, enrichment_store] |
+| `enabled_apps`             | List of apps to deploy  | [ui, rest, manager, dep_ingresses, enrichment_store, siembol_monitoring] |
 
 ### Certmanager
 | Parameter | Description         | Default  |
@@ -173,3 +173,22 @@ The command removes all the Kubernetes components associated with the chart and
 | `dep_ingresses.storm.service.port` | Service port | 8080 |
 | `dep_ingresses.storm.oauth2_proxy.enabled` | Enable Oauth2 Proxy for Storm UI | false |
 | `dep_ingresses.storm.oauth2_proxy.host` | Oauth2 Proxy host name | oauth-proxy.siembol.local |
+
+
+### Siembol Monitoring
+| Parameter                  | Description              | Default             |
+| ---------------------------| -------------------------| ------------------- |
+| `siembol_monitoring.appName` | Siembol Monitoring app name | siembol-monitoring |
+| `siembol_monitoring.image.repository` | Siembol Monitoring image repository | gresearchdev/siembol-siembol-monitoring |
+| `siembol_monitoring.image.tag` | Siembol Monitoring image tag | latest |
+| `siembol_monitoring.image.pullPolicy` | Siembol Monitoring image pull policy | Always |
+| `siembol_monitoring.service.port` | Service target port | 8083 |
+| `siembol_monitoring.resources.requests.cpu` | Pod CPU requests | 0.1 |
+| `siembol_monitoring.resources.requests.memory` | Pod memory requests | 128Mi |
+| `siembol_monitoring.resources.limits.cpu` | Pod CPU limits | 0.4 |
+| `siembol_monitoring.resources.limits.memory` | Pod memory limits | 256Mi |
+| `siembol_monitoring.readinessProbe.initialDelaySeconds` | Initial delay seconds for readinessProbe | 20 |
+| `siembol_monitoring.readinessProbe.periodSeconds` | Period seconds for readinessProbe | 30 |
+| `siembol_monitoring.livenessProbe.initialDelaySeconds` | Initial delay seconds for livenessProbe | 30 |
+| `siembol_monitoring.livenessProbe.periodSeconds` | Period seconds for livenessProbe | 30 |
+

deployment/helm-k8s/templates/_helpers.tpl

@@ -37,7 +37,7 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this
 */}}
 {{- define "siembol.ui.fullname" -}}
 {{- $name := default .Chart.Name .Values.ui.appName -}}
-{{- printf "%s-%s" (include "siembol.fullname" .) $name | trunc 63 | trimSuffix "-" -}}
+{{- printf "%s" $name | trunc 63 | trimSuffix "-" -}}
 {{- end -}}
 
 {{/*
@@ -46,7 +46,7 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this
 */}}
 {{- define "siembol.rest.fullname" -}}
 {{- $name := default .Chart.Name .Values.rest.appName -}}
-{{- printf "%s-%s" (include "siembol.fullname" .) $name | trunc 63 | trimSuffix "-" -}}
+{{- printf "%s" $name | trunc 63 | trimSuffix "-" -}}
 {{- end -}}
 
 {{/*
@@ -55,7 +55,7 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this
 */}}
 {{- define "siembol.response.fullname" -}}
 {{- $name := default .Chart.Name .Values.response.appName -}}
-{{- printf "%s-%s" (include "siembol.fullname" .) $name | trunc 63 | trimSuffix "-" -}}
+{{- printf "%s" $name | trunc 63 | trimSuffix "-" -}}
 {{- end -}}
 
 {{/*
@@ -65,12 +65,7 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this
 
 {{- define "siembol.manager.appname.fullname" -}}
 {{- $name := default .Chart.Name .Values.manager.appName -}}
-{{- printf "%s-%s" (include "siembol.fullname" .) $name | trunc 63 | trimSuffix "-" -}}
-{{- end -}}
-
-{{- define "siembol.manager.appname.cleanup.name" -}}
-{{- $name := default .Chart.Name .Values.manager.cleanupjob.name -}}
-{{- printf "%s-%s" (include "siembol.fullname" .) $name | trunc 63 | trimSuffix "-" -}}
+{{- printf "%s" $name | trunc 63 | trimSuffix "-" -}}
 {{- end -}}
 
 {{/*
@@ -79,7 +74,7 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this
 */}}
 {{- define "siembol.enrichment_store.fullname" -}}
 {{- $name := default .Chart.Name .Values.enrichment_store.appName -}}
-{{- printf "%s-%s" (include "siembol.fullname" .) $name | trunc 63 | trimSuffix "-" -}}
+{{- printf "%s" $name | trunc 63 | trimSuffix "-" -}}
 {{- end -}}
 
 {{/*
@@ -124,8 +119,10 @@ Set the ZooKeeper server for the siembol chart
 {{- end -}}
 
 {{/*
-Set the name for the Kafka chart
+Create a fully qualified siembol monitoring fullname.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
 */}}
-{{- define "kafka.fullname" -}}
-{{- printf "%s-%s" .Release.Name "kafka" | trunc 63 | trimSuffix "-" -}}
+{{- define "siembol.siembol_monitoring.fullname" -}}
+{{- $name := default .Chart.Name .Values.siembol_monitoring.appName -}}
+{{- printf "%s" $name | trunc 63 | trimSuffix "-" -}}
 {{- end -}}

deployment/helm-k8s/templates/siembol_monitoring/deployment.yaml

@@ -0,0 +1,61 @@
+{{- if (has "siembol_monitoring" .Values.enabled_apps) -}}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: {{ include "siembol.siembol_monitoring.fullname" $ }}
+  name: {{ include "siembol.siembol_monitoring.fullname" $ }}
+  namespace: {{ .Values.namespace }}
+spec:
+  replicas: 1 
+  selector:
+    matchLabels:
+      app: {{ include "siembol.siembol_monitoring.fullname" $ }}
+  strategy:
+    rollingUpdate:
+      maxSurge: 1
+      maxUnavailable: 0
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: {{ include "siembol.siembol_monitoring.fullname" $ }}
+      name: {{ include "siembol.siembol_monitoring.fullname" $ }}
+    spec:
+      containers:
+      - image: {{ .Values.siembol_monitoring.image.repository }}:{{ .Values.siembol_monitoring.image.tag }}
+        imagePullPolicy: {{ .Values.siembol_monitoring.image.pullPolicy }}
+        name: {{ include "siembol.siembol_monitoring.fullname" $ }}
+        resources:
+          requests:
+            memory: {{ .Values.siembol_monitoring.resources.requests.memory }}
+            cpu: {{ .Values.siembol_monitoring.resources.requests.cpu }}
+          limits:
+            memory: {{ .Values.siembol_monitoring.resources.limits.memory }}
+            cpu: {{ .Values.siembol_monitoring.resources.limits.cpu }}
+        securityContext:
+          runAsUser: 101
+        readinessProbe:
+          httpGet:
+            path: /health
+            port: {{ .Values.siembol_monitoring.service.port }}
+            scheme: HTTP
+          initialDelaySeconds: {{ .Values.siembol_monitoring.readinessProbe.initialDelaySeconds }}
+          periodSeconds: {{ .Values.siembol_monitoring.readinessProbe.periodSeconds }}
+        livenessProbe:
+          httpGet:
+            path: /health
+            port: {{ .Values.siembol_monitoring.service.port }}
+            scheme: HTTP
+        volumeMounts:
+        - name: cacerts
+          mountPath: /etc/ssl/certs/java/cacerts
+          subPath: cacerts
+          readOnly: true        
+      restartPolicy: Always
+      volumes:
+      - name: cacerts
+        secret:
+          secretName: cacerts
+      terminationGracePeriodSeconds: 300
+{{- end -}}

deployment/helm-k8s/templates/siembol_monitoring/service.yaml

@@ -0,0 +1,16 @@
+{{- if (has "siembol_monitoring" .Values.enabled_apps) -}}
+kind: Service
+apiVersion: v1
+metadata:
+  name: {{ include "siembol.siembol_monitoring.fullname" $ }}
+  namespace: {{ .Values.namespace }}
+  labels:
+    metrics: "true"
+spec:
+  selector:
+    app: {{ include "siembol.siembol_monitoring.fullname" $ }}
+  ports:
+  - protocol: TCP
+    port: {{ .Values.siembol_monitoring.service.port }}
+    name: web
+{{- end -}}

deployment/helm-k8s/values.yaml

@@ -14,6 +14,7 @@ enabled_apps:
 - manager
 - dep_ingresses
 - enrichment_store
+- siembol_monitoring
 
 # Enable certmanager 
 certmanager:
@@ -237,3 +238,31 @@ response:
   livenessProbe:
     initialDelaySeconds: 100
     periodSeconds: 10
+
+# -----------------------------------------------------------------------------
+# siembol monitoring parameters
+# -----------------------------------------------------------------------------
+siembol_monitoring:
+  appName: "siembol-monitoring"
+  image:
+    repository: "gresearchdev/siembol-siembol-monitoring"
+    tag: "latest"
+    pullPolicy: Always
+
+  service:
+    port: 8083
+
+  resources:
+    requests:
+      cpu: 0.1
+      memory: "128Mi"
+    limits:
+      cpu: 0.5
+      memory: "256Mi"
+
+  readinessProbe:
+    initialDelaySeconds: 20
+    periodSeconds: 30
+  livenessProbe:
+    initialDelaySeconds: 30
+    periodSeconds: 30

deployment/quickstart_install/ps-scripts/demoInstall.ps1

@@ -1,5 +1,5 @@
 $CONFIG_MAP_NAME_GIT="github-details"
-$GIT_SECRET_NAME="siembol-config-editor-rest-secrets"
+$GIT_SECRET_NAME="config-editor-rest-secrets"
 $NAMESPACE="siembol"
 
 function Git-Details {
@@ -42,4 +42,4 @@ if ($zookeeper_status -eq 'True')  {
 } else {
     Write-Output "Zookeeper pod is not running yet, please try again in a few seconds"
     exit 1
-}
+}

deployment/quickstart_install/sh-scripts/demoInstall.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 CONFIG_MAP_NAME_GIT="github-details"
-GIT_SECRET_NAME="siembol-config-editor-rest-secrets"
+GIT_SECRET_NAME="config-editor-rest-secrets"
 NAMESPACE="siembol"
 
 git_details () {

docs/deployment/deployment.md

@@ -18,6 +18,7 @@ Docker images are built both from snapshots and releases.
     - [storm-topology-manager](https://hub.docker.com/r/gresearchdev/siembol-storm-topology-manager/)
     - [config-editor-rest](https://hub.docker.com/r/gresearchdev/siembol-config-editor-rest/)
     - [responding-stream](https://hub.docker.com/r/gresearchdev/siembol-responding-stream/)
+    - [siembol-monitoring](https://hub.docker.com/r/gresearchdev/siembol-monitoring/)
 
 - Config editor UI
     - A Single page Angular application 
@@ -160,6 +161,17 @@ We have a folder for ingress specifications for extra components which allows fo
 | `dep_ingresses.storm.oauth2_proxy.enabled` | Enable oauth2 proxy for this ingress | false |
 | `dep_ingresses.storm.oauth2_proxy.host` | Host for oauth2 proxy | oauth-proxy.siembol.local |
 
+#### Siembol Monitoring
+Siembol monitoring is a springboot application with components to monitor Siembol. It has one component: siembol heartbeat. 
+
+##### Siembol Heartbeat
+Siembol heartbeat can be used to monitor all components of Siembol are working correctly. It has two main components:
+- kafka producers: send a heartbeat message to a kafka topic at an interval of time specified in the config. Multiple producers can be defined with different kafka properties, e.g. different kafka cluster or topics.
+- kafka consumer: reads the heartbeat message after it has been processed by Siembol services. Calculates the total latency and latency between services depending on the services enabled. These metrics are exposed and ready to be scraped by Prometheus.
+
+For the heartbeat to work config has to be added to each Siembol service type used to process the heartbeat message and write it to the topic read by the consumer. 
+
+See [siembol monitoring](how-tos/how_to_setup_siembol_monitoring.md) for configuration details.
 
 
 ### Enable & Disable components
@@ -172,6 +184,7 @@ By default the enabled_apps list consists of these components:
 - manager
 - dep_ingresses
 - enrichment_store
+- siembol_monitoring
 ```
 Any component can be removed by removing it from the list in [values.yaml](../../deployment/helm-k8s/values.yaml) or you can add another component such as `response` e.g.
 ```bash

docs/deployment/how-tos/how_to_setup_siembol_monitoring.md

@@ -0,0 +1,41 @@
+# How to set up Siembol monitoring
+Siembol monitoring is currently composed of one component: Siembol heartbeat.
+
+## Siembol Heartbeat
+Siembol heartbeat is a springboot application with two main components: kafka producers and a kafka consumer. The kafka producers send heartbeat messages to kafka at an interval of time.  The kafka consumer reads the messages after they have been processed by various Siembol services and calculate total latency and latency between Siembol services.  
+
+The properties of the heartbeat are defined in its `application.properties` file.
+
+### General properties
+- the interval in seconds the producers will send heartbeat messages, the default is 60 seconds
+```properties
+siembol-monitoring.heartbeat-properties.heartbeat-interval-seconds=60
+```
+- any optional additional fields to add to the heartbeat message
+```properties
+siembol-monitoring.heartbeat-properties.message.key1=value1
+siembol-monitoring.heartbeat-properties.message.key2=value2
+```
+
+### Producer properties
+One or multiple producers can be defined in the properties to monitor data from different kafka clusters, for example
+
+```properties
+siembol-monitoring.heartbeat-properties.heartbeat-producers.local-kafka-cluster.output-topic=siembol.heartbeat
+siembol-monitoring.heartbeat-properties.heartbeat-producers.local-kafka-cluster.kafka-properties.[bootstrap.servers]=kafka-0.kafka-headless.siembol.svc.cluster.local:9092
+siembol-monitoring.heartbeat-properties.heartbeat-producers.local-kafka-cluster.kafka-properties.[security.protocol]=PLAINTEXT
+```
+Any arbitrary additional kafka producer properties can be added (https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html).
+
+### Consumer properties
+```properties
+siembol-monitoring.heartbeat-properties.heartbeat-consumer.kafka-properties.[bootstrap.servers]=kafka-0.kafka-headless.siembol.svc.cluster.local:9092
+siembol-monitoring.heartbeat-properties.heartbeat-consumer.kafka-properties.[application.id]=siembol.heartbeat.reader
+siembol-monitoring.heartbeat-properties.heartbeat-consumer.kafka-properties.[auto.offset.reset]=earliest
+siembol-monitoring.heartbeat-properties.heartbeat-consumer.kafka-properties.[security.protocol]=PLAINTEXT
+siembol-monitoring.heartbeat-properties.heartbeat-consumer.enabled-services=parsingapp,enrichment,response
+```
+Any arbitrary additional kafka streams properties can be added (https://kafka.apache.org/10/documentation/streams/developer-guide/config-streams.html).
+The `enabled-services` property is to specify between which Siembol services latency should be computed.
+
+