diff --git a/deployment/helm-k8s/Chart.yaml b/deployment/helm-k8s/Chart.yaml index 4ba9d6099..cbb189979 100644 --- a/deployment/helm-k8s/Chart.yaml +++ b/deployment/helm-k8s/Chart.yaml @@ -3,7 +3,7 @@ apiVersion: v2 description: Siembol is an open-source, real-time Security Information & Event Management tool based on big data technologies, providing a scalable, advanced security analytics framework name: siembol type: application -version: 0.1.7 +version: 0.1.8 keywords: - kafka - zookeeper diff --git a/deployment/helm-k8s/README.md b/deployment/helm-k8s/README.md index bf38b253d..b65ac56a1 100644 --- a/deployment/helm-k8s/README.md +++ b/deployment/helm-k8s/README.md @@ -32,7 +32,7 @@ The command removes all the Kubernetes components associated with the chart and | ---------------------------| -------------------------| ------------------- | | `namespace` | Name of namespace | siembol | | `environment` | Environment used | dev | -| `enabled_apps` | List of apps to deploy | [ui, rest, manager, dep_ingresses, enrichment_store] | +| `enabled_apps` | List of apps to deploy | [ui, rest, manager, dep_ingresses, enrichment_store, siembol_monitoring] | ### Certmanager | Parameter | Description | Default | @@ -173,3 +173,22 @@ The command removes all the Kubernetes components associated with the chart and | `dep_ingresses.storm.service.port` | Service port | 8080 | | `dep_ingresses.storm.oauth2_proxy.enabled` | Enable Oauth2 Proxy for Storm UI | false | | `dep_ingresses.storm.oauth2_proxy.host` | Oauth2 Proxy host name | oauth-proxy.siembol.local | + + +### Siembol Monitoring +| Parameter | Description | Default | +| ---------------------------| -------------------------| ------------------- | +| `siembol_monitoring.appName` | Siembol Monitoring app name | siembol-monitoring | +| `siembol_monitoring.image.repository` | Siembol Monitoring image repository | gresearchdev/siembol-siembol-monitoring | +| `siembol_monitoring.image.tag` | Siembol Monitoring image tag | latest | +| `siembol_monitoring.image.pullPolicy` | Siembol Monitoring image pull policy | Always | +| `siembol_monitoring.service.port` | Service target port | 8083 | +| `siembol_monitoring.resources.requests.cpu` | Pod CPU requests | 0.1 | +| `siembol_monitoring.resources.requests.memory` | Pod memory requests | 128Mi | +| `siembol_monitoring.resources.limits.cpu` | Pod CPU limits | 0.4 | +| `siembol_monitoring.resources.limits.memory` | Pod memory limits | 256Mi | +| `siembol_monitoring.readinessProbe.initialDelaySeconds` | Initial delay seconds for readinessProbe | 20 | +| `siembol_monitoring.readinessProbe.periodSeconds` | Period seconds for readinessProbe | 30 | +| `siembol_monitoring.livenessProbe.initialDelaySeconds` | Initial delay seconds for livenessProbe | 30 | +| `siembol_monitoring.livenessProbe.periodSeconds` | Period seconds for livenessProbe | 30 | + diff --git a/deployment/helm-k8s/templates/_helpers.tpl b/deployment/helm-k8s/templates/_helpers.tpl index 94e731772..20f9cfe58 100644 --- a/deployment/helm-k8s/templates/_helpers.tpl +++ b/deployment/helm-k8s/templates/_helpers.tpl @@ -37,7 +37,7 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this */}} {{- define "siembol.ui.fullname" -}} {{- $name := default .Chart.Name .Values.ui.appName -}} -{{- printf "%s-%s" (include "siembol.fullname" .) $name | trunc 63 | trimSuffix "-" -}} +{{- printf "%s" $name | trunc 63 | trimSuffix "-" -}} {{- end -}} {{/* @@ -46,7 +46,7 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this */}} {{- define "siembol.rest.fullname" -}} {{- $name := default .Chart.Name .Values.rest.appName -}} -{{- printf "%s-%s" (include "siembol.fullname" .) $name | trunc 63 | trimSuffix "-" -}} +{{- printf "%s" $name | trunc 63 | trimSuffix "-" -}} {{- end -}} {{/* @@ -55,7 +55,7 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this */}} {{- define "siembol.response.fullname" -}} {{- $name := default .Chart.Name .Values.response.appName -}} -{{- printf "%s-%s" (include "siembol.fullname" .) $name | trunc 63 | trimSuffix "-" -}} +{{- printf "%s" $name | trunc 63 | trimSuffix "-" -}} {{- end -}} {{/* @@ -65,12 +65,7 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this {{- define "siembol.manager.appname.fullname" -}} {{- $name := default .Chart.Name .Values.manager.appName -}} -{{- printf "%s-%s" (include "siembol.fullname" .) $name | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{- define "siembol.manager.appname.cleanup.name" -}} -{{- $name := default .Chart.Name .Values.manager.cleanupjob.name -}} -{{- printf "%s-%s" (include "siembol.fullname" .) $name | trunc 63 | trimSuffix "-" -}} +{{- printf "%s" $name | trunc 63 | trimSuffix "-" -}} {{- end -}} {{/* @@ -79,7 +74,7 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this */}} {{- define "siembol.enrichment_store.fullname" -}} {{- $name := default .Chart.Name .Values.enrichment_store.appName -}} -{{- printf "%s-%s" (include "siembol.fullname" .) $name | trunc 63 | trimSuffix "-" -}} +{{- printf "%s" $name | trunc 63 | trimSuffix "-" -}} {{- end -}} {{/* @@ -124,8 +119,10 @@ Set the ZooKeeper server for the siembol chart {{- end -}} {{/* -Set the name for the Kafka chart +Create a fully qualified siembol monitoring fullname. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). */}} -{{- define "kafka.fullname" -}} -{{- printf "%s-%s" .Release.Name "kafka" | trunc 63 | trimSuffix "-" -}} +{{- define "siembol.siembol_monitoring.fullname" -}} +{{- $name := default .Chart.Name .Values.siembol_monitoring.appName -}} +{{- printf "%s" $name | trunc 63 | trimSuffix "-" -}} {{- end -}} diff --git a/deployment/helm-k8s/templates/siembol_monitoring/deployment.yaml b/deployment/helm-k8s/templates/siembol_monitoring/deployment.yaml new file mode 100644 index 000000000..bb1027848 --- /dev/null +++ b/deployment/helm-k8s/templates/siembol_monitoring/deployment.yaml @@ -0,0 +1,61 @@ +{{- if (has "siembol_monitoring" .Values.enabled_apps) -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: {{ include "siembol.siembol_monitoring.fullname" $ }} + name: {{ include "siembol.siembol_monitoring.fullname" $ }} + namespace: {{ .Values.namespace }} +spec: + replicas: 1 + selector: + matchLabels: + app: {{ include "siembol.siembol_monitoring.fullname" $ }} + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + type: RollingUpdate + template: + metadata: + labels: + app: {{ include "siembol.siembol_monitoring.fullname" $ }} + name: {{ include "siembol.siembol_monitoring.fullname" $ }} + spec: + containers: + - image: {{ .Values.siembol_monitoring.image.repository }}:{{ .Values.siembol_monitoring.image.tag }} + imagePullPolicy: {{ .Values.siembol_monitoring.image.pullPolicy }} + name: {{ include "siembol.siembol_monitoring.fullname" $ }} + resources: + requests: + memory: {{ .Values.siembol_monitoring.resources.requests.memory }} + cpu: {{ .Values.siembol_monitoring.resources.requests.cpu }} + limits: + memory: {{ .Values.siembol_monitoring.resources.limits.memory }} + cpu: {{ .Values.siembol_monitoring.resources.limits.cpu }} + securityContext: + runAsUser: 101 + readinessProbe: + httpGet: + path: /health + port: {{ .Values.siembol_monitoring.service.port }} + scheme: HTTP + initialDelaySeconds: {{ .Values.siembol_monitoring.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.siembol_monitoring.readinessProbe.periodSeconds }} + livenessProbe: + httpGet: + path: /health + port: {{ .Values.siembol_monitoring.service.port }} + scheme: HTTP + volumeMounts: + - name: cacerts + mountPath: /etc/ssl/certs/java/cacerts + subPath: cacerts + readOnly: true + restartPolicy: Always + volumes: + - name: cacerts + secret: + secretName: cacerts + terminationGracePeriodSeconds: 300 +{{- end -}} \ No newline at end of file diff --git a/deployment/helm-k8s/templates/siembol_monitoring/service.yaml b/deployment/helm-k8s/templates/siembol_monitoring/service.yaml new file mode 100644 index 000000000..c386add9d --- /dev/null +++ b/deployment/helm-k8s/templates/siembol_monitoring/service.yaml @@ -0,0 +1,16 @@ +{{- if (has "siembol_monitoring" .Values.enabled_apps) -}} +kind: Service +apiVersion: v1 +metadata: + name: {{ include "siembol.siembol_monitoring.fullname" $ }} + namespace: {{ .Values.namespace }} + labels: + metrics: "true" +spec: + selector: + app: {{ include "siembol.siembol_monitoring.fullname" $ }} + ports: + - protocol: TCP + port: {{ .Values.siembol_monitoring.service.port }} + name: web +{{- end -}} \ No newline at end of file diff --git a/deployment/helm-k8s/values.yaml b/deployment/helm-k8s/values.yaml index 5757eebb2..7b53a8219 100644 --- a/deployment/helm-k8s/values.yaml +++ b/deployment/helm-k8s/values.yaml @@ -14,6 +14,7 @@ enabled_apps: - manager - dep_ingresses - enrichment_store +- siembol_monitoring # Enable certmanager certmanager: @@ -237,3 +238,31 @@ response: livenessProbe: initialDelaySeconds: 100 periodSeconds: 10 + +# ----------------------------------------------------------------------------- +# siembol monitoring parameters +# ----------------------------------------------------------------------------- +siembol_monitoring: + appName: "siembol-monitoring" + image: + repository: "gresearchdev/siembol-siembol-monitoring" + tag: "latest" + pullPolicy: Always + + service: + port: 8083 + + resources: + requests: + cpu: 0.1 + memory: "128Mi" + limits: + cpu: 0.5 + memory: "256Mi" + + readinessProbe: + initialDelaySeconds: 20 + periodSeconds: 30 + livenessProbe: + initialDelaySeconds: 30 + periodSeconds: 30 diff --git a/deployment/quickstart_install/ps-scripts/demoInstall.ps1 b/deployment/quickstart_install/ps-scripts/demoInstall.ps1 index 3851596b6..9dff8dc32 100644 --- a/deployment/quickstart_install/ps-scripts/demoInstall.ps1 +++ b/deployment/quickstart_install/ps-scripts/demoInstall.ps1 @@ -1,5 +1,5 @@ $CONFIG_MAP_NAME_GIT="github-details" -$GIT_SECRET_NAME="siembol-config-editor-rest-secrets" +$GIT_SECRET_NAME="config-editor-rest-secrets" $NAMESPACE="siembol" function Git-Details { @@ -42,4 +42,4 @@ if ($zookeeper_status -eq 'True') { } else { Write-Output "Zookeeper pod is not running yet, please try again in a few seconds" exit 1 -} \ No newline at end of file +} diff --git a/deployment/quickstart_install/sh-scripts/demoInstall.sh b/deployment/quickstart_install/sh-scripts/demoInstall.sh index cf56357cd..10dbe3383 100755 --- a/deployment/quickstart_install/sh-scripts/demoInstall.sh +++ b/deployment/quickstart_install/sh-scripts/demoInstall.sh @@ -1,7 +1,7 @@ #!/bin/bash CONFIG_MAP_NAME_GIT="github-details" -GIT_SECRET_NAME="siembol-config-editor-rest-secrets" +GIT_SECRET_NAME="config-editor-rest-secrets" NAMESPACE="siembol" git_details () { diff --git a/docs/deployment/deployment.md b/docs/deployment/deployment.md index 833442c19..0a4446ae5 100644 --- a/docs/deployment/deployment.md +++ b/docs/deployment/deployment.md @@ -18,6 +18,7 @@ Docker images are built both from snapshots and releases. - [storm-topology-manager](https://hub.docker.com/r/gresearchdev/siembol-storm-topology-manager/) - [config-editor-rest](https://hub.docker.com/r/gresearchdev/siembol-config-editor-rest/) - [responding-stream](https://hub.docker.com/r/gresearchdev/siembol-responding-stream/) + - [siembol-monitoring](https://hub.docker.com/r/gresearchdev/siembol-monitoring/) - Config editor UI - A Single page Angular application @@ -160,6 +161,17 @@ We have a folder for ingress specifications for extra components which allows fo | `dep_ingresses.storm.oauth2_proxy.enabled` | Enable oauth2 proxy for this ingress | false | | `dep_ingresses.storm.oauth2_proxy.host` | Host for oauth2 proxy | oauth-proxy.siembol.local | +#### Siembol Monitoring +Siembol monitoring is a springboot application with components to monitor Siembol. It has one component: siembol heartbeat. + +##### Siembol Heartbeat +Siembol heartbeat can be used to monitor all components of Siembol are working correctly. It has two main components: +- kafka producers: send a heartbeat message to a kafka topic at an interval of time specified in the config. Multiple producers can be defined with different kafka properties, e.g. different kafka cluster or topics. +- kafka consumer: reads the heartbeat message after it has been processed by Siembol services. Calculates the total latency and latency between services depending on the services enabled. These metrics are exposed and ready to be scraped by Prometheus. + +For the heartbeat to work config has to be added to each Siembol service type used to process the heartbeat message and write it to the topic read by the consumer. + +See [siembol monitoring](how-tos/how_to_setup_siembol_monitoring.md) for configuration details. ### Enable & Disable components @@ -172,6 +184,7 @@ By default the enabled_apps list consists of these components: - manager - dep_ingresses - enrichment_store +- siembol_monitoring ``` Any component can be removed by removing it from the list in [values.yaml](../../deployment/helm-k8s/values.yaml) or you can add another component such as `response` e.g. ```bash diff --git a/docs/deployment/how-tos/how_to_setup_siembol_monitoring.md b/docs/deployment/how-tos/how_to_setup_siembol_monitoring.md new file mode 100644 index 000000000..254c7347d --- /dev/null +++ b/docs/deployment/how-tos/how_to_setup_siembol_monitoring.md @@ -0,0 +1,41 @@ +# How to set up Siembol monitoring +Siembol monitoring is currently composed of one component: Siembol heartbeat. + +## Siembol Heartbeat +Siembol heartbeat is a springboot application with two main components: kafka producers and a kafka consumer. The kafka producers send heartbeat messages to kafka at an interval of time. The kafka consumer reads the messages after they have been processed by various Siembol services and calculate total latency and latency between Siembol services. + +The properties of the heartbeat are defined in its `application.properties` file. + +### General properties +- the interval in seconds the producers will send heartbeat messages, the default is 60 seconds +```properties +siembol-monitoring.heartbeat-properties.heartbeat-interval-seconds=60 +``` +- any optional additional fields to add to the heartbeat message +```properties +siembol-monitoring.heartbeat-properties.message.key1=value1 +siembol-monitoring.heartbeat-properties.message.key2=value2 +``` + +### Producer properties +One or multiple producers can be defined in the properties to monitor data from different kafka clusters, for example + +```properties +siembol-monitoring.heartbeat-properties.heartbeat-producers.local-kafka-cluster.output-topic=siembol.heartbeat +siembol-monitoring.heartbeat-properties.heartbeat-producers.local-kafka-cluster.kafka-properties.[bootstrap.servers]=kafka-0.kafka-headless.siembol.svc.cluster.local:9092 +siembol-monitoring.heartbeat-properties.heartbeat-producers.local-kafka-cluster.kafka-properties.[security.protocol]=PLAINTEXT +``` +Any arbitrary additional kafka producer properties can be added (https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html). + +### Consumer properties +```properties +siembol-monitoring.heartbeat-properties.heartbeat-consumer.kafka-properties.[bootstrap.servers]=kafka-0.kafka-headless.siembol.svc.cluster.local:9092 +siembol-monitoring.heartbeat-properties.heartbeat-consumer.kafka-properties.[application.id]=siembol.heartbeat.reader +siembol-monitoring.heartbeat-properties.heartbeat-consumer.kafka-properties.[auto.offset.reset]=earliest +siembol-monitoring.heartbeat-properties.heartbeat-consumer.kafka-properties.[security.protocol]=PLAINTEXT +siembol-monitoring.heartbeat-properties.heartbeat-consumer.enabled-services=parsingapp,enrichment,response +``` +Any arbitrary additional kafka streams properties can be added (https://kafka.apache.org/10/documentation/streams/developer-guide/config-streams.html). +The `enabled-services` property is to specify between which Siembol services latency should be computed. + +