release v0.11.8

crs4 · Nov 21, 2023 · ac0601e · ac0601e
2 parents 7ae2de2 + 251445b
commit ac0601e
Show file tree

Hide file tree

Showing 39 changed files with 5,231 additions and 1,130 deletions.
diff --git a/Makefile b/Makefile
@@ -139,7 +139,9 @@ lifemonitor: docker/lifemonitor.Dockerfile certs app.py gunicorn.conf.py ## Buil
 		printf "\n$(yellow)WARNING: $(bold)Skip build of LifeMonitor Docker image !!! $(reset)\n" ; \
 	else \
 		printf "\n$(bold)Building LifeMonitor Docker image...$(reset)\n" ; \
-		$(build_kit) docker $(build_cmd) $(cache_from_opt) $(cache_to_opt) \
+		$(build_kit) docker $(build_cmd) \
+			--build-arg USER_ID=$(id -u) --build-arg GROUP_ID=$(id -g) \
+			$(cache_from_opt) $(cache_to_opt) \
 			${sw_version_arg} ${build_number_arg} ${tags_opt} ${labels_opt} ${platforms_opt} \
 			-f docker/lifemonitor.Dockerfile -t crs4/lifemonitor . ;\
 		printf "$(done)\n" ; \
@@ -221,20 +223,34 @@ start-dev: images compose-files dev reset_compose permissions ## Start LifeMonit
 start-testing: compose-files aux_images ro_crates images reset_compose permissions ## Start LifeMonitor in a Testing environment
 	@printf "\n$(bold)Starting testing services...$(reset)\n" ; \
 	base=$$(if [[ -f "docker-compose.yml" ]]; then echo "-f docker-compose.yml"; fi) ; \
-	echo "$$(USER_UID=$$(id -u) USER_GID=$$(id -g) \
-	         $(docker_compose) $${base} \
-                   -f docker-compose.extra.yml \
-				   -f docker-compose.base.yml \
-				   -f docker-compose.monitoring.yml \
-				   -f docker-compose.dev.yml \
-				   -f docker-compose.test.yml \
-				   config)" > docker-compose.yml \
+	echo "$$($(docker_compose) $${base} \
+				-f docker-compose.extra.yml \
+				-f docker-compose.base.yml \
+				-f docker-compose.monitoring.yml \
+				-f docker-compose.dev.yml \
+				-f docker-compose.test.yml \
+				config)" > docker-compose.yml \
 	&& cp {,.test.}docker-compose.yml \
 	&& $(docker_compose) -f docker-compose.yml up -d db lmtests seek jenkins webserver worker ws_server ;\
 	$(docker_compose) -f ./docker-compose.yml \
 		exec -T lmtests /bin/bash -c "tests/wait-for-seek.sh 600"; \
 	printf "$(done)\n"
 
+start-maintenance: compose-files aux_images ro_crates images reset_compose permissions ## Start LifeMonitor in a Testing environment
+	@printf "\n$(bold)Starting testing services...$(reset)\n" ; \
+	base=$$(if [[ -f "docker-compose.yml" ]]; then echo "-f docker-compose.yml"; fi) ; \
+	echo "$$(USER_UID=$$(id -u) USER_GID=$$(id -g) \
+				$(docker_compose) $${base} \
+				-f docker-compose.extra.yml \
+				-f docker-compose.base.yml \
+				-f docker-compose.monitoring.yml \
+				-f docker-compose.dev.yml \
+				-f docker-compose.maintenance.yml \
+				config)" > docker-compose.yml \
+	&& cp {,.maintenance.}docker-compose.yml \
+	&& $(docker_compose) -f docker-compose.yml up -d db redis lm ws_server nginx console ;\
+	printf "$(done)\n"
+
 start-nginx: certs docker-compose.base.yml permissions ## Start a nginx front-end proxy for the LifeMonitor back-end
 	@printf "\n$(bold)Starting nginx proxy...$(reset)\n" ; \
 	base=$$(if [[ -f "docker-compose.yml" ]]; then echo "-f docker-compose.yml"; fi) ; \
@@ -262,7 +278,6 @@ start-aux-services: aux_images ro_crates docker-compose.extra.yml permissions ##
 
 run-tests: start-testing ## Run all tests in the Testing Environment
 	@printf "\n$(bold)Running tests...$(reset)\n" ; \
-	USER_UID=$$(id -u) USER_GID=$$(id -g) \
 	$(docker_compose) exec -T lmtests /bin/bash -c "pytest --durations=10 --color=yes tests"
 
 

diff --git a/docker-compose.monitoring.yml b/docker-compose.monitoring.yml
@@ -2,7 +2,7 @@ version: "3.5"
 
 services:
   prometheus:
-    image: prom/prometheus:v2.24.1
+    image: prom/prometheus:v2.48.0
     ports:
       - "9090:9090"
     volumes:

diff --git a/docker/lifemonitor.Dockerfile b/docker/lifemonitor.Dockerfile
@@ -8,8 +8,18 @@ RUN apt-get update -q \
         postgresql-client-11 default-jre \
  && apt-get clean -y && rm -rf /var/lib/apt/lists
 
+# Set the parametric USER ID
+ARG USER_ID
+ENV USER_ID=${USER_ID:-1000}
+
+# Set the parametric GROUP ID
+ARG GROUP_ID
+ENV GROUP_ID=${GROUP_ID:-1000}
+
 # Create a user 'lm' with HOME at /lm and set 'lm' as default git user
-RUN useradd -d /lm -m lm
+RUN groupadd -g ${GROUP_ID} lm && \
+    useradd -u ${USER_ID} -g lm -d /lm -m lm
+
 # Set the default user
 ENV USER=lm
 
@@ -67,6 +77,7 @@ RUN mkdir -p /var/data/lm \
     && chown -R lm:lm /var/data/lm \
     && ln -s /var/data/lm /lm/data \
     && chown -R lm:lm /lm/data \
+    && mkdir -p /var/log/lm && chown -R lm:lm /var/log/lm \
     && mkdir /lm/.nextflow && chmod -R 777 /lm/.nextflow
 
 # Set the default user

diff --git a/docker/worker_entrypoint.sh b/docker/worker_entrypoint.sh
@@ -84,7 +84,6 @@ while : ; do
     ${threads:-} \
     lifemonitor.tasks.worker:broker lifemonitor.tasks ${queues}
   exit_code=$?
-  exit_code=$?
   if [[ $exit_code == 3 ]]; then
     log "dramatiq worker could not connect to message broker (exit code ${exit_code})" 
     log "Restarting..."

diff --git a/k8s/Chart.yaml b/k8s/Chart.yaml
@@ -7,12 +7,12 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.10.0
+version: 0.11.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
-appVersion: 0.11.7
+appVersion: 0.11.8
 
 # Chart dependencies
 dependencies:

diff --git a/k8s/backup-key.secret.yaml b/k8s/backup-key.secret.yaml
@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: lifemonitor-api-backup-key
+type: Opaque
+data:
+  encryptionKey: <base64-encoded-encryption-key>
diff --git a/k8s/templates/_helpers.tpl b/k8s/templates/_helpers.tpl
@@ -67,6 +67,13 @@ Define lifemonitor TLS secret name
 {{- printf "%s-tls" .Release.Name }}
 {{- end }}
 
+{{/*
+Define lifemonitor secret name for backup key
+*/}}
+{{- define "chart.lifemonitor.backup.key" -}}
+{{- printf "%s-backup-key" .Release.Name }}
+{{- end }}
+
 
 {{/*
 Define volume name of LifeMonitor backup data 
@@ -167,6 +174,8 @@ Define mount points shared by some pods.
 - mountPath: "/lm/certs/"
   name: lifemonitor-tls
   readOnly: true
+- name: lifemonitor-logs
+  mountPath: "/var/log"
 - name: lifemonitor-settings
   mountPath: "/lm/settings.conf"
   subPath: settings.conf

diff --git a/k8s/templates/backend.deployment.yaml b/k8s/templates/backend.deployment.yaml
@@ -54,14 +54,20 @@ spec:
           {{- include "lifemonitor.common-env" . | nindent 12 }}
           volumeMounts:
           {{- include "lifemonitor.common-volume-mounts" . | nindent 12 }}
+          resources:
+            {{- toYaml .Values.lifemonitor.initContainers.initBackend.resources | nindent 12 }}
       containers:
         - name: backend
           securityContext:
             {{- toYaml .Values.lifemonitor.securityContext | nindent 12 }}
           image: {{ include "chart.lifemonitor.image" . }}
           imagePullPolicy: {{ .Values.lifemonitor.imagePullPolicy }}
           env:
-          {{- include "lifemonitor.common-env" . | nindent 12 }}
+          {{- include "lifemonitor.common-env" . | nindent 12 }}          
+            {{- if  .Values.maintenanceMode.enabled }}
+            - name: FLASK_ENV
+              value: "maintenance"
+            {{- end }}
           volumeMounts:
           {{- include "lifemonitor.common-volume-mounts" . | nindent 12 }}
           ports:
@@ -85,8 +91,27 @@ spec:
             periodSeconds: 3
           resources:
             {{- toYaml .Values.lifemonitor.resources | nindent 12 }}
+        {{- if .Values.monitoring.loki.enabled }}
+        - name: log-exporter
+          image: {{ .Values.monitoring.loki.exporter.image }}
+          imagePullPolicy: {{ $.Values.monitoring.loki.exporter.imagePullPolicy }}
+          args:
+            - "-config.file=/etc/promtail/promtail.yaml" # Found in the ConfigMap
+          resources:
+            {{- toYaml $.Values.monitoring.loki.exporter.resources | nindent 12 }}
+          volumeMounts:
+            - name: promtail-config
+              mountPath: /etc/promtail
+            - name: lifemonitor-logs
+              mountPath: /var/log
+        {{- end }}
       volumes:
       {{- include "lifemonitor.common-volume" . | nindent 8 }}
+      {{- if .Values.monitoring.loki.enabled }}
+        - name: promtail-config
+          configMap:
+            name: "{{.Release.Name}}-promtail-backend-configmap"
+      {{- end }}
       {{- with .Values.lifemonitor.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}

diff --git a/k8s/templates/backup.job.yaml b/k8s/templates/backup.job.yaml
@@ -29,12 +29,22 @@ spec:
             {{- include "lifemonitor.common-volume-mounts" . | nindent 12 }}
             - name: lifemonitor-backup
               mountPath: "/var/data/backup"
+            {{- if .Values.backup.encryptionKeySecret }}
+            - name: lifemonitor-backup-encryption-key
+              mountPath: "/lm/backup/encryption.key"
+              subPath: encryptionKey
+            {{- end }}
           restartPolicy: OnFailure
           volumes:
             {{- include "lifemonitor.common-volume" . | nindent 10 }}
           - name: lifemonitor-backup
             persistentVolumeClaim:
               claimName: {{ .Values.backup.existingClaim }}
+          {{- if .Values.backup.encryptionKeySecret }}
+          - name: lifemonitor-backup-encryption-key
+            secret:
+              secretName: {{ .Values.backup.encryptionKeySecret }}
+          {{- end }}
           {{- with .Values.lifemonitor.nodeSelector }}
           nodeSelector:
             {{- toYaml . | nindent 10 }}

diff --git a/k8s/templates/console.deployment.yaml b/k8s/templates/console.deployment.yaml
@@ -0,0 +1,73 @@
+{{- if or (.Values.maintenanceMode.enabled) (.Values.console.enabled ) }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "chart.fullname" . }}-console
+  labels:
+    {{- include "chart.labels" . | nindent 4 }}
+spec:
+  {{- if not .Values.lifemonitor.autoscaling.enabled }}
+  replicas: {{ .Values.lifemonitor.replicaCount }}
+  {{- end }}
+  selector:
+    matchLabels:
+      {{- include "chart.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      annotations:
+        checksum/settings: {{ include (print $.Template.BasePath "/settings.secret.yaml") . | sha256sum }}
+      {{- with .Values.lifemonitor.podAnnotations }}
+        {{- toYaml . | nindent 8 }}
+      {{- end }}       
+      labels:
+        {{- include "chart.selectorLabels" . | nindent 8 }}
+    spec:
+      {{- with .Values.lifemonitor.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      serviceAccountName: {{ include "chart.serviceAccountName" . }}
+      securityContext:
+        {{- toYaml .Values.lifemonitor.podSecurityContext | nindent 8 }}
+      containers:
+        - name: app
+          securityContext:
+            {{- toYaml .Values.lifemonitor.securityContext | nindent 12 }}
+          image: {{ include "chart.lifemonitor.image" . }}
+          imagePullPolicy: {{ .Values.lifemonitor.imagePullPolicy }}
+          command: ["/bin/sh","-c"]
+          args: ["wait-for-redis.sh && wait-for-postgres.sh && sleep infinity"]
+          env:
+          {{- include "lifemonitor.common-env" . | nindent 12 }}
+            - name: LIFEMONITOR_MAINTENANCE_MODE
+              value: {{ .Values.maintenanceMode.enabled | quote }}
+            - name: LIFEMONITOR_CONSOLE_ENABLED
+              value: {{ .Values.console.enabled | quote }}
+          volumeMounts:
+          {{- include "lifemonitor.common-volume-mounts" . | nindent 12 }}
+            - name: lifemonitor-backup
+              mountPath: "/var/data/backup"
+          ports:
+            - name: http
+              containerPort: 8000
+              protocol: TCP         
+          resources:
+            {{- toYaml .Values.lifemonitor.resources | nindent 12 }}
+      volumes:
+        - name: lifemonitor-backup
+          persistentVolumeClaim:
+            claimName: {{ .Values.backup.existingClaim }}
+      {{- include "lifemonitor.common-volume" . | nindent 8 }}
+      {{- with .Values.lifemonitor.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.lifemonitor.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.lifemonitor.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+{{- end }}
diff --git a/k8s/templates/data-logs.pvc.yaml b/k8s/templates/data-logs.pvc.yaml