From 104dca01a2a9cb5f7ab07e0024afb355e2580233 Mon Sep 17 00:00:00 2001 From: Yossi Tamari Date: Wed, 28 Feb 2024 15:45:10 -0800 Subject: [PATCH 1/2] fix: redis package version upgrade --- shared/src/EMBC.Utilities.Caching/Cache.cs | 2 +- shared/src/EMBC.Utilities.Hosting/EMBC.Utilities.Hosting.csproj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/shared/src/EMBC.Utilities.Caching/Cache.cs b/shared/src/EMBC.Utilities.Caching/Cache.cs index dbff482dc..7b8fc6d7e 100644 --- a/shared/src/EMBC.Utilities.Caching/Cache.cs +++ b/shared/src/EMBC.Utilities.Caching/Cache.cs @@ -83,7 +83,7 @@ public async Task Refresh(string key, Func> getter, TimeSpan expirati } } - private static T? Deserialize(byte[] data) => data == null || data.Length == 0 ? default(T?) : JsonSerializer.Deserialize(data); + private static T? Deserialize(byte[]? data) => data == null || data.Length == 0 ? default(T?) : JsonSerializer.Deserialize(data); private static byte[] Serialize(T obj) => obj == null ? Array.Empty() : JsonSerializer.SerializeToUtf8Bytes(obj); } diff --git a/shared/src/EMBC.Utilities.Hosting/EMBC.Utilities.Hosting.csproj b/shared/src/EMBC.Utilities.Hosting/EMBC.Utilities.Hosting.csproj index 46467e782..6e62c735e 100644 --- a/shared/src/EMBC.Utilities.Hosting/EMBC.Utilities.Hosting.csproj +++ b/shared/src/EMBC.Utilities.Hosting/EMBC.Utilities.Hosting.csproj @@ -46,7 +46,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive From 86a9acd583e78af70510ce4ee16d5ce1b2b30fdc Mon Sep 17 00:00:00 2001 From: Yossi Tamari Date: Wed, 28 Feb 2024 15:45:36 -0800 Subject: [PATCH 2/2] task: added redis sentinel chart --- tools/helm/.gitignore | 3 + tools/helm/era/Chart.yaml | 7 + tools/helm/era/README.md | 6 + tools/helm/redis-sentinel/Chart.yaml | 6 + .../redis-sentinel/templates/_helpers.tpl | 11 ++ .../redis-sentinel/templates/configmap.yml | 87 +++++++++++ .../helm/redis-sentinel/templates/netpol.yml | 29 ++++ tools/helm/redis-sentinel/templates/pdb.tpl | 11 ++ .../helm/redis-sentinel/templates/secret.yml | 9 ++ .../helm/redis-sentinel/templates/service.yml | 14 ++ .../redis-sentinel/templates/statefulset.yml | 145 ++++++++++++++++++ tools/helm/redis-sentinel/values.yaml | 17 ++ 12 files changed, 345 insertions(+) create mode 100644 tools/helm/.gitignore create mode 100644 tools/helm/redis-sentinel/Chart.yaml create mode 100644 tools/helm/redis-sentinel/templates/_helpers.tpl create mode 100644 tools/helm/redis-sentinel/templates/configmap.yml create mode 100644 tools/helm/redis-sentinel/templates/netpol.yml create mode 100644 tools/helm/redis-sentinel/templates/pdb.tpl create mode 100644 tools/helm/redis-sentinel/templates/secret.yml create mode 100644 tools/helm/redis-sentinel/templates/service.yml create mode 100644 tools/helm/redis-sentinel/templates/statefulset.yml create mode 100644 tools/helm/redis-sentinel/values.yaml diff --git a/tools/helm/.gitignore b/tools/helm/.gitignore new file mode 100644 index 000000000..bd3075b16 --- /dev/null +++ b/tools/helm/.gitignore @@ -0,0 +1,3 @@ +envs +**/charts +**/Chart.lock diff --git a/tools/helm/era/Chart.yaml b/tools/helm/era/Chart.yaml index c2d9fb9f5..000b1f10b 100644 --- a/tools/helm/era/Chart.yaml +++ b/tools/helm/era/Chart.yaml @@ -4,3 +4,10 @@ description: A Helm chart for ERA type: application version: 1.0.0 appVersion: "1.0.0" + +dependencies: + - name: redis-sentinel + alias: redis + version: 1.0.0 + condition: redis.enabled + repository: "file://../redis-sentinel" diff --git a/tools/helm/era/README.md b/tools/helm/era/README.md index 14d06b520..ac9057841 100644 --- a/tools/helm/era/README.md +++ b/tools/helm/era/README.md @@ -4,6 +4,12 @@ This directory contains a Helm chart to deploy ERA ## Usage +To build and update the chart's dependencies, run the following command: + +```sh +helm dep update +``` + To install a new environment, ensure the values.yaml matches the environment, then run the following command: ```sh diff --git a/tools/helm/redis-sentinel/Chart.yaml b/tools/helm/redis-sentinel/Chart.yaml new file mode 100644 index 000000000..2ef7bce66 --- /dev/null +++ b/tools/helm/redis-sentinel/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: redis-sentinel +description: A Helm chart to deploy Redis with Sentinel support into OpenShift +type: application +version: 1.0.0 +appVersion: "7.0" diff --git a/tools/helm/redis-sentinel/templates/_helpers.tpl b/tools/helm/redis-sentinel/templates/_helpers.tpl new file mode 100644 index 000000000..7364a57b6 --- /dev/null +++ b/tools/helm/redis-sentinel/templates/_helpers.tpl @@ -0,0 +1,11 @@ +/* standard labels */ +{{- define "standard.labels" -}} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/instance: {{ .Release.Name }} +helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} +{{- end -}} + +/* standard name */ +{{- define "standard.name" -}} +{{ print .Release.Name "-" .Chart.Name }} +{{- end -}} diff --git a/tools/helm/redis-sentinel/templates/configmap.yml b/tools/helm/redis-sentinel/templates/configmap.yml new file mode 100644 index 000000000..bd9788668 --- /dev/null +++ b/tools/helm/redis-sentinel/templates/configmap.yml @@ -0,0 +1,87 @@ +{{- $name := include "standard.name" . -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $name }}-config + labels: {{ include "standard.labels" . | nindent 4 }} +data: + REDIS_NODES: "{{ $name }}-0.{{ $name }}-svc,{{ $name }}-1.{{ $name }}-svc,{{ $name }}-2.{{ $name }}-svc" + redis.conf: | + appendonly no + masteruser default + protected-mode no + save 900 1 + save 300 10 + save 60 10000 + sentinel_init.sh: | + #!/bin/bash + for i in ${REDIS_NODES//,/ } + do + echo "finding master at $i" + MASTER=$(redis-cli --no-auth-warning --raw -h $i -a ${REDIS_PASSWORD} info replication | awk '{print $1}' | grep master_host: | cut -d ":" -f2) + + if [ "${MASTER}" == "" ]; then + echo "no master found, defaulting to ${NODE_NAME_PREFIX}-0" + MASTER=${NODE_NAME_PREFIX}-0.${REDIS_SERVICE} + else + echo "found ${MASTER}" + break + fi + + done + echo "sentinel monitor ${SENTINEL_SET} ${MASTER} ${REDIS_PORT} 2" >> /tmp/master + echo "port ${SENTINEL_PORT} + sentinel resolve-hostnames yes + sentinel announce-hostnames yes + $(cat /tmp/master) + sentinel down-after-milliseconds ${SENTINEL_SET} 1000 + sentinel failover-timeout ${SENTINEL_SET} 10000 + sentinel parallel-syncs ${SENTINEL_SET} 1 + sentinel sentinel-pass ${REDIS_PASSWORD} + sentinel auth-pass ${SENTINEL_SET} ${REDIS_PASSWORD} + requirepass ${REDIS_PASSWORD} + sentinel announce-ip ${HOSTNAME}.${REDIS_SERVICE} + sentinel announce-port ${SENTINEL_PORT} + " > /etc/redis/sentinel.conf + cat /etc/redis/sentinel.conf + redis_init.sh: | + #!/bin/bash + cp /tmp/redis/redis.conf /etc/redis/redis.conf + echo "requirepass ${REDIS_PASSWORD}" >> /etc/redis/redis.conf + echo "masterauth ${REDIS_PASSWORD}" >> /etc/redis/redis.conf + echo "replica-announce-ip ${HOSTNAME}.${REDIS_SERVICE}" >> /etc/redis/redis.conf + echo "replica-announce-port ${REDIS_PORT} " >> /etc/redis/redis.conf + + echo "finding master..." + if [ "$(timeout 5 redis-cli -h ${REDIS_SERVICE} -p ${SENTINEL_PORT} -a ${REDIS_PASSWORD} ping)" != "PONG" ]; then + echo "sentinel not found, defaulting to ${NODE_NAME_PREFIX}-0" + if [ ${HOSTNAME} == "${NODE_NAME_PREFIX}-0" ]; then + echo "this is ${NODE_NAME_PREFIX}-0, not updating config..." + else + echo "updating redis.conf..." + echo "repl-ping-replica-period 3" >> /etc/redis/redis.conf + echo "slave-read-only no" >> /etc/redis/redis.conf + echo "slaveof ${NODE_NAME_PREFIX}-0.${REDIS_SERVICE} ${REDIS_PORT}" >> /etc/redis/redis.conf + fi + else + echo "sentinel found, finding master" + MASTER=$(redis-cli -h ${REDIS_SERVICE} -p ${SENTINEL_PORT} -a ${REDIS_PASSWORD} sentinel get-master-addr-by-name ${SENTINEL_SET} | grep -E "(^${NODE_NAME_PREFIX}-*)|([0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3})") + if [ "${HOSTNAME}.${REDIS_SERVICE}" == ${MASTER} ]; then + echo "this is master, not updating config..." + else + echo "master found : ${MASTER}, updating redis.conf" + echo "slave-read-only no" >> /etc/redis/redis.conf + echo "slaveof ${MASTER} ${REDIS_PORT}" >> /etc/redis/redis.conf + echo "repl-ping-replica-period 3" >> /etc/redis/redis.conf + fi + fi + cat /etc/redis/redis.conf + init.sh: | + #!/bin/bash + /scripts/redis_init.sh + /scripts/sentinel_init.sh + start.sh: | + #!/bin/bash + redis-server /etc/redis/redis.conf & + redis-sentinel /etc/redis/sentinel.conf & + sleep infinity diff --git a/tools/helm/redis-sentinel/templates/netpol.yml b/tools/helm/redis-sentinel/templates/netpol.yml new file mode 100644 index 000000000..380e7a30f --- /dev/null +++ b/tools/helm/redis-sentinel/templates/netpol.yml @@ -0,0 +1,29 @@ +{{- $name := include "standard.name" . -}} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ $name }}-netpol + labels: {{ include "standard.labels" . | nindent 4 }} +spec: + podSelector: + matchLabels: + name: {{ $name }} + ingress: + - from: + - podSelector: + matchLabels: + name: {{ $name }} + ports: + - port: {{ .Values.redisPort }} + protocol: TCP + - port: {{ .Values.sentinelPort }} + protocol: TCP + - from: + - podSelector: + matchLabels: {} + ports: + - port: {{ .Values.redisPort }} + protocol: TCP + - port: {{ .Values.sentinelPort }} + protocol: TCP + \ No newline at end of file diff --git a/tools/helm/redis-sentinel/templates/pdb.tpl b/tools/helm/redis-sentinel/templates/pdb.tpl new file mode 100644 index 000000000..d3badba1f --- /dev/null +++ b/tools/helm/redis-sentinel/templates/pdb.tpl @@ -0,0 +1,11 @@ +{{- $name := include "standard.name" . -}} +kind: PodDisruptionBudget +apiVersion: policy/v1 +metadata: + name: {{ $name }}-pdb + labels: {{ include "standard.labels" . | nindent 4 }} +spec: + minAvailable: 1 + selector: + matchLabels: + name: {{ $name }} diff --git a/tools/helm/redis-sentinel/templates/secret.yml b/tools/helm/redis-sentinel/templates/secret.yml new file mode 100644 index 000000000..060931896 --- /dev/null +++ b/tools/helm/redis-sentinel/templates/secret.yml @@ -0,0 +1,9 @@ +{{- $name := include "standard.name" . -}} +kind: Secret +apiVersion: v1 +metadata: + name: {{ $name }}-secret + labels: {{ include "standard.labels" . | nindent 4 }} +type: Opaque +stringData: + REDIS_PASSWORD: {{ .Values.password | required "Values.password" | quote }} diff --git a/tools/helm/redis-sentinel/templates/service.yml b/tools/helm/redis-sentinel/templates/service.yml new file mode 100644 index 000000000..c9a42c28b --- /dev/null +++ b/tools/helm/redis-sentinel/templates/service.yml @@ -0,0 +1,14 @@ +{{- $name := include "standard.name" . -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ $name }}-svc + labels: {{ include "standard.labels" . | nindent 4 }} +spec: + type: ClusterIP + ports: + - port: {{ .Values.sentinelPort }} + targetPort: {{ .Values.sentinelPort }} + name: sentinel + selector: + name: {{ $name }} diff --git a/tools/helm/redis-sentinel/templates/statefulset.yml b/tools/helm/redis-sentinel/templates/statefulset.yml new file mode 100644 index 000000000..c9340f70e --- /dev/null +++ b/tools/helm/redis-sentinel/templates/statefulset.yml @@ -0,0 +1,145 @@ +{{- $name := include "standard.name" . -}} +{{- $image := print .Values.imageRepository ":" .Values.imageTag -}} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ $name }} + labels: {{ include "standard.labels" . | nindent 4 }} +spec: + serviceName: {{ $name }}-svc + replicas: 3 + selector: + matchLabels: + name: {{ $name }} + template: + metadata: + labels: {{ include "standard.labels" . | nindent 8 }} + name: {{ $name }} + spec: + initContainers: + - name: {{ $name }}-config + image: {{ $image }} + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 50m + memory: 64Mi + env: + - name: REDIS_NODES + valueFrom: + configMapKeyRef: + name: {{ $name }}-config + key: REDIS_NODES + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: {{ $name }}-secret + key: REDIS_PASSWORD + - name: REDIS_PORT + value: {{ .Values.redisPort | required "Values.redisPort" | quote }} + - name: SENTINEL_PORT + value: {{ .Values.sentinelPort | required "Values.sentinelPort" | quote }} + - name: REDIS_SERVICE + value: {{ $name }}-svc + - name: NODE_NAME_PREFIX + value: {{ $name }} + - name: SENTINEL_SET + value: {{ .Values.setName | default "mymaster" }} + command: ["sh", "-c", "/scripts/init.sh"] + volumeMounts: + - name: redis-config + mountPath: /etc/redis/ + - name: config + mountPath: /tmp/redis/ + - name: scripts + mountPath: /scripts/ + containers: + - name: {{ $name }} + image: {{ $image }} + lifecycle: + preStop: + exec: + command: + ["/bin/sh", "-c", "redis-cli -h $HOSTNAME -p {{ .Values.redisPort }} shutdown save && redis-cli -h $HOSTNAME -p {{ .Values.sentinelPort }} shutdown save"] + livenessProbe: + exec: + command: + [ + "/bin/sh", + "-c", + 'test "$(redis-cli -h $HOSTNAME ping)" = "PONG"', + ] + initialDelaySeconds: 15 + timeoutSeconds: 1 + failureThreshold: 3 + readinessProbe: + exec: + command: + [ + "/bin/sh", + "-c", + 'test "$(redis-cli -h $HOSTNAME ping)" = "PONG"', + ] + initialDelaySeconds: 10 + timeoutSeconds: 1 + failureThreshold: 3 + command: ["/scripts/start.sh"] + ports: + - containerPort: {{ .Values.redisPort }} + name: redis + - containerPort: {{ .Values.sentinelPort }} + name: sentinel + env: + - name: REDISCLI_AUTH + valueFrom: + secretKeyRef: + name: {{ $name }}-secret + key: REDIS_PASSWORD + volumeMounts: + - name: data + mountPath: /data + readOnly: false + - name: redis-config + mountPath: /etc/redis/ + - name: config + mountPath: /tmp/redis/ + - name: scripts + mountPath: /scripts/ + resources: {{ toYaml .Values.resources | nindent 12 }} + volumes: + - name: redis-config + emptyDir: {} + - name: scripts + configMap: + name: {{ $name }}-config + defaultMode: 0777 + items: + - key: redis_init.sh + path: redis_init.sh + - key: sentinel_init.sh + path: sentinel_init.sh + - key: start.sh + path: start.sh + - key: init.sh + path: init.sh + - name: config + configMap: + name: {{ $name }}-config + items: + - key: redis.conf + path: redis.conf + volumeClaimTemplates: + - kind: PersistentVolumeClaim + metadata: + name: data + annotations: + volume.beta.kubernetes.io/storage-class: {{ .Values.persistentVolumeClass }} + spec: + storageClassName: {{ .Values.persistentVolumeClass }} + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.persistentVolumeSize }} diff --git a/tools/helm/redis-sentinel/values.yaml b/tools/helm/redis-sentinel/values.yaml new file mode 100644 index 000000000..a31c0446c --- /dev/null +++ b/tools/helm/redis-sentinel/values.yaml @@ -0,0 +1,17 @@ +imageRepository: artifacts.developer.gov.bc.ca/docker-remote/redis +imageTag: 7 + +redisPort: 6379 +sentinelPort: 26379 +persistentVolumeSize: 1Gi +persistentVolumeClass: netapp-block-standard + +resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 250m + memory: 512Mi + +password: ~