From 0f09425c35b9dfa830fa5c15945acdc67a13d8b1 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Mon, 28 Oct 2024 14:50:34 +0700 Subject: [PATCH 01/16] Add CrdtMerge deployment for k8s and GHA --- .github/workflows/deploy-branch.yaml | 10 +- .github/workflows/deploy.yaml | 6 + .github/workflows/integration-test-gha.yaml | 11 +- .github/workflows/lexbox-crdtmerge.yaml | 98 +++++++++++++ .github/workflows/release-pipeline.yaml | 10 +- backend/CrdtMerge/Dockerfile | 25 ++++ backend/CrdtMerge/appsettings.json | 2 +- deployment/base/crdtmerge-deployment.yaml | 144 ++++++++++++++++++++ deployment/base/pvc.yaml | 18 +++ deployment/base/secrets.yaml | 10 ++ deployment/local-dev/crdtmerge-secrets.yaml | 8 ++ deployment/local-dev/kustomization.yaml | 1 + skaffold.yaml | 22 +++ 13 files changed, 361 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/lexbox-crdtmerge.yaml create mode 100644 backend/CrdtMerge/Dockerfile create mode 100644 deployment/base/crdtmerge-deployment.yaml create mode 100644 deployment/local-dev/crdtmerge-secrets.yaml diff --git a/.github/workflows/deploy-branch.yaml b/.github/workflows/deploy-branch.yaml index f8a199eff..8b1d9ca92 100644 --- a/.github/workflows/deploy-branch.yaml +++ b/.github/workflows/deploy-branch.yaml @@ -41,10 +41,18 @@ jobs: version: ${{ needs.set-version.outputs.version }} label-latest: false + build-crdtmerge: + name: Build CrdtMerge + needs: [ set-version ] + uses: ./.github/workflows/lexbox-crdtmerge.yaml + with: + version: ${{ needs.set-version.outputs.version }} + label-latest: false + deploy: name: Deploy Develop uses: ./.github/workflows/deploy.yaml - needs: [ build-api, build-ui, build-hgweb, set-version ] + needs: [ build-api, build-ui, build-hgweb, build-crdtmerge, set-version ] secrets: inherit with: version: ${{ needs.set-version.outputs.version }} diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index 0ab79e53e..f31011c4a 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -50,6 +50,7 @@ jobs: url: https://${{ inputs.deploy-domain }} outputs: api-version: ${{ steps.get-api-version.outputs.result }} + crdtmerge-version: ${{ steps.get-crdtmerge-version.outputs.result }} ui-version: ${{ steps.get-ui-version.outputs.result }} steps: - name: Checkout lexbox repo @@ -80,6 +81,11 @@ jobs: id: get-api-version with: cmd: yq '.images.[] | select(.name == "ghcr.io/sillsdev/lexbox-api").newTag' "fleet/${{ inputs.k8s-environment }}/kustomization.yaml" + - name: Get CrdtMerge version + uses: mikefarah/yq@0b34c9a00de1c575a34eea05af1d956a525c4fc1 # v4.34.2 + id: get-crdtmerge-version + with: + cmd: yq '.images.[] | select(.name == "ghcr.io/sillsdev/lexbox-crdtmerge").newTag' "fleet/${{ inputs.k8s-environment }}/kustomization.yaml" - name: Get UI version uses: mikefarah/yq@0b34c9a00de1c575a34eea05af1d956a525c4fc1 # v4.34.2 id: get-ui-version diff --git a/.github/workflows/integration-test-gha.yaml b/.github/workflows/integration-test-gha.yaml index 8161dcb22..c0ddb9636 100644 --- a/.github/workflows/integration-test-gha.yaml +++ b/.github/workflows/integration-test-gha.yaml @@ -39,7 +39,16 @@ jobs: uses: mikefarah/yq@0b34c9a00de1c575a34eea05af1d956a525c4fc1 # v4.34.2 with: cmd: yq eval -i '(.images.[] | select(.name == "ghcr.io/sillsdev/lexbox-api").newTag) = "${{ inputs.lexbox-api-tag }}"' "./deployment/gha/kustomization.yaml" - # It's also possible that hgweb and/or ui image may have changed; if so, pull them and update kustomization.yaml for them as well + # It's also possible that hgweb, crdtmerge, and/or ui image may have changed; if so, pull them and update kustomization.yaml for them as well + - name: Pull crdtmerge if updated + id: crdtmerge_image + continue-on-error: true + run: docker pull ghcr.io/sillsdev/lexbox-crdtmerge:${{ inputs.lexbox-api-tag }} + - name: Update image crdtmerge version + if: ${{ steps.crdtmerge_image.outcome == 'success' }} + uses: mikefarah/yq@0b34c9a00de1c575a34eea05af1d956a525c4fc1 # v4.34.2 + with: + cmd: yq eval -i '(.images.[] | select(.name == "ghcr.io/sillsdev/lexbox-crdtmerge").newTag) = "${{ inputs.lexbox-api-tag }}"' "./deployment/gha/kustomization.yaml" - name: Pull hgweb if updated id: hgweb_image continue-on-error: true diff --git a/.github/workflows/lexbox-crdtmerge.yaml b/.github/workflows/lexbox-crdtmerge.yaml new file mode 100644 index 000000000..a01a2163b --- /dev/null +++ b/.github/workflows/lexbox-crdtmerge.yaml @@ -0,0 +1,98 @@ +name: Build CrdtMerge + +# https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#on +on: + workflow_call: + inputs: + version: + description: 'The version of the image to build' + required: true + type: string + label-latest: + description: 'The label to apply to the latest image' + type: boolean + default: false + +env: + IMAGE_NAME: ghcr.io/sillsdev/lexbox-crdtmerge + + +jobs: + publish-api: + timeout-minutes: 60 + runs-on: ubuntu-latest + + # postgres db is for automated tests + # services: + # postgres: + # image: postgres:15-alpine + # env: + # POSTGRES_PASSWORD: 972b722e63f549938d07bd8c4ee5086c + # POSTGRES_DB: lexbox-tests + # # Set health checks to wait until postgres has started + # options: >- + # --health-cmd pg_isready + # --health-interval 10s + # --health-timeout 5s + # --health-retries 5 + # ports: + # # Maps tcp port 5432 on service container to the host + # - 5433:5432 + + env: + # https://docs.docker.com/develop/develop-images/build_enhancements/ + DOCKER_BUILDKIT: 1 + + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - uses: actions/setup-dotnet@v4 + with: + dotnet-version: '8.x' + - name: Dotnet build + run: dotnet build backend/CrdtMerge/CrdtMerge.csproj + # TODO: Write CrdtMerge unit tests, probably based on existing sync tests + # - name: Unit tests + # run: dotnet test backend/CrdtMerge/CrdtMerge.csproj --logger:"xunit;LogFileName={assembly}.results.xml" --results-directory ./test-results --filter "Category!=Integration&Category!=FlakyIntegration" --blame-hang-timeout 10m + # - name: Publish unit test results + # uses: EnricoMi/publish-unit-test-result-action@8885e273a4343cd7b48eaa72428dea0c3067ea98 # v2.14.0 + # if: always() + # with: + # check_name: C# Unit Tests + # files: ./test-results/*.xml + # - name: Upload test results + # if: always() + # uses: actions/upload-artifact@v4 + # with: + # name: dotnet-unit-test-results + # path: ./test-results + + - name: Docker meta + id: meta + if: ${{ !env.ACT }} + uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1 + with: + images: ${{ env.IMAGE_NAME }} + tags: | + type=ref,event=branch + type=ref,event=pr + type=raw,enable=${{ inputs.label-latest }},value=latest + type=raw,value=${{ inputs.version }} + + - name: ghcr.io login + uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3.1.0 + if: ${{ !env.ACT }} + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0 + with: + context: backend + build-args: | + APP_VERSION=${{ inputs.version }} + push: ${{ !env.ACT && github.repository == 'sillsdev/languageforge-lexbox' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/release-pipeline.yaml b/.github/workflows/release-pipeline.yaml index 92450db98..9f65e9e3c 100644 --- a/.github/workflows/release-pipeline.yaml +++ b/.github/workflows/release-pipeline.yaml @@ -43,10 +43,18 @@ jobs: version: ${{ needs.set-version.outputs.version }} label-latest: true + build-crdtmerge: + name: Build crdtmerge + needs: [ set-version ] + uses: ./.github/workflows/lexbox-crdtmerge.yaml + with: + version: ${{ needs.set-version.outputs.version }} + label-latest: true + deploy: name: Deploy Staging uses: ./.github/workflows/deploy.yaml - needs: [ build-api, build-ui, build-hgweb, set-version ] + needs: [ build-api, build-ui, build-hgweb, build-crdtmerge, set-version ] secrets: inherit with: version: ${{ needs.set-version.outputs.version }} diff --git a/backend/CrdtMerge/Dockerfile b/backend/CrdtMerge/Dockerfile new file mode 100644 index 000000000..0746776db --- /dev/null +++ b/backend/CrdtMerge/Dockerfile @@ -0,0 +1,25 @@ +# syntax=docker/dockerfile:1 +FROM mcr.microsoft.com/dotnet/aspnet:9.0 AS base +WORKDIR /app +EXPOSE 80 +EXPOSE 443 + +FROM mcr.microsoft.com/dotnet/sdk:9.0 AS build + +COPY . . +RUN --mount=type=cache,target=/root/.nuget/packages dotnet restore "CrdtMerge/CrdtMerge.csproj" + +ARG APP_VERSION +LABEL version=$APP_VERSION + +RUN --mount=type=cache,target=/root/.nuget/packages dotnet build /p:InformationalVersion=$APP_VERSION "CrdtMerge/CrdtMerge.csproj" -c Release -o /app/build + +FROM build AS publish +RUN --mount=type=cache,target=/root/.nuget/packages dotnet publish /p:InformationalVersion=$APP_VERSION "CrdtMerge/CrdtMerge.csproj" -c Release -o /app/publish + +FROM base AS final +RUN mkdir -p /var/lib/crdtmerge && chown -R www-data:www-data /var/lib/crdtmerge +WORKDIR /app +COPY --from=publish /app/publish . +USER www-data:www-data +ENTRYPOINT ["dotnet", "CrdtMerge.dll"] diff --git a/backend/CrdtMerge/appsettings.json b/backend/CrdtMerge/appsettings.json index d766d8006..38010f509 100644 --- a/backend/CrdtMerge/appsettings.json +++ b/backend/CrdtMerge/appsettings.json @@ -1,5 +1,5 @@ { - "CrdtMergeConfig": { + "SendReceiveConfig": { "LexboxUsername": null }, "Logging": { diff --git a/deployment/base/crdtmerge-deployment.yaml b/deployment/base/crdtmerge-deployment.yaml new file mode 100644 index 000000000..9a2e813ad --- /dev/null +++ b/deployment/base/crdtmerge-deployment.yaml @@ -0,0 +1,144 @@ +# https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service +apiVersion: v1 +kind: Service +metadata: + name: crdtmerge + namespace: languagedepot + labels: + app: crdtmerge +spec: + type: ClusterIP + clusterIP: None + selector: + app: crdtmerge + ports: + - name: http + protocol: TCP + port: 80 + +--- + +# https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#writing-a-deployment-spec +apiVersion: apps/v1 +kind: Deployment +metadata: + name: crdtmerge + namespace: languagedepot + labels: + app: crdtmerge +spec: + selector: + matchLabels: + app: crdtmerge + strategy: + rollingUpdate: + maxSurge: 2 + maxUnavailable: 0 + type: RollingUpdate + template: + # https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates + metadata: + labels: + app: crdtmerge + spec: + securityContext: + runAsGroup: 33 + runAsUser: 33 + runAsNonRoot: true + containers: + - name: crdtmerge + image: ghcr.io/sillsdev/lexbox-crdtmerge:develop + imagePullPolicy: IfNotPresent + # https://kubernetes.io/docs/concepts/configuration/manage-resources-containers + resources: + requests: + memory: 1500Mi + limits: + memory: 2400Mi + startupProbe: + httpGet: + port: 80 + path: /api/healthz + failureThreshold: 30 + periodSeconds: 10 + ports: + - containerPort: 80 + + volumeMounts: + - name: crdtmerge + mountPath: /var/lib/crdtmerge + + env: + - name: DOTNET_URLS + value: http://0.0.0.0:80 + - name: ASPNETCORE_ENVIRONMENT + valueFrom: + configMapKeyRef: + name: app-config + key: environment-name + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: K8S_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POSTGRES_DB + valueFrom: + secretKeyRef: + key: POSTGRES_DB + name: db + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + key: POSTGRES_PASSWORD + name: db + - name: DbConfig__LexBoxConnectionString + value: Host=db;Port=5432;Username=postgres;Password=$(POSTGRES_PASSWORD);Database=$(POSTGRES_DB) + - name: SendReceiveConfig__ProjectStorageRoot + value: /var/lib/crdtmerge/projects + - name: HgConfig__SendReceiveDomain + valueFrom: + configMapKeyRef: + name: app-config + key: hg-domain + - name: SendReceiveConfig__LexboxUrl + value: http://lexbox:5158/ + # - name: SendReceiveConfig__HgUrl + # value: http://lexbox:5158/hg/ + - name: SendReceiveConfig__LexboxUsername + valueFrom: + secretKeyRef: + key: CRDT_MERGE_SEND_RECEIVE_USERNAME + name: crdtmerge + - name: SendReceiveConfig__LexboxPassword + valueFrom: + secretKeyRef: + key: CRDT_MERGE_SEND_RECEIVE_PASSWORD + name: crdtmerge + - name: SendReceiveConfig__FdoDataModelVersion + value: "7000072" + + initContainers: + - name: populate-crdt-project-storage + securityContext: + # Make sure we're authorized to set ownership + runAsUser: 0 + runAsGroup: 0 + runAsNonRoot: false + image: busybox:1.36.1 + command: + - 'sh' + - '-c' + - | + mkdir -p /crdtmerge/projects + chown www-data:www-data /crdtmerge/projects + volumeMounts: + - name: crdtmerge + mountPath: /crdtmerge + + volumes: + - name: crdtmerge + persistentVolumeClaim: + claimName: crdtmerge # established in pvc.yaml diff --git a/deployment/base/pvc.yaml b/deployment/base/pvc.yaml index 8ffbdb558..e2e318625 100644 --- a/deployment/base/pvc.yaml +++ b/deployment/base/pvc.yaml @@ -33,3 +33,21 @@ spec: requests: storage: 10Gi storageClassName: weekly-snapshots-retain-4 # provided by LTOps + +--- + +# https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: crdtmerge + namespace: languagedepot + labels: + app.kubernetes.io/part-of: languagedepot +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: weekly-snapshots-retain-4 # provided by LTOps diff --git a/deployment/base/secrets.yaml b/deployment/base/secrets.yaml index 94236d32a..28b3817c8 100644 --- a/deployment/base/secrets.yaml +++ b/deployment/base/secrets.yaml @@ -84,3 +84,13 @@ metadata: stringData: KOPIA_PASSWORD: '' kopia.config: '' +--- + +apiVersion: v1 +kind: Secret +metadata: + name: crdtmerge + namespace: languagedepot +stringData: + CRDT_MERGE_SEND_RECEIVE_USERNAME: '' + CRDT_MERGE_SEND_RECEIVE_PASSWORD: '' diff --git a/deployment/local-dev/crdtmerge-secrets.yaml b/deployment/local-dev/crdtmerge-secrets.yaml new file mode 100644 index 000000000..180e85498 --- /dev/null +++ b/deployment/local-dev/crdtmerge-secrets.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: crdtmerge + namespace: languagedepot +stringData: + CRDT_MERGE_SEND_RECEIVE_USERNAME: 'admin' + CRDT_MERGE_SEND_RECEIVE_PASSWORD: 'pass' diff --git a/deployment/local-dev/kustomization.yaml b/deployment/local-dev/kustomization.yaml index 02e526433..dae570b77 100644 --- a/deployment/local-dev/kustomization.yaml +++ b/deployment/local-dev/kustomization.yaml @@ -6,6 +6,7 @@ resources: - ../base/ #- secrets.yaml - ingress-deployment.yaml +- crdtmerge-secrets.yaml - db-secrets.yaml - lf-classic-secrets.yaml - self-signed-ssl.yaml diff --git a/skaffold.yaml b/skaffold.yaml index bb40c87d7..fdcea20fe 100644 --- a/skaffold.yaml +++ b/skaffold.yaml @@ -14,6 +14,10 @@ build: manual: - src: '**' dest: /src/backend + - image: ghcr.io/sillsdev/lexbox-crdtmerge + context: backend + docker: + dockerfile: CrdtMerge/Dockerfile - image: ghcr.io/sillsdev/lexbox-ui context: frontend docker: @@ -41,6 +45,12 @@ portForward: resourceName: lexbox namespace: languagedepot port: 5158 + # CrdtMerge + - resourceType: Service + resourceName: crdtmerge + namespace: languagedepot + port: 80 + localPort: 5275 # OpenTelemetry - resourceType: Service resourceName: lexbox @@ -89,6 +99,12 @@ profiles: namespace: languagedepot port: 5432 localPort: 5433 + # CrdtMerge + - resourceType: Service + resourceName: crdtmerge + namespace: languagedepot + port: 80 + localPort: 5275 # OpenTelemetry - resourceType: Service resourceName: lexbox @@ -141,6 +157,12 @@ profiles: resourceName: lexbox namespace: languagedepot port: 5158 + # CrdtMerge + - resourceType: Service + resourceName: crdtmerge + namespace: languagedepot + port: 80 + localPort: 5275 # OpenTelemetry - resourceType: Service resourceName: lexbox From 6f6314ff1c9b1c2300b3fdad7469760a28980ecc Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 29 Oct 2024 11:51:04 +0700 Subject: [PATCH 02/16] Also build develop version of CrdtMerge on PR --- .github/workflows/develop-crdtmerge.yaml | 72 ++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 .github/workflows/develop-crdtmerge.yaml diff --git a/.github/workflows/develop-crdtmerge.yaml b/.github/workflows/develop-crdtmerge.yaml new file mode 100644 index 000000000..3cdfd7e0e --- /dev/null +++ b/.github/workflows/develop-crdtmerge.yaml @@ -0,0 +1,72 @@ +name: Develop CrdtMerge CI/CD +on: + workflow_dispatch: + push: + paths: + - 'backend/CrdtMerge/**' + - 'backend/FixFwData/**' + - 'backend/FwLite/FwDataMiniLcmBridge/**' + - 'backend/FwLite/LcmCrdt/**' + - 'backend/FwLite/MiniLcm/**' + - 'backend/FwLiteProjectSync/FwLiteProjectSync/**' + - 'backend/LexCore/**' + - 'backend/LexData/**' + - '.github/workflows/lexbox-crdtmerge.yaml' + - '.github/workflows/deploy.yaml' + - 'deployment/base/crdtmerge-deployment.yaml' + branches: + - develop + pull_request: + paths: + - 'backend/CrdtMerge/**' + - 'backend/FixFwData/**' + - 'backend/FwLite/FwDataMiniLcmBridge/**' + - 'backend/FwLite/LcmCrdt/**' + - 'backend/FwLite/MiniLcm/**' + - 'backend/FwLiteProjectSync/FwLiteProjectSync/**' + - 'backend/LexCore/**' + - 'backend/LexData/**' + - '.github/workflows/lexbox-crdtmerge.yaml' + - '.github/workflows/deploy.yaml' + - 'deployment/base/crdtmerge-deployment.yaml' + branches: + - develop + +jobs: + set-version: + name: Set Version + runs-on: ubuntu-latest + outputs: + version: ${{ steps.setVersion.outputs.VERSION }} + steps: + - name: Set Version + id: setVersion + # set version to date in vYYYY-MM-DD-commitSha format + run: | + shortSha=$(echo ${{ github.sha }} | cut -c1-8) + echo "VERSION=v$(date --rfc-3339=date)-$shortSha" >> ${GITHUB_OUTPUT} + build-crdtmerge: + name: Build CrdtMerge + needs: set-version + uses: ./.github/workflows/lexbox-crdtmerge.yaml + with: + version: ${{ needs.set-version.outputs.version }} + deploy-crdtmerge: + name: Deploy CrdtMerge + if: ${{github.ref == 'refs/heads/develop'}} + needs: [ build-crdtmerge, set-version ] + uses: ./.github/workflows/deploy.yaml + secrets: inherit + with: + version: ${{ needs.set-version.outputs.version }} + image: 'ghcr.io/sillsdev/lexbox-crdtmerge' + k8s-environment: develop + deploy-domain: lexbox.dev.languagetechnology.org + + # TODO: Run CrdtMerge tests once we have developed them, but we don't need to run the whole integration test suite if only CrdtMerge changes are being pushed + # integration-test-gha: + # name: GHA integration tests + # needs: [build-crdtmerge, set-version] + # uses: ./.github/workflows/integration-test-gha.yaml + # with: + # lexbox-crdtmerge-tag: ${{ needs.set-version.outputs.version }} From 2bf26c6f86a5e1799213cbea73faa85ee9bfab2c Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 29 Oct 2024 11:53:47 +0700 Subject: [PATCH 03/16] CrdtMerge build needs dotnet 9.x --- .github/workflows/lexbox-crdtmerge.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lexbox-crdtmerge.yaml b/.github/workflows/lexbox-crdtmerge.yaml index a01a2163b..d660c94e2 100644 --- a/.github/workflows/lexbox-crdtmerge.yaml +++ b/.github/workflows/lexbox-crdtmerge.yaml @@ -18,7 +18,7 @@ env: jobs: - publish-api: + publish-crdtmerge: timeout-minutes: 60 runs-on: ubuntu-latest @@ -49,7 +49,7 @@ jobs: submodules: true - uses: actions/setup-dotnet@v4 with: - dotnet-version: '8.x' + dotnet-version: '9.x' - name: Dotnet build run: dotnet build backend/CrdtMerge/CrdtMerge.csproj # TODO: Write CrdtMerge unit tests, probably based on existing sync tests From bd61f807fdbbff9114c3d3fc6802dd1972f7afb4 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Tue, 29 Oct 2024 11:57:59 +0700 Subject: [PATCH 04/16] Use correct Dockerfile when building CrdtMerge --- .github/workflows/lexbox-crdtmerge.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/lexbox-crdtmerge.yaml b/.github/workflows/lexbox-crdtmerge.yaml index d660c94e2..5fb363043 100644 --- a/.github/workflows/lexbox-crdtmerge.yaml +++ b/.github/workflows/lexbox-crdtmerge.yaml @@ -91,6 +91,7 @@ jobs: - uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0 with: context: backend + file: backend/CrdtMerge/Dockerfile build-args: | APP_VERSION=${{ inputs.version }} push: ${{ !env.ACT && github.repository == 'sillsdev/languageforge-lexbox' }} From 31efdc3d048d08df51a921c796b70b7e42786119 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Wed, 30 Oct 2024 13:19:32 +0700 Subject: [PATCH 05/16] Add health checks at /api/healthz --- backend/CrdtMerge/Program.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/backend/CrdtMerge/Program.cs b/backend/CrdtMerge/Program.cs index e2604a9ec..1e9e8392c 100644 --- a/backend/CrdtMerge/Program.cs +++ b/backend/CrdtMerge/Program.cs @@ -14,6 +14,8 @@ // Learn more about configuring OpenAPI at https://aka.ms/aspnet/openapi builder.Services.AddOpenApi(); +builder.Services.AddHealthChecks(); + builder.Services.AddLexData( autoApplyMigrations: false, useOpenIddict: false @@ -33,6 +35,8 @@ app.UseHttpsRedirection(); +app.MapHealthChecks("/api/healthz"); + app.MapPost("/sync", ExecuteMergeRequest); app.Run(); From efd2665bc1aa8c2002cc29f99b26778c71d6157c Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Wed, 30 Oct 2024 13:21:55 +0700 Subject: [PATCH 06/16] Delete HgConfig value we're not actually using --- deployment/base/crdtmerge-deployment.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/deployment/base/crdtmerge-deployment.yaml b/deployment/base/crdtmerge-deployment.yaml index 9a2e813ad..ae9a7ad1a 100644 --- a/deployment/base/crdtmerge-deployment.yaml +++ b/deployment/base/crdtmerge-deployment.yaml @@ -98,11 +98,6 @@ spec: value: Host=db;Port=5432;Username=postgres;Password=$(POSTGRES_PASSWORD);Database=$(POSTGRES_DB) - name: SendReceiveConfig__ProjectStorageRoot value: /var/lib/crdtmerge/projects - - name: HgConfig__SendReceiveDomain - valueFrom: - configMapKeyRef: - name: app-config - key: hg-domain - name: SendReceiveConfig__LexboxUrl value: http://lexbox:5158/ # - name: SendReceiveConfig__HgUrl From 6028ea11a9d2634d24e3127985fc2b2d8c735ef4 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Wed, 30 Oct 2024 13:35:16 +0700 Subject: [PATCH 07/16] Use SEED_USER_PASSWORD for CRDT S/R on non-prod Instead of defining a new secret that will have to be added to develop and staging environments, we can reuse SEED_USER_PASSWORD and have things "just work". Then only prod will need a separate user account created and its credentials stored in a new secret. --- .../develop/crdtmerge-deployment.patch.yaml | 24 +++++++++++++++++++ deployment/develop/kustomization.yaml | 1 + .../local-dev/crdtmerge-deployment.patch.yaml | 24 +++++++++++++++++++ deployment/local-dev/crdtmerge-secrets.yaml | 8 ------- deployment/local-dev/kustomization.yaml | 2 +- .../staging/crdtmerge-deployment.patch.yaml | 24 +++++++++++++++++++ deployment/staging/kustomization.yaml | 1 + 7 files changed, 75 insertions(+), 9 deletions(-) create mode 100644 deployment/develop/crdtmerge-deployment.patch.yaml create mode 100644 deployment/local-dev/crdtmerge-deployment.patch.yaml delete mode 100644 deployment/local-dev/crdtmerge-secrets.yaml create mode 100644 deployment/staging/crdtmerge-deployment.patch.yaml diff --git a/deployment/develop/crdtmerge-deployment.patch.yaml b/deployment/develop/crdtmerge-deployment.patch.yaml new file mode 100644 index 000000000..70dc8eb7d --- /dev/null +++ b/deployment/develop/crdtmerge-deployment.patch.yaml @@ -0,0 +1,24 @@ +#file: noinspection KubernetesMissingKeys,KubernetesUnknownValues +apiVersion: apps/v1 +kind: Deployment +metadata: + name: crdtmerge + namespace: languagedepot + labels: + app: crdtmerge +spec: + template: + spec: + containers: + - name: crdtmerge + imagePullPolicy: IfNotPresent + env: + - name: SendReceiveConfig__LexboxUsername + value: "admin" + valueFrom: # don't use secret as defined in base + - name: SendReceiveConfig__LexboxPassword + valueFrom: + secretKeyRef: + key: SEED_USER_PASSWORD + name: db + optional: true diff --git a/deployment/develop/kustomization.yaml b/deployment/develop/kustomization.yaml index 7ba6e3340..f4eea9c18 100644 --- a/deployment/develop/kustomization.yaml +++ b/deployment/develop/kustomization.yaml @@ -7,6 +7,7 @@ resources: components: - ../init-repos patches: + - path: crdtmerge-deployment.patch.yaml - path: lexbox-deployment.patch.yaml - path: app-config.yaml - path: ingress-config.yaml diff --git a/deployment/local-dev/crdtmerge-deployment.patch.yaml b/deployment/local-dev/crdtmerge-deployment.patch.yaml new file mode 100644 index 000000000..70dc8eb7d --- /dev/null +++ b/deployment/local-dev/crdtmerge-deployment.patch.yaml @@ -0,0 +1,24 @@ +#file: noinspection KubernetesMissingKeys,KubernetesUnknownValues +apiVersion: apps/v1 +kind: Deployment +metadata: + name: crdtmerge + namespace: languagedepot + labels: + app: crdtmerge +spec: + template: + spec: + containers: + - name: crdtmerge + imagePullPolicy: IfNotPresent + env: + - name: SendReceiveConfig__LexboxUsername + value: "admin" + valueFrom: # don't use secret as defined in base + - name: SendReceiveConfig__LexboxPassword + valueFrom: + secretKeyRef: + key: SEED_USER_PASSWORD + name: db + optional: true diff --git a/deployment/local-dev/crdtmerge-secrets.yaml b/deployment/local-dev/crdtmerge-secrets.yaml deleted file mode 100644 index 180e85498..000000000 --- a/deployment/local-dev/crdtmerge-secrets.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: crdtmerge - namespace: languagedepot -stringData: - CRDT_MERGE_SEND_RECEIVE_USERNAME: 'admin' - CRDT_MERGE_SEND_RECEIVE_PASSWORD: 'pass' diff --git a/deployment/local-dev/kustomization.yaml b/deployment/local-dev/kustomization.yaml index dae570b77..0c9927242 100644 --- a/deployment/local-dev/kustomization.yaml +++ b/deployment/local-dev/kustomization.yaml @@ -6,7 +6,6 @@ resources: - ../base/ #- secrets.yaml - ingress-deployment.yaml -- crdtmerge-secrets.yaml - db-secrets.yaml - lf-classic-secrets.yaml - self-signed-ssl.yaml @@ -40,6 +39,7 @@ patches: path: delete-oauth-certs.yaml - path: app-config.yaml + - path: crdtmerge-deployment.patch.yaml - path: lexbox-deployment.patch.yaml - path: ui-deployment.patch.yaml - path: hg-repos-pvc.patch.yaml diff --git a/deployment/staging/crdtmerge-deployment.patch.yaml b/deployment/staging/crdtmerge-deployment.patch.yaml new file mode 100644 index 000000000..70dc8eb7d --- /dev/null +++ b/deployment/staging/crdtmerge-deployment.patch.yaml @@ -0,0 +1,24 @@ +#file: noinspection KubernetesMissingKeys,KubernetesUnknownValues +apiVersion: apps/v1 +kind: Deployment +metadata: + name: crdtmerge + namespace: languagedepot + labels: + app: crdtmerge +spec: + template: + spec: + containers: + - name: crdtmerge + imagePullPolicy: IfNotPresent + env: + - name: SendReceiveConfig__LexboxUsername + value: "admin" + valueFrom: # don't use secret as defined in base + - name: SendReceiveConfig__LexboxPassword + valueFrom: + secretKeyRef: + key: SEED_USER_PASSWORD + name: db + optional: true diff --git a/deployment/staging/kustomization.yaml b/deployment/staging/kustomization.yaml index 66768dc21..c863fa415 100644 --- a/deployment/staging/kustomization.yaml +++ b/deployment/staging/kustomization.yaml @@ -7,6 +7,7 @@ resources: components: - ../init-repos patches: + - path: crdtmerge-deployment.patch.yaml - path: lexbox-deployment.patch.yaml - path: app-config.yaml - path: hg-repos-volume.yaml From 140de483c8534875deaf4f4a5d401bd820506bef Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Wed, 30 Oct 2024 16:06:39 +0700 Subject: [PATCH 08/16] Big rename: CrdtMerge -> FwHeadless Also renamed SendReceiveConfig section of appsettings.json to FwHeadlessConfig so we get all the renames done at once. --- .github/workflows/deploy-branch.yaml | 8 +-- .github/workflows/deploy.yaml | 8 +-- ...rdtmerge.yaml => develop-fw-headless.yaml} | 34 ++++++------ .github/workflows/integration-test-gha.yaml | 14 ++--- ...crdtmerge.yaml => lexbox-fw-headless.yaml} | 14 ++--- .github/workflows/release-pipeline.yaml | 8 +-- LexBox.sln | 2 +- backend/{CrdtMerge => FwHeadless}/Dockerfile | 10 ++-- .../FwHeadless.csproj} | 0 .../FwHeadlessConfig.cs} | 4 +- .../FwHeadlessKernel.cs} | 10 ++-- backend/{CrdtMerge => FwHeadless}/Program.cs | 6 +-- .../ProjectLookupService.cs | 2 +- .../Properties/launchSettings.json | 0 .../SendReceiveHelpers.cs | 4 +- .../SendReceiveService.cs | 4 +- .../appsettings.Development.json | 2 +- .../appsettings.json | 2 +- ...yment.yaml => fw-headless-deployment.yaml} | 52 +++++++++---------- deployment/base/pvc.yaml | 2 +- deployment/base/secrets.yaml | 2 +- ...yaml => fw-headless-deployment.patch.yaml} | 10 ++-- deployment/develop/kustomization.yaml | 2 +- ...yaml => fw-headless-deployment.patch.yaml} | 10 ++-- deployment/local-dev/kustomization.yaml | 2 +- ...yaml => fw-headless-deployment.patch.yaml} | 10 ++-- deployment/staging/kustomization.yaml | 2 +- skaffold.yaml | 16 +++--- 28 files changed, 120 insertions(+), 120 deletions(-) rename .github/workflows/{develop-crdtmerge.yaml => develop-fw-headless.yaml} (66%) rename .github/workflows/{lexbox-crdtmerge.yaml => lexbox-fw-headless.yaml} (84%) rename backend/{CrdtMerge => FwHeadless}/Dockerfile (59%) rename backend/{CrdtMerge/CrdtMerge.csproj => FwHeadless/FwHeadless.csproj} (100%) rename backend/{CrdtMerge/CrdtMergeConfig.cs => FwHeadless/FwHeadlessConfig.cs} (90%) rename backend/{CrdtMerge/CrdtMergeKernel.cs => FwHeadless/FwHeadlessKernel.cs} (69%) rename backend/{CrdtMerge => FwHeadless}/Program.cs (97%) rename backend/{CrdtMerge => FwHeadless}/ProjectLookupService.cs (94%) rename backend/{CrdtMerge => FwHeadless}/Properties/launchSettings.json (100%) rename backend/{CrdtMerge => FwHeadless}/SendReceiveHelpers.cs (96%) rename backend/{CrdtMerge => FwHeadless}/SendReceiveService.cs (91%) rename backend/{CrdtMerge => FwHeadless}/appsettings.Development.json (95%) rename backend/{CrdtMerge => FwHeadless}/appsettings.json (87%) rename deployment/base/{crdtmerge-deployment.yaml => fw-headless-deployment.yaml} (75%) rename deployment/develop/{crdtmerge-deployment.patch.yaml => fw-headless-deployment.patch.yaml} (74%) rename deployment/local-dev/{crdtmerge-deployment.patch.yaml => fw-headless-deployment.patch.yaml} (74%) rename deployment/staging/{crdtmerge-deployment.patch.yaml => fw-headless-deployment.patch.yaml} (74%) diff --git a/.github/workflows/deploy-branch.yaml b/.github/workflows/deploy-branch.yaml index 8b1d9ca92..5746a2520 100644 --- a/.github/workflows/deploy-branch.yaml +++ b/.github/workflows/deploy-branch.yaml @@ -41,10 +41,10 @@ jobs: version: ${{ needs.set-version.outputs.version }} label-latest: false - build-crdtmerge: - name: Build CrdtMerge + build-fw-headless: + name: Build FwHeadless needs: [ set-version ] - uses: ./.github/workflows/lexbox-crdtmerge.yaml + uses: ./.github/workflows/lexbox-fw-headless.yaml with: version: ${{ needs.set-version.outputs.version }} label-latest: false @@ -52,7 +52,7 @@ jobs: deploy: name: Deploy Develop uses: ./.github/workflows/deploy.yaml - needs: [ build-api, build-ui, build-hgweb, build-crdtmerge, set-version ] + needs: [ build-api, build-ui, build-hgweb, build-fw-headless, set-version ] secrets: inherit with: version: ${{ needs.set-version.outputs.version }} diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index f31011c4a..45d5e8045 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -50,7 +50,7 @@ jobs: url: https://${{ inputs.deploy-domain }} outputs: api-version: ${{ steps.get-api-version.outputs.result }} - crdtmerge-version: ${{ steps.get-crdtmerge-version.outputs.result }} + fw-headless-version: ${{ steps.get-fw-headless-version.outputs.result }} ui-version: ${{ steps.get-ui-version.outputs.result }} steps: - name: Checkout lexbox repo @@ -81,11 +81,11 @@ jobs: id: get-api-version with: cmd: yq '.images.[] | select(.name == "ghcr.io/sillsdev/lexbox-api").newTag' "fleet/${{ inputs.k8s-environment }}/kustomization.yaml" - - name: Get CrdtMerge version + - name: Get FwHeadless version uses: mikefarah/yq@0b34c9a00de1c575a34eea05af1d956a525c4fc1 # v4.34.2 - id: get-crdtmerge-version + id: get-fw-headless-version with: - cmd: yq '.images.[] | select(.name == "ghcr.io/sillsdev/lexbox-crdtmerge").newTag' "fleet/${{ inputs.k8s-environment }}/kustomization.yaml" + cmd: yq '.images.[] | select(.name == "ghcr.io/sillsdev/lexbox-fw-headless").newTag' "fleet/${{ inputs.k8s-environment }}/kustomization.yaml" - name: Get UI version uses: mikefarah/yq@0b34c9a00de1c575a34eea05af1d956a525c4fc1 # v4.34.2 id: get-ui-version diff --git a/.github/workflows/develop-crdtmerge.yaml b/.github/workflows/develop-fw-headless.yaml similarity index 66% rename from .github/workflows/develop-crdtmerge.yaml rename to .github/workflows/develop-fw-headless.yaml index 3cdfd7e0e..6037f51a2 100644 --- a/.github/workflows/develop-crdtmerge.yaml +++ b/.github/workflows/develop-fw-headless.yaml @@ -1,9 +1,9 @@ -name: Develop CrdtMerge CI/CD +name: Develop FwHeadless CI/CD on: workflow_dispatch: push: paths: - - 'backend/CrdtMerge/**' + - 'backend/FwHeadless/**' - 'backend/FixFwData/**' - 'backend/FwLite/FwDataMiniLcmBridge/**' - 'backend/FwLite/LcmCrdt/**' @@ -11,14 +11,14 @@ on: - 'backend/FwLiteProjectSync/FwLiteProjectSync/**' - 'backend/LexCore/**' - 'backend/LexData/**' - - '.github/workflows/lexbox-crdtmerge.yaml' + - '.github/workflows/lexbox-fw-headless.yaml' - '.github/workflows/deploy.yaml' - - 'deployment/base/crdtmerge-deployment.yaml' + - 'deployment/base/fw-headless-deployment.yaml' branches: - develop pull_request: paths: - - 'backend/CrdtMerge/**' + - 'backend/FwHeadless/**' - 'backend/FixFwData/**' - 'backend/FwLite/FwDataMiniLcmBridge/**' - 'backend/FwLite/LcmCrdt/**' @@ -26,9 +26,9 @@ on: - 'backend/FwLiteProjectSync/FwLiteProjectSync/**' - 'backend/LexCore/**' - 'backend/LexData/**' - - '.github/workflows/lexbox-crdtmerge.yaml' + - '.github/workflows/lexbox-fw-headless.yaml' - '.github/workflows/deploy.yaml' - - 'deployment/base/crdtmerge-deployment.yaml' + - 'deployment/base/fw-headless-deployment.yaml' branches: - develop @@ -45,28 +45,28 @@ jobs: run: | shortSha=$(echo ${{ github.sha }} | cut -c1-8) echo "VERSION=v$(date --rfc-3339=date)-$shortSha" >> ${GITHUB_OUTPUT} - build-crdtmerge: - name: Build CrdtMerge + build-fw-headless: + name: Build FwHeadless needs: set-version - uses: ./.github/workflows/lexbox-crdtmerge.yaml + uses: ./.github/workflows/lexbox-fw-headless.yaml with: version: ${{ needs.set-version.outputs.version }} - deploy-crdtmerge: - name: Deploy CrdtMerge + deploy-fw-headless: + name: Deploy FwHeadless if: ${{github.ref == 'refs/heads/develop'}} - needs: [ build-crdtmerge, set-version ] + needs: [ build-fw-headless, set-version ] uses: ./.github/workflows/deploy.yaml secrets: inherit with: version: ${{ needs.set-version.outputs.version }} - image: 'ghcr.io/sillsdev/lexbox-crdtmerge' + image: 'ghcr.io/sillsdev/lexbox-fw-headless' k8s-environment: develop deploy-domain: lexbox.dev.languagetechnology.org - # TODO: Run CrdtMerge tests once we have developed them, but we don't need to run the whole integration test suite if only CrdtMerge changes are being pushed + # TODO: Run FwHeadless tests once we have developed them, but we don't need to run the whole integration test suite if only FwHeadless changes are being pushed # integration-test-gha: # name: GHA integration tests - # needs: [build-crdtmerge, set-version] + # needs: [build-fw-headless, set-version] # uses: ./.github/workflows/integration-test-gha.yaml # with: - # lexbox-crdtmerge-tag: ${{ needs.set-version.outputs.version }} + # lexbox-fw-headless-tag: ${{ needs.set-version.outputs.version }} diff --git a/.github/workflows/integration-test-gha.yaml b/.github/workflows/integration-test-gha.yaml index c0ddb9636..98a5dbe86 100644 --- a/.github/workflows/integration-test-gha.yaml +++ b/.github/workflows/integration-test-gha.yaml @@ -39,16 +39,16 @@ jobs: uses: mikefarah/yq@0b34c9a00de1c575a34eea05af1d956a525c4fc1 # v4.34.2 with: cmd: yq eval -i '(.images.[] | select(.name == "ghcr.io/sillsdev/lexbox-api").newTag) = "${{ inputs.lexbox-api-tag }}"' "./deployment/gha/kustomization.yaml" - # It's also possible that hgweb, crdtmerge, and/or ui image may have changed; if so, pull them and update kustomization.yaml for them as well - - name: Pull crdtmerge if updated - id: crdtmerge_image + # It's also possible that hgweb, fw-headless, and/or ui image may have changed; if so, pull them and update kustomization.yaml for them as well + - name: Pull fw-headless if updated + id: fw-headless_image continue-on-error: true - run: docker pull ghcr.io/sillsdev/lexbox-crdtmerge:${{ inputs.lexbox-api-tag }} - - name: Update image crdtmerge version - if: ${{ steps.crdtmerge_image.outcome == 'success' }} + run: docker pull ghcr.io/sillsdev/lexbox-fw-headless:${{ inputs.lexbox-api-tag }} + - name: Update image fw-headless version + if: ${{ steps.fw-headless_image.outcome == 'success' }} uses: mikefarah/yq@0b34c9a00de1c575a34eea05af1d956a525c4fc1 # v4.34.2 with: - cmd: yq eval -i '(.images.[] | select(.name == "ghcr.io/sillsdev/lexbox-crdtmerge").newTag) = "${{ inputs.lexbox-api-tag }}"' "./deployment/gha/kustomization.yaml" + cmd: yq eval -i '(.images.[] | select(.name == "ghcr.io/sillsdev/lexbox-fw-headless").newTag) = "${{ inputs.lexbox-api-tag }}"' "./deployment/gha/kustomization.yaml" - name: Pull hgweb if updated id: hgweb_image continue-on-error: true diff --git a/.github/workflows/lexbox-crdtmerge.yaml b/.github/workflows/lexbox-fw-headless.yaml similarity index 84% rename from .github/workflows/lexbox-crdtmerge.yaml rename to .github/workflows/lexbox-fw-headless.yaml index 5fb363043..3d49847ea 100644 --- a/.github/workflows/lexbox-crdtmerge.yaml +++ b/.github/workflows/lexbox-fw-headless.yaml @@ -1,4 +1,4 @@ -name: Build CrdtMerge +name: Build FwHeadless # https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#on on: @@ -14,11 +14,11 @@ on: default: false env: - IMAGE_NAME: ghcr.io/sillsdev/lexbox-crdtmerge + IMAGE_NAME: ghcr.io/sillsdev/lexbox-fw-headless jobs: - publish-crdtmerge: + publish-fw-headless: timeout-minutes: 60 runs-on: ubuntu-latest @@ -51,10 +51,10 @@ jobs: with: dotnet-version: '9.x' - name: Dotnet build - run: dotnet build backend/CrdtMerge/CrdtMerge.csproj - # TODO: Write CrdtMerge unit tests, probably based on existing sync tests + run: dotnet build backend/FwHeadless/FwHeadless.csproj + # TODO: Write FwHeadless unit tests, probably based on existing sync tests # - name: Unit tests - # run: dotnet test backend/CrdtMerge/CrdtMerge.csproj --logger:"xunit;LogFileName={assembly}.results.xml" --results-directory ./test-results --filter "Category!=Integration&Category!=FlakyIntegration" --blame-hang-timeout 10m + # run: dotnet test backend/FwHeadless/FwHeadless.csproj --logger:"xunit;LogFileName={assembly}.results.xml" --results-directory ./test-results --filter "Category!=Integration&Category!=FlakyIntegration" --blame-hang-timeout 10m # - name: Publish unit test results # uses: EnricoMi/publish-unit-test-result-action@8885e273a4343cd7b48eaa72428dea0c3067ea98 # v2.14.0 # if: always() @@ -91,7 +91,7 @@ jobs: - uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0 with: context: backend - file: backend/CrdtMerge/Dockerfile + file: backend/FwHeadless/Dockerfile build-args: | APP_VERSION=${{ inputs.version }} push: ${{ !env.ACT && github.repository == 'sillsdev/languageforge-lexbox' }} diff --git a/.github/workflows/release-pipeline.yaml b/.github/workflows/release-pipeline.yaml index 9f65e9e3c..006e32548 100644 --- a/.github/workflows/release-pipeline.yaml +++ b/.github/workflows/release-pipeline.yaml @@ -43,10 +43,10 @@ jobs: version: ${{ needs.set-version.outputs.version }} label-latest: true - build-crdtmerge: - name: Build crdtmerge + build-fw-headless: + name: Build fw-headless needs: [ set-version ] - uses: ./.github/workflows/lexbox-crdtmerge.yaml + uses: ./.github/workflows/lexbox-fw-headless.yaml with: version: ${{ needs.set-version.outputs.version }} label-latest: true @@ -54,7 +54,7 @@ jobs: deploy: name: Deploy Staging uses: ./.github/workflows/deploy.yaml - needs: [ build-api, build-ui, build-hgweb, build-crdtmerge, set-version ] + needs: [ build-api, build-ui, build-hgweb, build-fw-headless, set-version ] secrets: inherit with: version: ${{ needs.set-version.outputs.version }} diff --git a/LexBox.sln b/LexBox.sln index 22a2bad17..612c0e95e 100644 --- a/LexBox.sln +++ b/LexBox.sln @@ -51,7 +51,7 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LcmDebugger", "backend\LfNe EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MiniLcm.Tests", "backend\FwLite\MiniLcm.Tests\MiniLcm.Tests.csproj", "{00AE5440-0E36-4488-935B-5B11301BA57D}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CrdtMerge", "backend\CrdtMerge\CrdtMerge.csproj", "{ECBA46AB-AF87-4D4D-9716-FD77264B817F}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FwHeadless", "backend\FwHeadless\FwHeadless.csproj", "{ECBA46AB-AF87-4D4D-9716-FD77264B817F}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution diff --git a/backend/CrdtMerge/Dockerfile b/backend/FwHeadless/Dockerfile similarity index 59% rename from backend/CrdtMerge/Dockerfile rename to backend/FwHeadless/Dockerfile index 0746776db..0b24a4f92 100644 --- a/backend/CrdtMerge/Dockerfile +++ b/backend/FwHeadless/Dockerfile @@ -7,19 +7,19 @@ EXPOSE 443 FROM mcr.microsoft.com/dotnet/sdk:9.0 AS build COPY . . -RUN --mount=type=cache,target=/root/.nuget/packages dotnet restore "CrdtMerge/CrdtMerge.csproj" +RUN --mount=type=cache,target=/root/.nuget/packages dotnet restore "FwHeadless/FwHeadless.csproj" ARG APP_VERSION LABEL version=$APP_VERSION -RUN --mount=type=cache,target=/root/.nuget/packages dotnet build /p:InformationalVersion=$APP_VERSION "CrdtMerge/CrdtMerge.csproj" -c Release -o /app/build +RUN --mount=type=cache,target=/root/.nuget/packages dotnet build /p:InformationalVersion=$APP_VERSION "FwHeadless/FwHeadless.csproj" -c Release -o /app/build FROM build AS publish -RUN --mount=type=cache,target=/root/.nuget/packages dotnet publish /p:InformationalVersion=$APP_VERSION "CrdtMerge/CrdtMerge.csproj" -c Release -o /app/publish +RUN --mount=type=cache,target=/root/.nuget/packages dotnet publish /p:InformationalVersion=$APP_VERSION "FwHeadless/FwHeadless.csproj" -c Release -o /app/publish FROM base AS final -RUN mkdir -p /var/lib/crdtmerge && chown -R www-data:www-data /var/lib/crdtmerge +RUN mkdir -p /var/lib/fw-headless && chown -R www-data:www-data /var/lib/fw-headless WORKDIR /app COPY --from=publish /app/publish . USER www-data:www-data -ENTRYPOINT ["dotnet", "CrdtMerge.dll"] +ENTRYPOINT ["dotnet", "FwHeadless.dll"] diff --git a/backend/CrdtMerge/CrdtMerge.csproj b/backend/FwHeadless/FwHeadless.csproj similarity index 100% rename from backend/CrdtMerge/CrdtMerge.csproj rename to backend/FwHeadless/FwHeadless.csproj diff --git a/backend/CrdtMerge/CrdtMergeConfig.cs b/backend/FwHeadless/FwHeadlessConfig.cs similarity index 90% rename from backend/CrdtMerge/CrdtMergeConfig.cs rename to backend/FwHeadless/FwHeadlessConfig.cs index 8d21a36b1..ec664817e 100644 --- a/backend/CrdtMerge/CrdtMergeConfig.cs +++ b/backend/FwHeadless/FwHeadlessConfig.cs @@ -1,8 +1,8 @@ using System.ComponentModel.DataAnnotations; -namespace CrdtMerge; +namespace FwHeadless; -public class CrdtMergeConfig +public class FwHeadlessConfig { [Required, Url, RegularExpression(@"^.+/$", ErrorMessage = "Must end with '/'")] public required string LexboxUrl { get; init; } diff --git a/backend/CrdtMerge/CrdtMergeKernel.cs b/backend/FwHeadless/FwHeadlessKernel.cs similarity index 69% rename from backend/CrdtMerge/CrdtMergeKernel.cs rename to backend/FwHeadless/FwHeadlessKernel.cs index 0c312c074..9a0d42b95 100644 --- a/backend/CrdtMerge/CrdtMergeKernel.cs +++ b/backend/FwHeadless/FwHeadlessKernel.cs @@ -2,16 +2,16 @@ using FwLiteProjectSync; using LcmCrdt; -namespace CrdtMerge; +namespace FwHeadless; -public static class CrdtMergeKernel +public static class FwHeadlessKernel { - public static void AddCrdtMerge(this IServiceCollection services) + public static void AddFwHeadless(this IServiceCollection services) { services .AddLogging(builder => builder.AddConsole().AddDebug().AddFilter("Microsoft.EntityFrameworkCore", LogLevel.Warning)); - services.AddOptions() - .BindConfiguration("SendReceiveConfig") + services.AddOptions() + .BindConfiguration("FwHeadlessConfig") .ValidateDataAnnotations() .ValidateOnStart(); services.AddScoped(); diff --git a/backend/CrdtMerge/Program.cs b/backend/FwHeadless/Program.cs similarity index 97% rename from backend/CrdtMerge/Program.cs rename to backend/FwHeadless/Program.cs index 1e9e8392c..261dedd9b 100644 --- a/backend/CrdtMerge/Program.cs +++ b/backend/FwHeadless/Program.cs @@ -1,4 +1,4 @@ -using CrdtMerge; +using FwHeadless; using FwDataMiniLcmBridge; using FwLiteProjectSync; using LcmCrdt; @@ -21,7 +21,7 @@ useOpenIddict: false ); -builder.Services.AddCrdtMerge(); +builder.Services.AddFwHeadless(); var app = builder.Build(); @@ -45,7 +45,7 @@ ILogger logger, IServiceProvider services, SendReceiveService srService, - IOptions config, + IOptions config, FwDataFactory fwDataFactory, ProjectsService projectsService, ProjectLookupService projectLookupService, diff --git a/backend/CrdtMerge/ProjectLookupService.cs b/backend/FwHeadless/ProjectLookupService.cs similarity index 94% rename from backend/CrdtMerge/ProjectLookupService.cs rename to backend/FwHeadless/ProjectLookupService.cs index ac8244e99..9cb8cb971 100644 --- a/backend/CrdtMerge/ProjectLookupService.cs +++ b/backend/FwHeadless/ProjectLookupService.cs @@ -1,7 +1,7 @@ using LexData; using Microsoft.EntityFrameworkCore; -namespace CrdtMerge; +namespace FwHeadless; public class ProjectLookupService(LexBoxDbContext dbContext) { diff --git a/backend/CrdtMerge/Properties/launchSettings.json b/backend/FwHeadless/Properties/launchSettings.json similarity index 100% rename from backend/CrdtMerge/Properties/launchSettings.json rename to backend/FwHeadless/Properties/launchSettings.json diff --git a/backend/CrdtMerge/SendReceiveHelpers.cs b/backend/FwHeadless/SendReceiveHelpers.cs similarity index 96% rename from backend/CrdtMerge/SendReceiveHelpers.cs rename to backend/FwHeadless/SendReceiveHelpers.cs index 8eb1404c0..9c8ac9494 100644 --- a/backend/CrdtMerge/SendReceiveHelpers.cs +++ b/backend/FwHeadless/SendReceiveHelpers.cs @@ -1,7 +1,7 @@ using FwDataMiniLcmBridge; using SIL.Progress; -namespace CrdtMerge; +namespace FwHeadless; public static class SendReceiveHelpers { @@ -12,7 +12,7 @@ public record ProjectPath(string Code, string Dir) public record SendReceiveAuth(string Username, string Password) { - public SendReceiveAuth(CrdtMergeConfig config) : this(config.LexboxUsername, config.LexboxPassword) { } + public SendReceiveAuth(FwHeadlessConfig config) : this(config.LexboxUsername, config.LexboxPassword) { } }; public record LfMergeBridgeResult(string Output, string ProgressMessages); diff --git a/backend/CrdtMerge/SendReceiveService.cs b/backend/FwHeadless/SendReceiveService.cs similarity index 91% rename from backend/CrdtMerge/SendReceiveService.cs rename to backend/FwHeadless/SendReceiveService.cs index 14db577ae..37023dc8f 100644 --- a/backend/CrdtMerge/SendReceiveService.cs +++ b/backend/FwHeadless/SendReceiveService.cs @@ -1,9 +1,9 @@ using FwDataMiniLcmBridge; using Microsoft.Extensions.Options; -namespace CrdtMerge; +namespace FwHeadless; -public class SendReceiveService(IOptions config) +public class SendReceiveService(IOptions config) { public SendReceiveHelpers.LfMergeBridgeResult SendReceive(FwDataProject project, string? projectCode, string? commitMessage = null) { diff --git a/backend/CrdtMerge/appsettings.Development.json b/backend/FwHeadless/appsettings.Development.json similarity index 95% rename from backend/CrdtMerge/appsettings.Development.json rename to backend/FwHeadless/appsettings.Development.json index 0995c2046..22a96c2a5 100644 --- a/backend/CrdtMerge/appsettings.Development.json +++ b/backend/FwHeadless/appsettings.Development.json @@ -1,5 +1,5 @@ { - "SendReceiveConfig": { + "FwHeadlessConfig": { "ProjectStorageRoot": "../../hgweb/repos", "LexboxUrl": "http://localhost/", "LexboxUsername": "admin", diff --git a/backend/CrdtMerge/appsettings.json b/backend/FwHeadless/appsettings.json similarity index 87% rename from backend/CrdtMerge/appsettings.json rename to backend/FwHeadless/appsettings.json index 38010f509..d56001975 100644 --- a/backend/CrdtMerge/appsettings.json +++ b/backend/FwHeadless/appsettings.json @@ -1,5 +1,5 @@ { - "SendReceiveConfig": { + "FwHeadlessConfig": { "LexboxUsername": null }, "Logging": { diff --git a/deployment/base/crdtmerge-deployment.yaml b/deployment/base/fw-headless-deployment.yaml similarity index 75% rename from deployment/base/crdtmerge-deployment.yaml rename to deployment/base/fw-headless-deployment.yaml index ae9a7ad1a..f61485e53 100644 --- a/deployment/base/crdtmerge-deployment.yaml +++ b/deployment/base/fw-headless-deployment.yaml @@ -2,15 +2,15 @@ apiVersion: v1 kind: Service metadata: - name: crdtmerge + name: fw-headless namespace: languagedepot labels: - app: crdtmerge + app: fw-headless spec: type: ClusterIP clusterIP: None selector: - app: crdtmerge + app: fw-headless ports: - name: http protocol: TCP @@ -22,14 +22,14 @@ spec: apiVersion: apps/v1 kind: Deployment metadata: - name: crdtmerge + name: fw-headless namespace: languagedepot labels: - app: crdtmerge + app: fw-headless spec: selector: matchLabels: - app: crdtmerge + app: fw-headless strategy: rollingUpdate: maxSurge: 2 @@ -39,15 +39,15 @@ spec: # https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates metadata: labels: - app: crdtmerge + app: fw-headless spec: securityContext: runAsGroup: 33 runAsUser: 33 runAsNonRoot: true containers: - - name: crdtmerge - image: ghcr.io/sillsdev/lexbox-crdtmerge:develop + - name: fw-headless + image: ghcr.io/sillsdev/lexbox-fw-headless:develop imagePullPolicy: IfNotPresent # https://kubernetes.io/docs/concepts/configuration/manage-resources-containers resources: @@ -65,8 +65,8 @@ spec: - containerPort: 80 volumeMounts: - - name: crdtmerge - mountPath: /var/lib/crdtmerge + - name: fw-headless + mountPath: /var/lib/fw-headless env: - name: DOTNET_URLS @@ -96,23 +96,23 @@ spec: name: db - name: DbConfig__LexBoxConnectionString value: Host=db;Port=5432;Username=postgres;Password=$(POSTGRES_PASSWORD);Database=$(POSTGRES_DB) - - name: SendReceiveConfig__ProjectStorageRoot - value: /var/lib/crdtmerge/projects - - name: SendReceiveConfig__LexboxUrl + - name: FwHeadlessConfig__ProjectStorageRoot + value: /var/lib/fw-headless/projects + - name: FwHeadlessConfig__LexboxUrl value: http://lexbox:5158/ - # - name: SendReceiveConfig__HgUrl + # - name: FwHeadlessConfig__HgUrl # value: http://lexbox:5158/hg/ - - name: SendReceiveConfig__LexboxUsername + - name: FwHeadlessConfig__LexboxUsername valueFrom: secretKeyRef: key: CRDT_MERGE_SEND_RECEIVE_USERNAME - name: crdtmerge - - name: SendReceiveConfig__LexboxPassword + name: fw-headless + - name: FwHeadlessConfig__LexboxPassword valueFrom: secretKeyRef: key: CRDT_MERGE_SEND_RECEIVE_PASSWORD - name: crdtmerge - - name: SendReceiveConfig__FdoDataModelVersion + name: fw-headless + - name: FwHeadlessConfig__FdoDataModelVersion value: "7000072" initContainers: @@ -127,13 +127,13 @@ spec: - 'sh' - '-c' - | - mkdir -p /crdtmerge/projects - chown www-data:www-data /crdtmerge/projects + mkdir -p /fw-headless/projects + chown www-data:www-data /fw-headless/projects volumeMounts: - - name: crdtmerge - mountPath: /crdtmerge + - name: fw-headless + mountPath: /fw-headless volumes: - - name: crdtmerge + - name: fw-headless persistentVolumeClaim: - claimName: crdtmerge # established in pvc.yaml + claimName: fw-headless # established in pvc.yaml diff --git a/deployment/base/pvc.yaml b/deployment/base/pvc.yaml index e2e318625..6ea3f7ce7 100644 --- a/deployment/base/pvc.yaml +++ b/deployment/base/pvc.yaml @@ -40,7 +40,7 @@ spec: apiVersion: v1 kind: PersistentVolumeClaim metadata: - name: crdtmerge + name: fw-headless namespace: languagedepot labels: app.kubernetes.io/part-of: languagedepot diff --git a/deployment/base/secrets.yaml b/deployment/base/secrets.yaml index 28b3817c8..7046f12a3 100644 --- a/deployment/base/secrets.yaml +++ b/deployment/base/secrets.yaml @@ -89,7 +89,7 @@ stringData: apiVersion: v1 kind: Secret metadata: - name: crdtmerge + name: fw-headless namespace: languagedepot stringData: CRDT_MERGE_SEND_RECEIVE_USERNAME: '' diff --git a/deployment/develop/crdtmerge-deployment.patch.yaml b/deployment/develop/fw-headless-deployment.patch.yaml similarity index 74% rename from deployment/develop/crdtmerge-deployment.patch.yaml rename to deployment/develop/fw-headless-deployment.patch.yaml index 70dc8eb7d..9df814838 100644 --- a/deployment/develop/crdtmerge-deployment.patch.yaml +++ b/deployment/develop/fw-headless-deployment.patch.yaml @@ -2,21 +2,21 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: crdtmerge + name: fw-headless namespace: languagedepot labels: - app: crdtmerge + app: fw-headless spec: template: spec: containers: - - name: crdtmerge + - name: fw-headless imagePullPolicy: IfNotPresent env: - - name: SendReceiveConfig__LexboxUsername + - name: FwHeadlessConfig__LexboxUsername value: "admin" valueFrom: # don't use secret as defined in base - - name: SendReceiveConfig__LexboxPassword + - name: FwHeadlessConfig__LexboxPassword valueFrom: secretKeyRef: key: SEED_USER_PASSWORD diff --git a/deployment/develop/kustomization.yaml b/deployment/develop/kustomization.yaml index f4eea9c18..c86d42508 100644 --- a/deployment/develop/kustomization.yaml +++ b/deployment/develop/kustomization.yaml @@ -7,7 +7,7 @@ resources: components: - ../init-repos patches: - - path: crdtmerge-deployment.patch.yaml + - path: fw-headless-deployment.patch.yaml - path: lexbox-deployment.patch.yaml - path: app-config.yaml - path: ingress-config.yaml diff --git a/deployment/local-dev/crdtmerge-deployment.patch.yaml b/deployment/local-dev/fw-headless-deployment.patch.yaml similarity index 74% rename from deployment/local-dev/crdtmerge-deployment.patch.yaml rename to deployment/local-dev/fw-headless-deployment.patch.yaml index 70dc8eb7d..9df814838 100644 --- a/deployment/local-dev/crdtmerge-deployment.patch.yaml +++ b/deployment/local-dev/fw-headless-deployment.patch.yaml @@ -2,21 +2,21 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: crdtmerge + name: fw-headless namespace: languagedepot labels: - app: crdtmerge + app: fw-headless spec: template: spec: containers: - - name: crdtmerge + - name: fw-headless imagePullPolicy: IfNotPresent env: - - name: SendReceiveConfig__LexboxUsername + - name: FwHeadlessConfig__LexboxUsername value: "admin" valueFrom: # don't use secret as defined in base - - name: SendReceiveConfig__LexboxPassword + - name: FwHeadlessConfig__LexboxPassword valueFrom: secretKeyRef: key: SEED_USER_PASSWORD diff --git a/deployment/local-dev/kustomization.yaml b/deployment/local-dev/kustomization.yaml index 0c9927242..1fb7ad2fa 100644 --- a/deployment/local-dev/kustomization.yaml +++ b/deployment/local-dev/kustomization.yaml @@ -39,7 +39,7 @@ patches: path: delete-oauth-certs.yaml - path: app-config.yaml - - path: crdtmerge-deployment.patch.yaml + - path: fw-headless-deployment.patch.yaml - path: lexbox-deployment.patch.yaml - path: ui-deployment.patch.yaml - path: hg-repos-pvc.patch.yaml diff --git a/deployment/staging/crdtmerge-deployment.patch.yaml b/deployment/staging/fw-headless-deployment.patch.yaml similarity index 74% rename from deployment/staging/crdtmerge-deployment.patch.yaml rename to deployment/staging/fw-headless-deployment.patch.yaml index 70dc8eb7d..9df814838 100644 --- a/deployment/staging/crdtmerge-deployment.patch.yaml +++ b/deployment/staging/fw-headless-deployment.patch.yaml @@ -2,21 +2,21 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: crdtmerge + name: fw-headless namespace: languagedepot labels: - app: crdtmerge + app: fw-headless spec: template: spec: containers: - - name: crdtmerge + - name: fw-headless imagePullPolicy: IfNotPresent env: - - name: SendReceiveConfig__LexboxUsername + - name: FwHeadlessConfig__LexboxUsername value: "admin" valueFrom: # don't use secret as defined in base - - name: SendReceiveConfig__LexboxPassword + - name: FwHeadlessConfig__LexboxPassword valueFrom: secretKeyRef: key: SEED_USER_PASSWORD diff --git a/deployment/staging/kustomization.yaml b/deployment/staging/kustomization.yaml index c863fa415..0d651c3c8 100644 --- a/deployment/staging/kustomization.yaml +++ b/deployment/staging/kustomization.yaml @@ -7,7 +7,7 @@ resources: components: - ../init-repos patches: - - path: crdtmerge-deployment.patch.yaml + - path: fw-headless-deployment.patch.yaml - path: lexbox-deployment.patch.yaml - path: app-config.yaml - path: hg-repos-volume.yaml diff --git a/skaffold.yaml b/skaffold.yaml index fdcea20fe..5f6e82600 100644 --- a/skaffold.yaml +++ b/skaffold.yaml @@ -14,10 +14,10 @@ build: manual: - src: '**' dest: /src/backend - - image: ghcr.io/sillsdev/lexbox-crdtmerge + - image: ghcr.io/sillsdev/lexbox-fw-headless context: backend docker: - dockerfile: CrdtMerge/Dockerfile + dockerfile: FwHeadless/Dockerfile - image: ghcr.io/sillsdev/lexbox-ui context: frontend docker: @@ -45,9 +45,9 @@ portForward: resourceName: lexbox namespace: languagedepot port: 5158 - # CrdtMerge + # FwHeadless - resourceType: Service - resourceName: crdtmerge + resourceName: fw-headless namespace: languagedepot port: 80 localPort: 5275 @@ -99,9 +99,9 @@ profiles: namespace: languagedepot port: 5432 localPort: 5433 - # CrdtMerge + # FwHeadless - resourceType: Service - resourceName: crdtmerge + resourceName: fw-headless namespace: languagedepot port: 80 localPort: 5275 @@ -157,9 +157,9 @@ profiles: resourceName: lexbox namespace: languagedepot port: 5158 - # CrdtMerge + # FwHeadless - resourceType: Service - resourceName: crdtmerge + resourceName: fw-headless namespace: languagedepot port: 80 localPort: 5275 From acf0a2742f479f735a78b916bf0a588d0c8c9641 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Thu, 31 Oct 2024 08:41:25 +0700 Subject: [PATCH 09/16] Size fw-headless volume according to enironment Will request: - 10Gi on GHA - 20Gi on local-dev - 20Gi on develop - 40Gi on staging - 80Gi on prod --- deployment/base/pvc.yaml | 2 +- deployment/develop/change-allocation-size.patch.yaml | 11 +++++++++++ deployment/develop/kustomization.yaml | 7 +++++++ deployment/gha/change-allocation-size.patch.yaml | 11 +++++++++++ deployment/gha/kustomization.yaml | 7 +++++++ .../local-dev/change-allocation-size.patch.yaml | 11 +++++++++++ deployment/local-dev/kustomization.yaml | 7 +++++++ deployment/staging/change-allocation-size.patch.yaml | 11 +++++++++++ deployment/staging/kustomization.yaml | 7 +++++++ 9 files changed, 73 insertions(+), 1 deletion(-) create mode 100644 deployment/develop/change-allocation-size.patch.yaml create mode 100644 deployment/gha/change-allocation-size.patch.yaml create mode 100644 deployment/local-dev/change-allocation-size.patch.yaml create mode 100644 deployment/staging/change-allocation-size.patch.yaml diff --git a/deployment/base/pvc.yaml b/deployment/base/pvc.yaml index 6ea3f7ce7..61dd8dcbb 100644 --- a/deployment/base/pvc.yaml +++ b/deployment/base/pvc.yaml @@ -49,5 +49,5 @@ spec: - ReadWriteOnce resources: requests: - storage: 10Gi + storage: 80Gi storageClassName: weekly-snapshots-retain-4 # provided by LTOps diff --git a/deployment/develop/change-allocation-size.patch.yaml b/deployment/develop/change-allocation-size.patch.yaml new file mode 100644 index 000000000..f299da9e4 --- /dev/null +++ b/deployment/develop/change-allocation-size.patch.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: fw-headless + namespace: languagedepot + labels: + app.kubernetes.io/part-of: languagedepot +spec: + resources: + requests: + storage: 20Gi diff --git a/deployment/develop/kustomization.yaml b/deployment/develop/kustomization.yaml index c86d42508..7aedd20c8 100644 --- a/deployment/develop/kustomization.yaml +++ b/deployment/develop/kustomization.yaml @@ -15,3 +15,10 @@ patches: kind: Ingress name: proxy namespace: languagedepot + - target: + version: v1 + kind: PersistentVolumeClaim + metadata: + name: fw-headless + namespace: languagedepot + path: change-allocation-size.patch.yaml diff --git a/deployment/gha/change-allocation-size.patch.yaml b/deployment/gha/change-allocation-size.patch.yaml new file mode 100644 index 000000000..1ddd07502 --- /dev/null +++ b/deployment/gha/change-allocation-size.patch.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: fw-headless + namespace: languagedepot + labels: + app.kubernetes.io/part-of: languagedepot +spec: + resources: + requests: + storage: 10Gi diff --git a/deployment/gha/kustomization.yaml b/deployment/gha/kustomization.yaml index d3a7091e4..4e69870f1 100644 --- a/deployment/gha/kustomization.yaml +++ b/deployment/gha/kustomization.yaml @@ -11,6 +11,13 @@ patches: version: v1 kind: PersistentVolumeClaim path: change-storage-class.patch.yaml + - target: + version: v1 + kind: PersistentVolumeClaim + metadata: + name: fw-headless + namespace: languagedepot + path: change-allocation-size.patch.yaml - path: app-config.yaml images: diff --git a/deployment/local-dev/change-allocation-size.patch.yaml b/deployment/local-dev/change-allocation-size.patch.yaml new file mode 100644 index 000000000..f299da9e4 --- /dev/null +++ b/deployment/local-dev/change-allocation-size.patch.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: fw-headless + namespace: languagedepot + labels: + app.kubernetes.io/part-of: languagedepot +spec: + resources: + requests: + storage: 20Gi diff --git a/deployment/local-dev/kustomization.yaml b/deployment/local-dev/kustomization.yaml index 1fb7ad2fa..2f1566b7d 100644 --- a/deployment/local-dev/kustomization.yaml +++ b/deployment/local-dev/kustomization.yaml @@ -31,6 +31,13 @@ patches: version: v1 kind: PersistentVolumeClaim path: change-storage-class.patch.yaml + - target: + version: v1 + kind: PersistentVolumeClaim + metadata: + name: fw-headless + namespace: languagedepot + path: change-allocation-size.patch.yaml - target: kind: Issuer path: delete-oauth-certs.yaml diff --git a/deployment/staging/change-allocation-size.patch.yaml b/deployment/staging/change-allocation-size.patch.yaml new file mode 100644 index 000000000..ac4a15b29 --- /dev/null +++ b/deployment/staging/change-allocation-size.patch.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: fw-headless + namespace: languagedepot + labels: + app.kubernetes.io/part-of: languagedepot +spec: + resources: + requests: + storage: 40Gi diff --git a/deployment/staging/kustomization.yaml b/deployment/staging/kustomization.yaml index 0d651c3c8..e7233002e 100644 --- a/deployment/staging/kustomization.yaml +++ b/deployment/staging/kustomization.yaml @@ -16,3 +16,10 @@ patches: kind: Ingress name: proxy namespace: languagedepot + - target: + version: v1 + kind: PersistentVolumeClaim + metadata: + name: fw-headless + namespace: languagedepot + path: change-allocation-size.patch.yaml From bc377eb9c77f71a5b2caa80ec357f8adbf095765 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Thu, 31 Oct 2024 10:44:52 +0700 Subject: [PATCH 10/16] Revert "Size fw-headless volume according to enironment" This reverts commit acf0a2742f479f735a78b916bf0a588d0c8c9641. We'll do this the simpler way. --- deployment/base/pvc.yaml | 2 +- deployment/develop/change-allocation-size.patch.yaml | 11 ----------- deployment/develop/kustomization.yaml | 7 ------- deployment/gha/change-allocation-size.patch.yaml | 11 ----------- deployment/gha/kustomization.yaml | 7 ------- .../local-dev/change-allocation-size.patch.yaml | 11 ----------- deployment/local-dev/kustomization.yaml | 7 ------- deployment/staging/change-allocation-size.patch.yaml | 11 ----------- deployment/staging/kustomization.yaml | 7 ------- 9 files changed, 1 insertion(+), 73 deletions(-) delete mode 100644 deployment/develop/change-allocation-size.patch.yaml delete mode 100644 deployment/gha/change-allocation-size.patch.yaml delete mode 100644 deployment/local-dev/change-allocation-size.patch.yaml delete mode 100644 deployment/staging/change-allocation-size.patch.yaml diff --git a/deployment/base/pvc.yaml b/deployment/base/pvc.yaml index 61dd8dcbb..6ea3f7ce7 100644 --- a/deployment/base/pvc.yaml +++ b/deployment/base/pvc.yaml @@ -49,5 +49,5 @@ spec: - ReadWriteOnce resources: requests: - storage: 80Gi + storage: 10Gi storageClassName: weekly-snapshots-retain-4 # provided by LTOps diff --git a/deployment/develop/change-allocation-size.patch.yaml b/deployment/develop/change-allocation-size.patch.yaml deleted file mode 100644 index f299da9e4..000000000 --- a/deployment/develop/change-allocation-size.patch.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: fw-headless - namespace: languagedepot - labels: - app.kubernetes.io/part-of: languagedepot -spec: - resources: - requests: - storage: 20Gi diff --git a/deployment/develop/kustomization.yaml b/deployment/develop/kustomization.yaml index 7aedd20c8..c86d42508 100644 --- a/deployment/develop/kustomization.yaml +++ b/deployment/develop/kustomization.yaml @@ -15,10 +15,3 @@ patches: kind: Ingress name: proxy namespace: languagedepot - - target: - version: v1 - kind: PersistentVolumeClaim - metadata: - name: fw-headless - namespace: languagedepot - path: change-allocation-size.patch.yaml diff --git a/deployment/gha/change-allocation-size.patch.yaml b/deployment/gha/change-allocation-size.patch.yaml deleted file mode 100644 index 1ddd07502..000000000 --- a/deployment/gha/change-allocation-size.patch.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: fw-headless - namespace: languagedepot - labels: - app.kubernetes.io/part-of: languagedepot -spec: - resources: - requests: - storage: 10Gi diff --git a/deployment/gha/kustomization.yaml b/deployment/gha/kustomization.yaml index 4e69870f1..d3a7091e4 100644 --- a/deployment/gha/kustomization.yaml +++ b/deployment/gha/kustomization.yaml @@ -11,13 +11,6 @@ patches: version: v1 kind: PersistentVolumeClaim path: change-storage-class.patch.yaml - - target: - version: v1 - kind: PersistentVolumeClaim - metadata: - name: fw-headless - namespace: languagedepot - path: change-allocation-size.patch.yaml - path: app-config.yaml images: diff --git a/deployment/local-dev/change-allocation-size.patch.yaml b/deployment/local-dev/change-allocation-size.patch.yaml deleted file mode 100644 index f299da9e4..000000000 --- a/deployment/local-dev/change-allocation-size.patch.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: fw-headless - namespace: languagedepot - labels: - app.kubernetes.io/part-of: languagedepot -spec: - resources: - requests: - storage: 20Gi diff --git a/deployment/local-dev/kustomization.yaml b/deployment/local-dev/kustomization.yaml index 2f1566b7d..1fb7ad2fa 100644 --- a/deployment/local-dev/kustomization.yaml +++ b/deployment/local-dev/kustomization.yaml @@ -31,13 +31,6 @@ patches: version: v1 kind: PersistentVolumeClaim path: change-storage-class.patch.yaml - - target: - version: v1 - kind: PersistentVolumeClaim - metadata: - name: fw-headless - namespace: languagedepot - path: change-allocation-size.patch.yaml - target: kind: Issuer path: delete-oauth-certs.yaml diff --git a/deployment/staging/change-allocation-size.patch.yaml b/deployment/staging/change-allocation-size.patch.yaml deleted file mode 100644 index ac4a15b29..000000000 --- a/deployment/staging/change-allocation-size.patch.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: fw-headless - namespace: languagedepot - labels: - app.kubernetes.io/part-of: languagedepot -spec: - resources: - requests: - storage: 40Gi diff --git a/deployment/staging/kustomization.yaml b/deployment/staging/kustomization.yaml index e7233002e..0d651c3c8 100644 --- a/deployment/staging/kustomization.yaml +++ b/deployment/staging/kustomization.yaml @@ -16,10 +16,3 @@ patches: kind: Ingress name: proxy namespace: languagedepot - - target: - version: v1 - kind: PersistentVolumeClaim - metadata: - name: fw-headless - namespace: languagedepot - path: change-allocation-size.patch.yaml From 53c7cb338517ec6e7d210ee3446d450fe81a4d3a Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Thu, 31 Oct 2024 10:47:23 +0700 Subject: [PATCH 11/16] Simpler way to handle fw-headless volume size Now we allocate 10 GiB everywhere, except 20 on staging and 150 GiB on production (similar size to hg-repos volume since we could potentially end up with copies of everything from hg-repos). --- deployment/production/fw-headless-volume.yaml | 9 +++++++++ deployment/production/kustomization.yaml | 1 + deployment/staging/fw-headless-volume.yaml | 9 +++++++++ deployment/staging/kustomization.yaml | 1 + 4 files changed, 20 insertions(+) create mode 100644 deployment/production/fw-headless-volume.yaml create mode 100644 deployment/staging/fw-headless-volume.yaml diff --git a/deployment/production/fw-headless-volume.yaml b/deployment/production/fw-headless-volume.yaml new file mode 100644 index 000000000..6ff2a11c1 --- /dev/null +++ b/deployment/production/fw-headless-volume.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: hg-repos + namespace: languagedepot +spec: + resources: + requests: + storage: 150Gi diff --git a/deployment/production/kustomization.yaml b/deployment/production/kustomization.yaml index f35c39514..e79de016c 100644 --- a/deployment/production/kustomization.yaml +++ b/deployment/production/kustomization.yaml @@ -9,6 +9,7 @@ resources: patches: - path: lexbox-deployment.patch.yaml - path: app-config.yaml + - path: fw-headless-volume.yaml - path: hg-repos-volume.yaml - path: hg-deployment.yaml - path: ingress-config-prod.yaml diff --git a/deployment/staging/fw-headless-volume.yaml b/deployment/staging/fw-headless-volume.yaml new file mode 100644 index 000000000..97bb99558 --- /dev/null +++ b/deployment/staging/fw-headless-volume.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: fw-headless + namespace: languagedepot +spec: + resources: + requests: + storage: 30Gi diff --git a/deployment/staging/kustomization.yaml b/deployment/staging/kustomization.yaml index 0d651c3c8..c56c8d483 100644 --- a/deployment/staging/kustomization.yaml +++ b/deployment/staging/kustomization.yaml @@ -7,6 +7,7 @@ resources: components: - ../init-repos patches: + - path: fw-headless-volume.yaml - path: fw-headless-deployment.patch.yaml - path: lexbox-deployment.patch.yaml - path: app-config.yaml From 11e2a4222f45a48f4975a53bbddedffdf330b7c3 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Thu, 31 Oct 2024 12:10:58 +0700 Subject: [PATCH 12/16] Add fw-headless-deployment to base kustomize.yaml --- deployment/base/kustomization.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/deployment/base/kustomization.yaml b/deployment/base/kustomization.yaml index fe9002a1a..0dc486889 100644 --- a/deployment/base/kustomization.yaml +++ b/deployment/base/kustomization.yaml @@ -9,6 +9,7 @@ resources: - pvc.yaml - db-deployment.yaml - hg-deployment.yaml +- fw-headless-deployment.yaml - lexbox-deployment.yaml - ui-deployment.yaml - proxy-deployment.yaml From f5a47fb9a326dd9d059f0e9142e65c13250a65ed Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Thu, 31 Oct 2024 12:31:28 +0700 Subject: [PATCH 13/16] Ensure Chorus can save user settings Chorus uses the XDG_DATA_HOME environment variable to save its settings. It doesn't exist in the base .NET SDK image, so we need to set it. We set it in the Dockerfile rather than in the k8s deployment because it should be the same everywhere. --- backend/FwHeadless/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/FwHeadless/Dockerfile b/backend/FwHeadless/Dockerfile index 0b24a4f92..762c416ac 100644 --- a/backend/FwHeadless/Dockerfile +++ b/backend/FwHeadless/Dockerfile @@ -18,8 +18,9 @@ FROM build AS publish RUN --mount=type=cache,target=/root/.nuget/packages dotnet publish /p:InformationalVersion=$APP_VERSION "FwHeadless/FwHeadless.csproj" -c Release -o /app/publish FROM base AS final -RUN mkdir -p /var/lib/fw-headless && chown -R www-data:www-data /var/lib/fw-headless +RUN mkdir -p /var/lib/fw-headless /var/www/.local/share && chown -R www-data:www-data /var/lib/fw-headless /var/www/.local/share WORKDIR /app COPY --from=publish /app/publish . USER www-data:www-data +ENV XDG_DATA_HOME=/var/www/.local/share ENTRYPOINT ["dotnet", "FwHeadless.dll"] From dee6a106f6a876e44123b64f574cdef468673b05 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Thu, 31 Oct 2024 12:47:44 +0700 Subject: [PATCH 14/16] Fix problems running Mercurial in container --- backend/FwHeadless/Dockerfile | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/backend/FwHeadless/Dockerfile b/backend/FwHeadless/Dockerfile index 762c416ac..1c6303a01 100644 --- a/backend/FwHeadless/Dockerfile +++ b/backend/FwHeadless/Dockerfile @@ -19,8 +19,15 @@ RUN --mount=type=cache,target=/root/.nuget/packages dotnet publish /p:Informatio FROM base AS final RUN mkdir -p /var/lib/fw-headless /var/www/.local/share && chown -R www-data:www-data /var/lib/fw-headless /var/www/.local/share +RUN apt-get update \ + && apt-get install --yes --no-install-recommends tini iputils-ping python3 \ + && rm -rf /var/lib/apt/lists/* WORKDIR /app COPY --from=publish /app/publish . +# Ensure Mercurial exec bit was not stripped by dotnet CLI tools +RUN chmod +x Mercurial/hg && chmod +x Mercurial/chg >/dev/null || true +# Fix up mercurial.ini path to fixutf8 +RUN sed -i -e 's/fixutf8 = \/FwHeadless/fixutf8 = \/app/' Mercurial/mercurial.ini USER www-data:www-data ENV XDG_DATA_HOME=/var/www/.local/share ENTRYPOINT ["dotnet", "FwHeadless.dll"] From d4107c11acfe9934e8ababed8f7057f7d2abf156 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Thu, 31 Oct 2024 12:56:18 +0700 Subject: [PATCH 15/16] Handle Unix signals in FwHeadless container Should make for cleaner shutdown and restart on deployment --- backend/FwHeadless/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/FwHeadless/Dockerfile b/backend/FwHeadless/Dockerfile index 1c6303a01..c57bab474 100644 --- a/backend/FwHeadless/Dockerfile +++ b/backend/FwHeadless/Dockerfile @@ -30,4 +30,5 @@ RUN chmod +x Mercurial/hg && chmod +x Mercurial/chg >/dev/null || true RUN sed -i -e 's/fixutf8 = \/FwHeadless/fixutf8 = \/app/' Mercurial/mercurial.ini USER www-data:www-data ENV XDG_DATA_HOME=/var/www/.local/share -ENTRYPOINT ["dotnet", "FwHeadless.dll"] +ENTRYPOINT ["tini", "--"] +CMD ["dotnet", "FwHeadless.dll"] From cad8585b9d8c2c61fd7c4e86e3157a2f63cdda19 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Thu, 31 Oct 2024 13:05:41 +0700 Subject: [PATCH 16/16] Correctly ignore chmod error message in Dockerfile --- backend/FwHeadless/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/FwHeadless/Dockerfile b/backend/FwHeadless/Dockerfile index c57bab474..022c350c4 100644 --- a/backend/FwHeadless/Dockerfile +++ b/backend/FwHeadless/Dockerfile @@ -25,7 +25,7 @@ RUN apt-get update \ WORKDIR /app COPY --from=publish /app/publish . # Ensure Mercurial exec bit was not stripped by dotnet CLI tools -RUN chmod +x Mercurial/hg && chmod +x Mercurial/chg >/dev/null || true +RUN chmod +x Mercurial/hg && chmod +x Mercurial/chg 2>/dev/null || true # Fix up mercurial.ini path to fixutf8 RUN sed -i -e 's/fixutf8 = \/FwHeadless/fixutf8 = \/app/' Mercurial/mercurial.ini USER www-data:www-data