From c7c515dc308d8177481de3f97785b109074a2530 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Thu, 31 Oct 2024 13:34:18 +0700 Subject: [PATCH] Add FwHeadless (renamed from CrdtMerge) deployment for k8s and GHA (#1171) This adds a new k8s deployment for FwHeadless (renamed from CrdtMerge) in its own pod, and adds GHA workflow steps to build and deploy it. --- .github/workflows/deploy-branch.yaml | 10 +- .github/workflows/deploy.yaml | 6 + .github/workflows/develop-fw-headless.yaml | 72 +++++++++ .github/workflows/integration-test-gha.yaml | 11 +- .github/workflows/lexbox-fw-headless.yaml | 99 +++++++++++++ .github/workflows/release-pipeline.yaml | 10 +- LexBox.sln | 2 +- backend/FwHeadless/Dockerfile | 34 +++++ .../FwHeadless.csproj} | 0 .../FwHeadlessConfig.cs} | 4 +- .../FwHeadlessKernel.cs} | 10 +- backend/{CrdtMerge => FwHeadless}/Program.cs | 10 +- .../ProjectLookupService.cs | 2 +- .../Properties/launchSettings.json | 0 .../SendReceiveHelpers.cs | 4 +- .../SendReceiveService.cs | 4 +- .../appsettings.Development.json | 2 +- .../appsettings.json | 2 +- deployment/base/fw-headless-deployment.yaml | 139 ++++++++++++++++++ deployment/base/kustomization.yaml | 1 + deployment/base/pvc.yaml | 18 +++ deployment/base/secrets.yaml | 10 ++ .../develop/fw-headless-deployment.patch.yaml | 24 +++ deployment/develop/kustomization.yaml | 1 + .../fw-headless-deployment.patch.yaml | 24 +++ deployment/local-dev/kustomization.yaml | 1 + deployment/production/fw-headless-volume.yaml | 9 ++ deployment/production/kustomization.yaml | 1 + .../staging/fw-headless-deployment.patch.yaml | 24 +++ deployment/staging/fw-headless-volume.yaml | 9 ++ deployment/staging/kustomization.yaml | 2 + skaffold.yaml | 22 +++ 32 files changed, 546 insertions(+), 21 deletions(-) create mode 100644 .github/workflows/develop-fw-headless.yaml create mode 100644 .github/workflows/lexbox-fw-headless.yaml create mode 100644 backend/FwHeadless/Dockerfile rename backend/{CrdtMerge/CrdtMerge.csproj => FwHeadless/FwHeadless.csproj} (100%) rename backend/{CrdtMerge/CrdtMergeConfig.cs => FwHeadless/FwHeadlessConfig.cs} (90%) rename backend/{CrdtMerge/CrdtMergeKernel.cs => FwHeadless/FwHeadlessKernel.cs} (69%) rename backend/{CrdtMerge => FwHeadless}/Program.cs (95%) rename backend/{CrdtMerge => FwHeadless}/ProjectLookupService.cs (94%) rename backend/{CrdtMerge => FwHeadless}/Properties/launchSettings.json (100%) rename backend/{CrdtMerge => FwHeadless}/SendReceiveHelpers.cs (96%) rename backend/{CrdtMerge => FwHeadless}/SendReceiveService.cs (91%) rename backend/{CrdtMerge => FwHeadless}/appsettings.Development.json (95%) rename backend/{CrdtMerge => FwHeadless}/appsettings.json (87%) create mode 100644 deployment/base/fw-headless-deployment.yaml create mode 100644 deployment/develop/fw-headless-deployment.patch.yaml create mode 100644 deployment/local-dev/fw-headless-deployment.patch.yaml create mode 100644 deployment/production/fw-headless-volume.yaml create mode 100644 deployment/staging/fw-headless-deployment.patch.yaml create mode 100644 deployment/staging/fw-headless-volume.yaml diff --git a/.github/workflows/deploy-branch.yaml b/.github/workflows/deploy-branch.yaml index f8a199eff..5746a2520 100644 --- a/.github/workflows/deploy-branch.yaml +++ b/.github/workflows/deploy-branch.yaml @@ -41,10 +41,18 @@ jobs: version: ${{ needs.set-version.outputs.version }} label-latest: false + build-fw-headless: + name: Build FwHeadless + needs: [ set-version ] + uses: ./.github/workflows/lexbox-fw-headless.yaml + with: + version: ${{ needs.set-version.outputs.version }} + label-latest: false + deploy: name: Deploy Develop uses: ./.github/workflows/deploy.yaml - needs: [ build-api, build-ui, build-hgweb, set-version ] + needs: [ build-api, build-ui, build-hgweb, build-fw-headless, set-version ] secrets: inherit with: version: ${{ needs.set-version.outputs.version }} diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index 0ab79e53e..45d5e8045 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -50,6 +50,7 @@ jobs: url: https://${{ inputs.deploy-domain }} outputs: api-version: ${{ steps.get-api-version.outputs.result }} + fw-headless-version: ${{ steps.get-fw-headless-version.outputs.result }} ui-version: ${{ steps.get-ui-version.outputs.result }} steps: - name: Checkout lexbox repo @@ -80,6 +81,11 @@ jobs: id: get-api-version with: cmd: yq '.images.[] | select(.name == "ghcr.io/sillsdev/lexbox-api").newTag' "fleet/${{ inputs.k8s-environment }}/kustomization.yaml" + - name: Get FwHeadless version + uses: mikefarah/yq@0b34c9a00de1c575a34eea05af1d956a525c4fc1 # v4.34.2 + id: get-fw-headless-version + with: + cmd: yq '.images.[] | select(.name == "ghcr.io/sillsdev/lexbox-fw-headless").newTag' "fleet/${{ inputs.k8s-environment }}/kustomization.yaml" - name: Get UI version uses: mikefarah/yq@0b34c9a00de1c575a34eea05af1d956a525c4fc1 # v4.34.2 id: get-ui-version diff --git a/.github/workflows/develop-fw-headless.yaml b/.github/workflows/develop-fw-headless.yaml new file mode 100644 index 000000000..6037f51a2 --- /dev/null +++ b/.github/workflows/develop-fw-headless.yaml @@ -0,0 +1,72 @@ +name: Develop FwHeadless CI/CD +on: + workflow_dispatch: + push: + paths: + - 'backend/FwHeadless/**' + - 'backend/FixFwData/**' + - 'backend/FwLite/FwDataMiniLcmBridge/**' + - 'backend/FwLite/LcmCrdt/**' + - 'backend/FwLite/MiniLcm/**' + - 'backend/FwLiteProjectSync/FwLiteProjectSync/**' + - 'backend/LexCore/**' + - 'backend/LexData/**' + - '.github/workflows/lexbox-fw-headless.yaml' + - '.github/workflows/deploy.yaml' + - 'deployment/base/fw-headless-deployment.yaml' + branches: + - develop + pull_request: + paths: + - 'backend/FwHeadless/**' + - 'backend/FixFwData/**' + - 'backend/FwLite/FwDataMiniLcmBridge/**' + - 'backend/FwLite/LcmCrdt/**' + - 'backend/FwLite/MiniLcm/**' + - 'backend/FwLiteProjectSync/FwLiteProjectSync/**' + - 'backend/LexCore/**' + - 'backend/LexData/**' + - '.github/workflows/lexbox-fw-headless.yaml' + - '.github/workflows/deploy.yaml' + - 'deployment/base/fw-headless-deployment.yaml' + branches: + - develop + +jobs: + set-version: + name: Set Version + runs-on: ubuntu-latest + outputs: + version: ${{ steps.setVersion.outputs.VERSION }} + steps: + - name: Set Version + id: setVersion + # set version to date in vYYYY-MM-DD-commitSha format + run: | + shortSha=$(echo ${{ github.sha }} | cut -c1-8) + echo "VERSION=v$(date --rfc-3339=date)-$shortSha" >> ${GITHUB_OUTPUT} + build-fw-headless: + name: Build FwHeadless + needs: set-version + uses: ./.github/workflows/lexbox-fw-headless.yaml + with: + version: ${{ needs.set-version.outputs.version }} + deploy-fw-headless: + name: Deploy FwHeadless + if: ${{github.ref == 'refs/heads/develop'}} + needs: [ build-fw-headless, set-version ] + uses: ./.github/workflows/deploy.yaml + secrets: inherit + with: + version: ${{ needs.set-version.outputs.version }} + image: 'ghcr.io/sillsdev/lexbox-fw-headless' + k8s-environment: develop + deploy-domain: lexbox.dev.languagetechnology.org + + # TODO: Run FwHeadless tests once we have developed them, but we don't need to run the whole integration test suite if only FwHeadless changes are being pushed + # integration-test-gha: + # name: GHA integration tests + # needs: [build-fw-headless, set-version] + # uses: ./.github/workflows/integration-test-gha.yaml + # with: + # lexbox-fw-headless-tag: ${{ needs.set-version.outputs.version }} diff --git a/.github/workflows/integration-test-gha.yaml b/.github/workflows/integration-test-gha.yaml index 8161dcb22..98a5dbe86 100644 --- a/.github/workflows/integration-test-gha.yaml +++ b/.github/workflows/integration-test-gha.yaml @@ -39,7 +39,16 @@ jobs: uses: mikefarah/yq@0b34c9a00de1c575a34eea05af1d956a525c4fc1 # v4.34.2 with: cmd: yq eval -i '(.images.[] | select(.name == "ghcr.io/sillsdev/lexbox-api").newTag) = "${{ inputs.lexbox-api-tag }}"' "./deployment/gha/kustomization.yaml" - # It's also possible that hgweb and/or ui image may have changed; if so, pull them and update kustomization.yaml for them as well + # It's also possible that hgweb, fw-headless, and/or ui image may have changed; if so, pull them and update kustomization.yaml for them as well + - name: Pull fw-headless if updated + id: fw-headless_image + continue-on-error: true + run: docker pull ghcr.io/sillsdev/lexbox-fw-headless:${{ inputs.lexbox-api-tag }} + - name: Update image fw-headless version + if: ${{ steps.fw-headless_image.outcome == 'success' }} + uses: mikefarah/yq@0b34c9a00de1c575a34eea05af1d956a525c4fc1 # v4.34.2 + with: + cmd: yq eval -i '(.images.[] | select(.name == "ghcr.io/sillsdev/lexbox-fw-headless").newTag) = "${{ inputs.lexbox-api-tag }}"' "./deployment/gha/kustomization.yaml" - name: Pull hgweb if updated id: hgweb_image continue-on-error: true diff --git a/.github/workflows/lexbox-fw-headless.yaml b/.github/workflows/lexbox-fw-headless.yaml new file mode 100644 index 000000000..3d49847ea --- /dev/null +++ b/.github/workflows/lexbox-fw-headless.yaml @@ -0,0 +1,99 @@ +name: Build FwHeadless + +# https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#on +on: + workflow_call: + inputs: + version: + description: 'The version of the image to build' + required: true + type: string + label-latest: + description: 'The label to apply to the latest image' + type: boolean + default: false + +env: + IMAGE_NAME: ghcr.io/sillsdev/lexbox-fw-headless + + +jobs: + publish-fw-headless: + timeout-minutes: 60 + runs-on: ubuntu-latest + + # postgres db is for automated tests + # services: + # postgres: + # image: postgres:15-alpine + # env: + # POSTGRES_PASSWORD: 972b722e63f549938d07bd8c4ee5086c + # POSTGRES_DB: lexbox-tests + # # Set health checks to wait until postgres has started + # options: >- + # --health-cmd pg_isready + # --health-interval 10s + # --health-timeout 5s + # --health-retries 5 + # ports: + # # Maps tcp port 5432 on service container to the host + # - 5433:5432 + + env: + # https://docs.docker.com/develop/develop-images/build_enhancements/ + DOCKER_BUILDKIT: 1 + + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - uses: actions/setup-dotnet@v4 + with: + dotnet-version: '9.x' + - name: Dotnet build + run: dotnet build backend/FwHeadless/FwHeadless.csproj + # TODO: Write FwHeadless unit tests, probably based on existing sync tests + # - name: Unit tests + # run: dotnet test backend/FwHeadless/FwHeadless.csproj --logger:"xunit;LogFileName={assembly}.results.xml" --results-directory ./test-results --filter "Category!=Integration&Category!=FlakyIntegration" --blame-hang-timeout 10m + # - name: Publish unit test results + # uses: EnricoMi/publish-unit-test-result-action@8885e273a4343cd7b48eaa72428dea0c3067ea98 # v2.14.0 + # if: always() + # with: + # check_name: C# Unit Tests + # files: ./test-results/*.xml + # - name: Upload test results + # if: always() + # uses: actions/upload-artifact@v4 + # with: + # name: dotnet-unit-test-results + # path: ./test-results + + - name: Docker meta + id: meta + if: ${{ !env.ACT }} + uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1 + with: + images: ${{ env.IMAGE_NAME }} + tags: | + type=ref,event=branch + type=ref,event=pr + type=raw,enable=${{ inputs.label-latest }},value=latest + type=raw,value=${{ inputs.version }} + + - name: ghcr.io login + uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3.1.0 + if: ${{ !env.ACT }} + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0 + with: + context: backend + file: backend/FwHeadless/Dockerfile + build-args: | + APP_VERSION=${{ inputs.version }} + push: ${{ !env.ACT && github.repository == 'sillsdev/languageforge-lexbox' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/release-pipeline.yaml b/.github/workflows/release-pipeline.yaml index 92450db98..006e32548 100644 --- a/.github/workflows/release-pipeline.yaml +++ b/.github/workflows/release-pipeline.yaml @@ -43,10 +43,18 @@ jobs: version: ${{ needs.set-version.outputs.version }} label-latest: true + build-fw-headless: + name: Build fw-headless + needs: [ set-version ] + uses: ./.github/workflows/lexbox-fw-headless.yaml + with: + version: ${{ needs.set-version.outputs.version }} + label-latest: true + deploy: name: Deploy Staging uses: ./.github/workflows/deploy.yaml - needs: [ build-api, build-ui, build-hgweb, set-version ] + needs: [ build-api, build-ui, build-hgweb, build-fw-headless, set-version ] secrets: inherit with: version: ${{ needs.set-version.outputs.version }} diff --git a/LexBox.sln b/LexBox.sln index 22a2bad17..612c0e95e 100644 --- a/LexBox.sln +++ b/LexBox.sln @@ -51,7 +51,7 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LcmDebugger", "backend\LfNe EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MiniLcm.Tests", "backend\FwLite\MiniLcm.Tests\MiniLcm.Tests.csproj", "{00AE5440-0E36-4488-935B-5B11301BA57D}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CrdtMerge", "backend\CrdtMerge\CrdtMerge.csproj", "{ECBA46AB-AF87-4D4D-9716-FD77264B817F}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FwHeadless", "backend\FwHeadless\FwHeadless.csproj", "{ECBA46AB-AF87-4D4D-9716-FD77264B817F}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution diff --git a/backend/FwHeadless/Dockerfile b/backend/FwHeadless/Dockerfile new file mode 100644 index 000000000..022c350c4 --- /dev/null +++ b/backend/FwHeadless/Dockerfile @@ -0,0 +1,34 @@ +# syntax=docker/dockerfile:1 +FROM mcr.microsoft.com/dotnet/aspnet:9.0 AS base +WORKDIR /app +EXPOSE 80 +EXPOSE 443 + +FROM mcr.microsoft.com/dotnet/sdk:9.0 AS build + +COPY . . +RUN --mount=type=cache,target=/root/.nuget/packages dotnet restore "FwHeadless/FwHeadless.csproj" + +ARG APP_VERSION +LABEL version=$APP_VERSION + +RUN --mount=type=cache,target=/root/.nuget/packages dotnet build /p:InformationalVersion=$APP_VERSION "FwHeadless/FwHeadless.csproj" -c Release -o /app/build + +FROM build AS publish +RUN --mount=type=cache,target=/root/.nuget/packages dotnet publish /p:InformationalVersion=$APP_VERSION "FwHeadless/FwHeadless.csproj" -c Release -o /app/publish + +FROM base AS final +RUN mkdir -p /var/lib/fw-headless /var/www/.local/share && chown -R www-data:www-data /var/lib/fw-headless /var/www/.local/share +RUN apt-get update \ + && apt-get install --yes --no-install-recommends tini iputils-ping python3 \ + && rm -rf /var/lib/apt/lists/* +WORKDIR /app +COPY --from=publish /app/publish . +# Ensure Mercurial exec bit was not stripped by dotnet CLI tools +RUN chmod +x Mercurial/hg && chmod +x Mercurial/chg 2>/dev/null || true +# Fix up mercurial.ini path to fixutf8 +RUN sed -i -e 's/fixutf8 = \/FwHeadless/fixutf8 = \/app/' Mercurial/mercurial.ini +USER www-data:www-data +ENV XDG_DATA_HOME=/var/www/.local/share +ENTRYPOINT ["tini", "--"] +CMD ["dotnet", "FwHeadless.dll"] diff --git a/backend/CrdtMerge/CrdtMerge.csproj b/backend/FwHeadless/FwHeadless.csproj similarity index 100% rename from backend/CrdtMerge/CrdtMerge.csproj rename to backend/FwHeadless/FwHeadless.csproj diff --git a/backend/CrdtMerge/CrdtMergeConfig.cs b/backend/FwHeadless/FwHeadlessConfig.cs similarity index 90% rename from backend/CrdtMerge/CrdtMergeConfig.cs rename to backend/FwHeadless/FwHeadlessConfig.cs index 8d21a36b1..ec664817e 100644 --- a/backend/CrdtMerge/CrdtMergeConfig.cs +++ b/backend/FwHeadless/FwHeadlessConfig.cs @@ -1,8 +1,8 @@ using System.ComponentModel.DataAnnotations; -namespace CrdtMerge; +namespace FwHeadless; -public class CrdtMergeConfig +public class FwHeadlessConfig { [Required, Url, RegularExpression(@"^.+/$", ErrorMessage = "Must end with '/'")] public required string LexboxUrl { get; init; } diff --git a/backend/CrdtMerge/CrdtMergeKernel.cs b/backend/FwHeadless/FwHeadlessKernel.cs similarity index 69% rename from backend/CrdtMerge/CrdtMergeKernel.cs rename to backend/FwHeadless/FwHeadlessKernel.cs index 0c312c074..9a0d42b95 100644 --- a/backend/CrdtMerge/CrdtMergeKernel.cs +++ b/backend/FwHeadless/FwHeadlessKernel.cs @@ -2,16 +2,16 @@ using FwLiteProjectSync; using LcmCrdt; -namespace CrdtMerge; +namespace FwHeadless; -public static class CrdtMergeKernel +public static class FwHeadlessKernel { - public static void AddCrdtMerge(this IServiceCollection services) + public static void AddFwHeadless(this IServiceCollection services) { services .AddLogging(builder => builder.AddConsole().AddDebug().AddFilter("Microsoft.EntityFrameworkCore", LogLevel.Warning)); - services.AddOptions() - .BindConfiguration("SendReceiveConfig") + services.AddOptions() + .BindConfiguration("FwHeadlessConfig") .ValidateDataAnnotations() .ValidateOnStart(); services.AddScoped(); diff --git a/backend/CrdtMerge/Program.cs b/backend/FwHeadless/Program.cs similarity index 95% rename from backend/CrdtMerge/Program.cs rename to backend/FwHeadless/Program.cs index e2604a9ec..261dedd9b 100644 --- a/backend/CrdtMerge/Program.cs +++ b/backend/FwHeadless/Program.cs @@ -1,4 +1,4 @@ -using CrdtMerge; +using FwHeadless; using FwDataMiniLcmBridge; using FwLiteProjectSync; using LcmCrdt; @@ -14,12 +14,14 @@ // Learn more about configuring OpenAPI at https://aka.ms/aspnet/openapi builder.Services.AddOpenApi(); +builder.Services.AddHealthChecks(); + builder.Services.AddLexData( autoApplyMigrations: false, useOpenIddict: false ); -builder.Services.AddCrdtMerge(); +builder.Services.AddFwHeadless(); var app = builder.Build(); @@ -33,6 +35,8 @@ app.UseHttpsRedirection(); +app.MapHealthChecks("/api/healthz"); + app.MapPost("/sync", ExecuteMergeRequest); app.Run(); @@ -41,7 +45,7 @@ ILogger logger, IServiceProvider services, SendReceiveService srService, - IOptions config, + IOptions config, FwDataFactory fwDataFactory, ProjectsService projectsService, ProjectLookupService projectLookupService, diff --git a/backend/CrdtMerge/ProjectLookupService.cs b/backend/FwHeadless/ProjectLookupService.cs similarity index 94% rename from backend/CrdtMerge/ProjectLookupService.cs rename to backend/FwHeadless/ProjectLookupService.cs index ac8244e99..9cb8cb971 100644 --- a/backend/CrdtMerge/ProjectLookupService.cs +++ b/backend/FwHeadless/ProjectLookupService.cs @@ -1,7 +1,7 @@ using LexData; using Microsoft.EntityFrameworkCore; -namespace CrdtMerge; +namespace FwHeadless; public class ProjectLookupService(LexBoxDbContext dbContext) { diff --git a/backend/CrdtMerge/Properties/launchSettings.json b/backend/FwHeadless/Properties/launchSettings.json similarity index 100% rename from backend/CrdtMerge/Properties/launchSettings.json rename to backend/FwHeadless/Properties/launchSettings.json diff --git a/backend/CrdtMerge/SendReceiveHelpers.cs b/backend/FwHeadless/SendReceiveHelpers.cs similarity index 96% rename from backend/CrdtMerge/SendReceiveHelpers.cs rename to backend/FwHeadless/SendReceiveHelpers.cs index 8eb1404c0..9c8ac9494 100644 --- a/backend/CrdtMerge/SendReceiveHelpers.cs +++ b/backend/FwHeadless/SendReceiveHelpers.cs @@ -1,7 +1,7 @@ using FwDataMiniLcmBridge; using SIL.Progress; -namespace CrdtMerge; +namespace FwHeadless; public static class SendReceiveHelpers { @@ -12,7 +12,7 @@ public record ProjectPath(string Code, string Dir) public record SendReceiveAuth(string Username, string Password) { - public SendReceiveAuth(CrdtMergeConfig config) : this(config.LexboxUsername, config.LexboxPassword) { } + public SendReceiveAuth(FwHeadlessConfig config) : this(config.LexboxUsername, config.LexboxPassword) { } }; public record LfMergeBridgeResult(string Output, string ProgressMessages); diff --git a/backend/CrdtMerge/SendReceiveService.cs b/backend/FwHeadless/SendReceiveService.cs similarity index 91% rename from backend/CrdtMerge/SendReceiveService.cs rename to backend/FwHeadless/SendReceiveService.cs index 14db577ae..37023dc8f 100644 --- a/backend/CrdtMerge/SendReceiveService.cs +++ b/backend/FwHeadless/SendReceiveService.cs @@ -1,9 +1,9 @@ using FwDataMiniLcmBridge; using Microsoft.Extensions.Options; -namespace CrdtMerge; +namespace FwHeadless; -public class SendReceiveService(IOptions config) +public class SendReceiveService(IOptions config) { public SendReceiveHelpers.LfMergeBridgeResult SendReceive(FwDataProject project, string? projectCode, string? commitMessage = null) { diff --git a/backend/CrdtMerge/appsettings.Development.json b/backend/FwHeadless/appsettings.Development.json similarity index 95% rename from backend/CrdtMerge/appsettings.Development.json rename to backend/FwHeadless/appsettings.Development.json index 0995c2046..22a96c2a5 100644 --- a/backend/CrdtMerge/appsettings.Development.json +++ b/backend/FwHeadless/appsettings.Development.json @@ -1,5 +1,5 @@ { - "SendReceiveConfig": { + "FwHeadlessConfig": { "ProjectStorageRoot": "../../hgweb/repos", "LexboxUrl": "http://localhost/", "LexboxUsername": "admin", diff --git a/backend/CrdtMerge/appsettings.json b/backend/FwHeadless/appsettings.json similarity index 87% rename from backend/CrdtMerge/appsettings.json rename to backend/FwHeadless/appsettings.json index d766d8006..d56001975 100644 --- a/backend/CrdtMerge/appsettings.json +++ b/backend/FwHeadless/appsettings.json @@ -1,5 +1,5 @@ { - "CrdtMergeConfig": { + "FwHeadlessConfig": { "LexboxUsername": null }, "Logging": { diff --git a/deployment/base/fw-headless-deployment.yaml b/deployment/base/fw-headless-deployment.yaml new file mode 100644 index 000000000..f61485e53 --- /dev/null +++ b/deployment/base/fw-headless-deployment.yaml @@ -0,0 +1,139 @@ +# https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service +apiVersion: v1 +kind: Service +metadata: + name: fw-headless + namespace: languagedepot + labels: + app: fw-headless +spec: + type: ClusterIP + clusterIP: None + selector: + app: fw-headless + ports: + - name: http + protocol: TCP + port: 80 + +--- + +# https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#writing-a-deployment-spec +apiVersion: apps/v1 +kind: Deployment +metadata: + name: fw-headless + namespace: languagedepot + labels: + app: fw-headless +spec: + selector: + matchLabels: + app: fw-headless + strategy: + rollingUpdate: + maxSurge: 2 + maxUnavailable: 0 + type: RollingUpdate + template: + # https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates + metadata: + labels: + app: fw-headless + spec: + securityContext: + runAsGroup: 33 + runAsUser: 33 + runAsNonRoot: true + containers: + - name: fw-headless + image: ghcr.io/sillsdev/lexbox-fw-headless:develop + imagePullPolicy: IfNotPresent + # https://kubernetes.io/docs/concepts/configuration/manage-resources-containers + resources: + requests: + memory: 1500Mi + limits: + memory: 2400Mi + startupProbe: + httpGet: + port: 80 + path: /api/healthz + failureThreshold: 30 + periodSeconds: 10 + ports: + - containerPort: 80 + + volumeMounts: + - name: fw-headless + mountPath: /var/lib/fw-headless + + env: + - name: DOTNET_URLS + value: http://0.0.0.0:80 + - name: ASPNETCORE_ENVIRONMENT + valueFrom: + configMapKeyRef: + name: app-config + key: environment-name + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: K8S_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POSTGRES_DB + valueFrom: + secretKeyRef: + key: POSTGRES_DB + name: db + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + key: POSTGRES_PASSWORD + name: db + - name: DbConfig__LexBoxConnectionString + value: Host=db;Port=5432;Username=postgres;Password=$(POSTGRES_PASSWORD);Database=$(POSTGRES_DB) + - name: FwHeadlessConfig__ProjectStorageRoot + value: /var/lib/fw-headless/projects + - name: FwHeadlessConfig__LexboxUrl + value: http://lexbox:5158/ + # - name: FwHeadlessConfig__HgUrl + # value: http://lexbox:5158/hg/ + - name: FwHeadlessConfig__LexboxUsername + valueFrom: + secretKeyRef: + key: CRDT_MERGE_SEND_RECEIVE_USERNAME + name: fw-headless + - name: FwHeadlessConfig__LexboxPassword + valueFrom: + secretKeyRef: + key: CRDT_MERGE_SEND_RECEIVE_PASSWORD + name: fw-headless + - name: FwHeadlessConfig__FdoDataModelVersion + value: "7000072" + + initContainers: + - name: populate-crdt-project-storage + securityContext: + # Make sure we're authorized to set ownership + runAsUser: 0 + runAsGroup: 0 + runAsNonRoot: false + image: busybox:1.36.1 + command: + - 'sh' + - '-c' + - | + mkdir -p /fw-headless/projects + chown www-data:www-data /fw-headless/projects + volumeMounts: + - name: fw-headless + mountPath: /fw-headless + + volumes: + - name: fw-headless + persistentVolumeClaim: + claimName: fw-headless # established in pvc.yaml diff --git a/deployment/base/kustomization.yaml b/deployment/base/kustomization.yaml index fe9002a1a..0dc486889 100644 --- a/deployment/base/kustomization.yaml +++ b/deployment/base/kustomization.yaml @@ -9,6 +9,7 @@ resources: - pvc.yaml - db-deployment.yaml - hg-deployment.yaml +- fw-headless-deployment.yaml - lexbox-deployment.yaml - ui-deployment.yaml - proxy-deployment.yaml diff --git a/deployment/base/pvc.yaml b/deployment/base/pvc.yaml index 8ffbdb558..6ea3f7ce7 100644 --- a/deployment/base/pvc.yaml +++ b/deployment/base/pvc.yaml @@ -33,3 +33,21 @@ spec: requests: storage: 10Gi storageClassName: weekly-snapshots-retain-4 # provided by LTOps + +--- + +# https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: fw-headless + namespace: languagedepot + labels: + app.kubernetes.io/part-of: languagedepot +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: weekly-snapshots-retain-4 # provided by LTOps diff --git a/deployment/base/secrets.yaml b/deployment/base/secrets.yaml index 94236d32a..7046f12a3 100644 --- a/deployment/base/secrets.yaml +++ b/deployment/base/secrets.yaml @@ -84,3 +84,13 @@ metadata: stringData: KOPIA_PASSWORD: '' kopia.config: '' +--- + +apiVersion: v1 +kind: Secret +metadata: + name: fw-headless + namespace: languagedepot +stringData: + CRDT_MERGE_SEND_RECEIVE_USERNAME: '' + CRDT_MERGE_SEND_RECEIVE_PASSWORD: '' diff --git a/deployment/develop/fw-headless-deployment.patch.yaml b/deployment/develop/fw-headless-deployment.patch.yaml new file mode 100644 index 000000000..9df814838 --- /dev/null +++ b/deployment/develop/fw-headless-deployment.patch.yaml @@ -0,0 +1,24 @@ +#file: noinspection KubernetesMissingKeys,KubernetesUnknownValues +apiVersion: apps/v1 +kind: Deployment +metadata: + name: fw-headless + namespace: languagedepot + labels: + app: fw-headless +spec: + template: + spec: + containers: + - name: fw-headless + imagePullPolicy: IfNotPresent + env: + - name: FwHeadlessConfig__LexboxUsername + value: "admin" + valueFrom: # don't use secret as defined in base + - name: FwHeadlessConfig__LexboxPassword + valueFrom: + secretKeyRef: + key: SEED_USER_PASSWORD + name: db + optional: true diff --git a/deployment/develop/kustomization.yaml b/deployment/develop/kustomization.yaml index 7ba6e3340..c86d42508 100644 --- a/deployment/develop/kustomization.yaml +++ b/deployment/develop/kustomization.yaml @@ -7,6 +7,7 @@ resources: components: - ../init-repos patches: + - path: fw-headless-deployment.patch.yaml - path: lexbox-deployment.patch.yaml - path: app-config.yaml - path: ingress-config.yaml diff --git a/deployment/local-dev/fw-headless-deployment.patch.yaml b/deployment/local-dev/fw-headless-deployment.patch.yaml new file mode 100644 index 000000000..9df814838 --- /dev/null +++ b/deployment/local-dev/fw-headless-deployment.patch.yaml @@ -0,0 +1,24 @@ +#file: noinspection KubernetesMissingKeys,KubernetesUnknownValues +apiVersion: apps/v1 +kind: Deployment +metadata: + name: fw-headless + namespace: languagedepot + labels: + app: fw-headless +spec: + template: + spec: + containers: + - name: fw-headless + imagePullPolicy: IfNotPresent + env: + - name: FwHeadlessConfig__LexboxUsername + value: "admin" + valueFrom: # don't use secret as defined in base + - name: FwHeadlessConfig__LexboxPassword + valueFrom: + secretKeyRef: + key: SEED_USER_PASSWORD + name: db + optional: true diff --git a/deployment/local-dev/kustomization.yaml b/deployment/local-dev/kustomization.yaml index 02e526433..1fb7ad2fa 100644 --- a/deployment/local-dev/kustomization.yaml +++ b/deployment/local-dev/kustomization.yaml @@ -39,6 +39,7 @@ patches: path: delete-oauth-certs.yaml - path: app-config.yaml + - path: fw-headless-deployment.patch.yaml - path: lexbox-deployment.patch.yaml - path: ui-deployment.patch.yaml - path: hg-repos-pvc.patch.yaml diff --git a/deployment/production/fw-headless-volume.yaml b/deployment/production/fw-headless-volume.yaml new file mode 100644 index 000000000..6ff2a11c1 --- /dev/null +++ b/deployment/production/fw-headless-volume.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: hg-repos + namespace: languagedepot +spec: + resources: + requests: + storage: 150Gi diff --git a/deployment/production/kustomization.yaml b/deployment/production/kustomization.yaml index f35c39514..e79de016c 100644 --- a/deployment/production/kustomization.yaml +++ b/deployment/production/kustomization.yaml @@ -9,6 +9,7 @@ resources: patches: - path: lexbox-deployment.patch.yaml - path: app-config.yaml + - path: fw-headless-volume.yaml - path: hg-repos-volume.yaml - path: hg-deployment.yaml - path: ingress-config-prod.yaml diff --git a/deployment/staging/fw-headless-deployment.patch.yaml b/deployment/staging/fw-headless-deployment.patch.yaml new file mode 100644 index 000000000..9df814838 --- /dev/null +++ b/deployment/staging/fw-headless-deployment.patch.yaml @@ -0,0 +1,24 @@ +#file: noinspection KubernetesMissingKeys,KubernetesUnknownValues +apiVersion: apps/v1 +kind: Deployment +metadata: + name: fw-headless + namespace: languagedepot + labels: + app: fw-headless +spec: + template: + spec: + containers: + - name: fw-headless + imagePullPolicy: IfNotPresent + env: + - name: FwHeadlessConfig__LexboxUsername + value: "admin" + valueFrom: # don't use secret as defined in base + - name: FwHeadlessConfig__LexboxPassword + valueFrom: + secretKeyRef: + key: SEED_USER_PASSWORD + name: db + optional: true diff --git a/deployment/staging/fw-headless-volume.yaml b/deployment/staging/fw-headless-volume.yaml new file mode 100644 index 000000000..97bb99558 --- /dev/null +++ b/deployment/staging/fw-headless-volume.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: fw-headless + namespace: languagedepot +spec: + resources: + requests: + storage: 30Gi diff --git a/deployment/staging/kustomization.yaml b/deployment/staging/kustomization.yaml index 66768dc21..c56c8d483 100644 --- a/deployment/staging/kustomization.yaml +++ b/deployment/staging/kustomization.yaml @@ -7,6 +7,8 @@ resources: components: - ../init-repos patches: + - path: fw-headless-volume.yaml + - path: fw-headless-deployment.patch.yaml - path: lexbox-deployment.patch.yaml - path: app-config.yaml - path: hg-repos-volume.yaml diff --git a/skaffold.yaml b/skaffold.yaml index bb40c87d7..5f6e82600 100644 --- a/skaffold.yaml +++ b/skaffold.yaml @@ -14,6 +14,10 @@ build: manual: - src: '**' dest: /src/backend + - image: ghcr.io/sillsdev/lexbox-fw-headless + context: backend + docker: + dockerfile: FwHeadless/Dockerfile - image: ghcr.io/sillsdev/lexbox-ui context: frontend docker: @@ -41,6 +45,12 @@ portForward: resourceName: lexbox namespace: languagedepot port: 5158 + # FwHeadless + - resourceType: Service + resourceName: fw-headless + namespace: languagedepot + port: 80 + localPort: 5275 # OpenTelemetry - resourceType: Service resourceName: lexbox @@ -89,6 +99,12 @@ profiles: namespace: languagedepot port: 5432 localPort: 5433 + # FwHeadless + - resourceType: Service + resourceName: fw-headless + namespace: languagedepot + port: 80 + localPort: 5275 # OpenTelemetry - resourceType: Service resourceName: lexbox @@ -141,6 +157,12 @@ profiles: resourceName: lexbox namespace: languagedepot port: 5158 + # FwHeadless + - resourceType: Service + resourceName: fw-headless + namespace: languagedepot + port: 80 + localPort: 5275 # OpenTelemetry - resourceType: Service resourceName: lexbox