Add FwHeadless (renamed from CrdtMerge) deployment for k8s and GHA

.github/workflows/deploy-branch.yaml

@@ -41,10 +41,18 @@ jobs:
       version: ${{ needs.set-version.outputs.version }}
       label-latest: false
 
+  build-fw-headless:
+    name: Build FwHeadless
+    needs: [ set-version ]
+    uses: ./.github/workflows/lexbox-fw-headless.yaml
+    with:
+      version: ${{ needs.set-version.outputs.version }}
+      label-latest: false
+
   deploy:
     name: Deploy Develop
     uses: ./.github/workflows/deploy.yaml
-    needs: [ build-api, build-ui, build-hgweb, set-version ]
+    needs: [ build-api, build-ui, build-hgweb, build-fw-headless, set-version ]
     secrets: inherit
     with:
       version: ${{ needs.set-version.outputs.version }}

.github/workflows/deploy.yaml

@@ -50,6 +50,7 @@ jobs:
       url: https://${{ inputs.deploy-domain }}
     outputs:
       api-version: ${{ steps.get-api-version.outputs.result }}
+      fw-headless-version: ${{ steps.get-fw-headless-version.outputs.result }}
       ui-version: ${{ steps.get-ui-version.outputs.result }}
     steps:
       - name: Checkout lexbox repo
@@ -80,6 +81,11 @@ jobs:
         id: get-api-version
         with:
           cmd: yq '.images.[] | select(.name == "ghcr.io/sillsdev/lexbox-api").newTag' "fleet/${{ inputs.k8s-environment }}/kustomization.yaml"
+      - name: Get FwHeadless version
+        uses: mikefarah/yq@0b34c9a00de1c575a34eea05af1d956a525c4fc1 # v4.34.2
+        id: get-fw-headless-version
+        with:
+          cmd: yq '.images.[] | select(.name == "ghcr.io/sillsdev/lexbox-fw-headless").newTag' "fleet/${{ inputs.k8s-environment }}/kustomization.yaml"
       - name: Get UI version
         uses: mikefarah/yq@0b34c9a00de1c575a34eea05af1d956a525c4fc1 # v4.34.2
         id: get-ui-version

.github/workflows/develop-fw-headless.yaml

@@ -0,0 +1,72 @@
+name: Develop FwHeadless CI/CD
+on:
+  workflow_dispatch:
+  push:
+    paths:
+      - 'backend/FwHeadless/**'
+      - 'backend/FixFwData/**'
+      - 'backend/FwLite/FwDataMiniLcmBridge/**'
+      - 'backend/FwLite/LcmCrdt/**'
+      - 'backend/FwLite/MiniLcm/**'
+      - 'backend/FwLiteProjectSync/FwLiteProjectSync/**'
+      - 'backend/LexCore/**'
+      - 'backend/LexData/**'
+      - '.github/workflows/lexbox-fw-headless.yaml'
+      - '.github/workflows/deploy.yaml'
+      - 'deployment/base/fw-headless-deployment.yaml'
+    branches:
+      - develop
+  pull_request:
+    paths:
+      - 'backend/FwHeadless/**'
+      - 'backend/FixFwData/**'
+      - 'backend/FwLite/FwDataMiniLcmBridge/**'
+      - 'backend/FwLite/LcmCrdt/**'
+      - 'backend/FwLite/MiniLcm/**'
+      - 'backend/FwLiteProjectSync/FwLiteProjectSync/**'
+      - 'backend/LexCore/**'
+      - 'backend/LexData/**'
+      - '.github/workflows/lexbox-fw-headless.yaml'
+      - '.github/workflows/deploy.yaml'
+      - 'deployment/base/fw-headless-deployment.yaml'
+    branches:
+      - develop
+
+jobs:
+  set-version:
+    name: Set Version
+    runs-on: ubuntu-latest
+    outputs:
+      version: ${{ steps.setVersion.outputs.VERSION }}
+    steps:
+      - name: Set Version
+        id: setVersion
+        # set version to date in vYYYY-MM-DD-commitSha format
+        run: |
+          shortSha=$(echo ${{ github.sha }} | cut -c1-8)
+          echo "VERSION=v$(date --rfc-3339=date)-$shortSha" >> ${GITHUB_OUTPUT}
+  build-fw-headless:
+    name: Build FwHeadless
+    needs: set-version
+    uses: ./.github/workflows/lexbox-fw-headless.yaml
+    with:
+      version: ${{ needs.set-version.outputs.version }}
+  deploy-fw-headless:
+    name: Deploy FwHeadless
+    if: ${{github.ref == 'refs/heads/develop'}}
+    needs: [ build-fw-headless, set-version ]
+    uses: ./.github/workflows/deploy.yaml
+    secrets: inherit
+    with:
+      version: ${{ needs.set-version.outputs.version }}
+      image: 'ghcr.io/sillsdev/lexbox-fw-headless'
+      k8s-environment: develop
+      deploy-domain: lexbox.dev.languagetechnology.org
+
+  # TODO: Run FwHeadless tests once we have developed them, but we don't need to run the whole integration test suite if only FwHeadless changes are being pushed
+  # integration-test-gha:
+  #   name: GHA integration tests
+  #   needs: [build-fw-headless, set-version]
+  #   uses: ./.github/workflows/integration-test-gha.yaml
+  #   with:
+  #     lexbox-fw-headless-tag: ${{ needs.set-version.outputs.version }}

.github/workflows/integration-test-gha.yaml

@@ -39,7 +39,16 @@ jobs:
         uses: mikefarah/yq@0b34c9a00de1c575a34eea05af1d956a525c4fc1 # v4.34.2
         with:
           cmd: yq eval -i '(.images.[] | select(.name == "ghcr.io/sillsdev/lexbox-api").newTag) = "${{ inputs.lexbox-api-tag }}"' "./deployment/gha/kustomization.yaml"
-      # It's also possible that hgweb and/or ui image may have changed; if so, pull them and update kustomization.yaml for them as well
+      # It's also possible that hgweb, fw-headless, and/or ui image may have changed; if so, pull them and update kustomization.yaml for them as well
+      - name: Pull fw-headless if updated
+        id: fw-headless_image
+        continue-on-error: true
+        run: docker pull ghcr.io/sillsdev/lexbox-fw-headless:${{ inputs.lexbox-api-tag }}
+      - name: Update image fw-headless version
+        if: ${{ steps.fw-headless_image.outcome == 'success' }}
+        uses: mikefarah/yq@0b34c9a00de1c575a34eea05af1d956a525c4fc1 # v4.34.2
+        with:
+          cmd: yq eval -i '(.images.[] | select(.name == "ghcr.io/sillsdev/lexbox-fw-headless").newTag) = "${{ inputs.lexbox-api-tag }}"' "./deployment/gha/kustomization.yaml"
       - name: Pull hgweb if updated
         id: hgweb_image
         continue-on-error: true

.github/workflows/lexbox-fw-headless.yaml

@@ -0,0 +1,99 @@
+name: Build FwHeadless
+
+# https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#on
+on:
+  workflow_call:
+    inputs:
+      version:
+        description: 'The version of the image to build'
+        required: true
+        type: string
+      label-latest:
+        description: 'The label to apply to the latest image'
+        type: boolean
+        default: false
+
+env:
+  IMAGE_NAME: ghcr.io/sillsdev/lexbox-fw-headless
+
+
+jobs:
+  publish-fw-headless:
+    timeout-minutes: 60
+    runs-on: ubuntu-latest
+
+    # postgres db is for automated tests
+    # services:
+    #   postgres:
+    #     image: postgres:15-alpine
+    #     env:
+    #       POSTGRES_PASSWORD: 972b722e63f549938d07bd8c4ee5086c
+    #       POSTGRES_DB: lexbox-tests
+    #     # Set health checks to wait until postgres has started
+    #     options: >-
+    #       --health-cmd pg_isready
+    #       --health-interval 10s
+    #       --health-timeout 5s
+    #       --health-retries 5
+    #     ports:
+    #       # Maps tcp port 5432 on service container to the host
+    #       - 5433:5432
+
+    env:
+      # https://docs.docker.com/develop/develop-images/build_enhancements/
+      DOCKER_BUILDKIT: 1
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+      - uses: actions/setup-dotnet@v4
+        with:
+          dotnet-version: '9.x'
+      - name: Dotnet build
+        run: dotnet build backend/FwHeadless/FwHeadless.csproj
+      # TODO: Write FwHeadless unit tests, probably based on existing sync tests
+      # - name: Unit tests
+      #   run: dotnet test backend/FwHeadless/FwHeadless.csproj --logger:"xunit;LogFileName={assembly}.results.xml" --results-directory ./test-results --filter "Category!=Integration&Category!=FlakyIntegration" --blame-hang-timeout 10m
+      # - name: Publish unit test results
+      #   uses: EnricoMi/publish-unit-test-result-action@8885e273a4343cd7b48eaa72428dea0c3067ea98 # v2.14.0
+      #   if: always()
+      #   with:
+      #     check_name: C# Unit Tests
+      #     files: ./test-results/*.xml
+      # - name: Upload test results
+      #   if: always()
+      #   uses: actions/upload-artifact@v4
+      #   with:
+      #     name: dotnet-unit-test-results
+      #     path: ./test-results
+
+      - name: Docker meta
+        id: meta
+        if: ${{ !env.ACT }}
+        uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1
+        with:
+          images: ${{ env.IMAGE_NAME }}
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=raw,enable=${{ inputs.label-latest }},value=latest
+            type=raw,value=${{ inputs.version }}
+
+      - name: ghcr.io login
+        uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3.1.0
+        if: ${{ !env.ACT }}
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0
+        with:
+          context: backend
+          file: backend/FwHeadless/Dockerfile
+          build-args: |
+            APP_VERSION=${{ inputs.version }}
+          push: ${{ !env.ACT && github.repository == 'sillsdev/languageforge-lexbox' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.github/workflows/release-pipeline.yaml

@@ -43,10 +43,18 @@ jobs:
       version: ${{ needs.set-version.outputs.version }}
       label-latest: true
 
+  build-fw-headless:
+    name: Build fw-headless
+    needs: [ set-version ]
+    uses: ./.github/workflows/lexbox-fw-headless.yaml
+    with:
+      version: ${{ needs.set-version.outputs.version }}
+      label-latest: true
+
   deploy:
     name: Deploy Staging
     uses: ./.github/workflows/deploy.yaml
-    needs: [ build-api, build-ui, build-hgweb, set-version ]
+    needs: [ build-api, build-ui, build-hgweb, build-fw-headless, set-version ]
     secrets: inherit
     with:
       version: ${{ needs.set-version.outputs.version }}

LexBox.sln

@@ -51,7 +51,7 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LcmDebugger", "backend\LfNe
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MiniLcm.Tests", "backend\FwLite\MiniLcm.Tests\MiniLcm.Tests.csproj", "{00AE5440-0E36-4488-935B-5B11301BA57D}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CrdtMerge", "backend\CrdtMerge\CrdtMerge.csproj", "{ECBA46AB-AF87-4D4D-9716-FD77264B817F}"
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FwHeadless", "backend\FwHeadless\FwHeadless.csproj", "{ECBA46AB-AF87-4D4D-9716-FD77264B817F}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution

backend/FwHeadless/Dockerfile

@@ -0,0 +1,34 @@
+# syntax=docker/dockerfile:1
+FROM mcr.microsoft.com/dotnet/aspnet:9.0 AS base
+WORKDIR /app
+EXPOSE 80
+EXPOSE 443
+
+FROM mcr.microsoft.com/dotnet/sdk:9.0 AS build
+
+COPY . .
+RUN --mount=type=cache,target=/root/.nuget/packages dotnet restore "FwHeadless/FwHeadless.csproj"
+
+ARG APP_VERSION
+LABEL version=$APP_VERSION
+
+RUN --mount=type=cache,target=/root/.nuget/packages dotnet build /p:InformationalVersion=$APP_VERSION "FwHeadless/FwHeadless.csproj" -c Release -o /app/build
+
+FROM build AS publish
+RUN --mount=type=cache,target=/root/.nuget/packages dotnet publish /p:InformationalVersion=$APP_VERSION "FwHeadless/FwHeadless.csproj" -c Release -o /app/publish
+
+FROM base AS final
+RUN mkdir -p /var/lib/fw-headless /var/www/.local/share && chown -R www-data:www-data /var/lib/fw-headless /var/www/.local/share
+RUN apt-get update \
+	&& apt-get install --yes --no-install-recommends tini iputils-ping python3 \
+	&& rm -rf /var/lib/apt/lists/*
+WORKDIR /app
+COPY --from=publish /app/publish .
+# Ensure Mercurial exec bit was not stripped by dotnet CLI tools
+RUN chmod +x Mercurial/hg && chmod +x Mercurial/chg 2>/dev/null || true
+# Fix up mercurial.ini path to fixutf8
+RUN sed -i -e 's/fixutf8 = \/FwHeadless/fixutf8 = \/app/' Mercurial/mercurial.ini
+USER www-data:www-data
+ENV XDG_DATA_HOME=/var/www/.local/share
+ENTRYPOINT ["tini", "--"]
+CMD ["dotnet", "FwHeadless.dll"]

backend/CrdtMerge/CrdtMergeConfig.cs → backend/FwHeadless/FwHeadlessConfig.cs

@@ -1,8 +1,8 @@
 using System.ComponentModel.DataAnnotations;
 
-namespace CrdtMerge;
+namespace FwHeadless;
 
-public class CrdtMergeConfig
+public class FwHeadlessConfig
 {
     [Required, Url, RegularExpression(@"^.+/$", ErrorMessage = "Must end with '/'")]
     public required string LexboxUrl { get; init; }

backend/CrdtMerge/CrdtMergeKernel.cs → backend/FwHeadless/FwHeadlessKernel.cs

@@ -2,16 +2,16 @@
 using FwLiteProjectSync;
 using LcmCrdt;
 
-namespace CrdtMerge;
+namespace FwHeadless;
 
-public static class CrdtMergeKernel
+public static class FwHeadlessKernel
 {
-    public static void AddCrdtMerge(this IServiceCollection services)
+    public static void AddFwHeadless(this IServiceCollection services)
     {
         services
             .AddLogging(builder => builder.AddConsole().AddDebug().AddFilter("Microsoft.EntityFrameworkCore", LogLevel.Warning));
-        services.AddOptions<CrdtMergeConfig>()
-            .BindConfiguration("SendReceiveConfig")
+        services.AddOptions<FwHeadlessConfig>()
+            .BindConfiguration("FwHeadlessConfig")
             .ValidateDataAnnotations()
             .ValidateOnStart();
         services.AddScoped<SendReceiveService>();

backend/CrdtMerge/Program.cs → backend/FwHeadless/Program.cs

@@ -1,4 +1,4 @@
-using CrdtMerge;
+using FwHeadless;
 using FwDataMiniLcmBridge;
 using FwLiteProjectSync;
 using LcmCrdt;
@@ -14,12 +14,14 @@
 // Learn more about configuring OpenAPI at https://aka.ms/aspnet/openapi
 builder.Services.AddOpenApi();
 
+builder.Services.AddHealthChecks();
+
 builder.Services.AddLexData(
     autoApplyMigrations: false,
     useOpenIddict: false
 );
 
-builder.Services.AddCrdtMerge();
+builder.Services.AddFwHeadless();
 
 var app = builder.Build();
 
@@ -33,6 +35,8 @@
 
 app.UseHttpsRedirection();
 
+app.MapHealthChecks("/api/healthz");
+
 app.MapPost("/sync", ExecuteMergeRequest);
 
 app.Run();
@@ -41,7 +45,7 @@
     ILogger<Program> logger,
     IServiceProvider services,
     SendReceiveService srService,
-    IOptions<CrdtMergeConfig> config,
+    IOptions<FwHeadlessConfig> config,
     FwDataFactory fwDataFactory,
     ProjectsService projectsService,
     ProjectLookupService projectLookupService,

backend/CrdtMerge/ProjectLookupService.cs → backend/FwHeadless/ProjectLookupService.cs

@@ -1,7 +1,7 @@
 using LexData;
 using Microsoft.EntityFrameworkCore;
 
-namespace CrdtMerge;
+namespace FwHeadless;
 
 public class ProjectLookupService(LexBoxDbContext dbContext)
 {

backend/CrdtMerge/SendReceiveHelpers.cs → backend/FwHeadless/SendReceiveHelpers.cs

@@ -1,7 +1,7 @@
 using FwDataMiniLcmBridge;
 using SIL.Progress;
 
-namespace CrdtMerge;
+namespace FwHeadless;
 
 public static class SendReceiveHelpers
 {
@@ -12,7 +12,7 @@ public record ProjectPath(string Code, string Dir)
 
     public record SendReceiveAuth(string Username, string Password)
     {
-        public SendReceiveAuth(CrdtMergeConfig config) : this(config.LexboxUsername, config.LexboxPassword) { }
+        public SendReceiveAuth(FwHeadlessConfig config) : this(config.LexboxUsername, config.LexboxPassword) { }
     };
 
     public record LfMergeBridgeResult(string Output, string ProgressMessages);