Sync CRDTs with lexbox during crdt merge

backend/FwHeadless/CrdtSyncService.cs

@@ -0,0 +1,27 @@
+using LcmCrdt;
+using LcmCrdt.RemoteSync;
+using SIL.Harmony;
+
+namespace FwHeadless;
+
+public class CrdtSyncService(
+    CrdtHttpSyncService httpSyncService,
+    IHttpClientFactory httpClientFactory,
+    CurrentProjectService currentProjectService,
+    DataModel dataModel,
+    ILogger<CrdtSyncService> logger)
+{
+    public async Task Sync()
+    {
+        var lexboxRemoteServer = await httpSyncService.CreateProjectSyncable(
+            currentProjectService.ProjectData,
+            httpClientFactory.CreateClient(FwHeadlessKernel.LexboxHttpClientName)
+        );
+        var syncResults = await dataModel.SyncWith(lexboxRemoteServer);
+        if (!syncResults.IsSynced) throw new InvalidOperationException("Sync failed");
+        logger.LogInformation(
+            "Synced with Lexbox, Downloaded changes: {MissingFromLocal}, Uploaded changes: {MissingFromRemote}",
+            syncResults.MissingFromLocal.Length,
+            syncResults.MissingFromRemote.Length);
+    }
+}

backend/FwHeadless/FwHeadlessKernel.cs

@@ -1,11 +1,13 @@
 using FwDataMiniLcmBridge;
 using FwLiteProjectSync;
 using LcmCrdt;
+using Microsoft.Extensions.Options;
 
 namespace FwHeadless;
 
 public static class FwHeadlessKernel
 {
+    public const string LexboxHttpClientName = "LexboxHttpClient";
     public static void AddFwHeadless(this IServiceCollection services)
     {
         services
@@ -20,5 +22,12 @@ public static void AddFwHeadless(this IServiceCollection services)
             .AddLcmCrdtClient()
             .AddFwDataBridge()
             .AddFwLiteProjectSync();
+        services.AddScoped<CrdtSyncService>();
+        services.AddTransient<HttpClientAuthHandler>();
+        services.AddHttpClient(LexboxHttpClientName,
+            (provider, client) =>
+            {
+                client.BaseAddress = new Uri(provider.GetRequiredService<IOptions<FwHeadlessConfig>>().Value.LexboxUrl);
+            }).AddHttpMessageHandler<HttpClientAuthHandler>();
     }
-};
+}

backend/FwHeadless/HttpClientAuthHandler.cs

@@ -0,0 +1,73 @@
+using System.Net;
+using LexCore;
+using LexCore.Auth;
+using Microsoft.Extensions.Caching.Memory;
+using Microsoft.Extensions.Options;
+
+namespace FwHeadless;
+
+public class HttpClientAuthHandler(IOptions<FwHeadlessConfig> config, IMemoryCache cache, ILogger<HttpClientAuthHandler> logger) : DelegatingHandler
+{
+    protected override HttpResponseMessage Send(HttpRequestMessage request, CancellationToken cancellationToken)
+    {
+        throw new NotSupportedException("use async apis");
+    }
+
+    protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
+    {
+        var lexboxUrl = new Uri(config.Value.LexboxUrl);
+        if (request.RequestUri?.Authority != lexboxUrl.Authority)
+        {
+            return await base.SendAsync(request, cancellationToken);
+        }
+        try
+        {
+            await SetAuthHeader(request, cancellationToken, lexboxUrl);
+        }
+        catch (Exception e)
+        {
+            throw new InvalidOperationException("Unable to set auth header", e);
+        }
+        return await base.SendAsync(request, cancellationToken);
+    }
+
+    private async Task SetAuthHeader(HttpRequestMessage request, CancellationToken cancellationToken, Uri lexboxUrl)
+    {
+        var cookieContainer = new CookieContainer();
+        cookieContainer.Add(new Cookie(LexAuthConstants.AuthCookieName, await GetToken(cancellationToken), null, lexboxUrl.Authority));
+        request.Headers.Add("Cookie", cookieContainer.GetCookieHeader(lexboxUrl));
+    }
+
+    private async ValueTask<string> GetToken(CancellationToken cancellationToken)
+    {
+        try
+        {
+            return await cache.GetOrCreateAsync("LexboxAuthToken",
+                async entry =>
+                {
+                    if (InnerHandler is null) throw new InvalidOperationException("InnerHandler is null");
+                    logger.LogInformation("Getting auth token");
+                    var client = new HttpClient(InnerHandler);
+                    client.BaseAddress = new Uri(config.Value.LexboxUrl);
+                    var response = await client.PostAsJsonAsync("/api/login",
+                        new LoginRequest(config.Value.LexboxPassword, config.Value.LexboxUsername),
+                        cancellationToken);
+                    response.EnsureSuccessStatusCode();
+                    var cookies = response.Headers.GetValues("Set-Cookie");
+                    var cookieContainer = new CookieContainer();
+                    cookieContainer.SetCookies(response.RequestMessage!.RequestUri!, cookies.Single());
+                    var authCookie = cookieContainer.GetAllCookies()
+                        .FirstOrDefault(c => c.Name == LexAuthConstants.AuthCookieName);
+                    if (authCookie is null) throw new InvalidOperationException("Auth cookie not found");
+                    entry.SetValue(authCookie.Value);
+                    entry.AbsoluteExpiration = authCookie.Expires;
+                    logger.LogInformation("Got auth token: {AuthToken}", authCookie.Value);
+                    return authCookie.Value;
+                }) ?? throw new NullReferenceException("unable to get the login token");
+        }
+        catch (Exception e)
+        {
+            throw new InvalidOperationException("Unable to get auth token", e);
+        }
+    }
+}

backend/FwHeadless/Program.cs

@@ -1,7 +1,9 @@
 using FwHeadless;
 using FwDataMiniLcmBridge;
+using FwDataMiniLcmBridge.Api;
 using FwLiteProjectSync;
 using LcmCrdt;
+using LcmCrdt.RemoteSync;
 using LexData;
 using Microsoft.AspNetCore.Http.HttpResults;
 using Microsoft.Extensions.Options;
@@ -41,7 +43,7 @@
 
 app.Run();
 
-static async Task<Results<Ok<CrdtFwdataProjectSyncService.SyncResult>, NotFound>> ExecuteMergeRequest(
+static async Task<Results<Ok<CrdtFwdataProjectSyncService.SyncResult>, NotFound, ProblemHttpResult>> ExecuteMergeRequest(
     ILogger<Program> logger,
     IServiceProvider services,
     SendReceiveService srService,
@@ -50,6 +52,8 @@
     ProjectsService projectsService,
     ProjectLookupService projectLookupService,
     CrdtFwdataProjectSyncService syncService,
+    CrdtHttpSyncService crdtHttpSyncService,
+    IHttpClientFactory httpClientFactory,
     Guid projectId,
     bool dryRun = false)
 {
@@ -67,6 +71,11 @@
         return TypedResults.NotFound();
     }
     logger.LogInformation("Project code is {projectCode}", projectCode);
+    //if we can't sync with lexbox fail fast
+    if (!await crdtHttpSyncService.TestAuth(httpClientFactory.CreateClient(FwHeadlessKernel.LexboxHttpClientName)))
+    {
+        return TypedResults.Problem("Unable to authenticate with Lexbox");
+    }
 
     var projectFolder = Path.Join(config.Value.ProjectStorageRoot, $"{projectCode}-{projectId}");
     if (!Directory.Exists(projectFolder)) Directory.CreateDirectory(projectFolder);
@@ -77,6 +86,30 @@
     logger.LogDebug("crdtFile: {crdtFile}", crdtFile);
     logger.LogDebug("fwDataFile: {fwDataFile}", fwDataProject.FilePath);
 
+    var fwdataApi = SetupFwData(fwDataProject, srService, projectCode, logger, fwDataFactory);
+    var crdtProject = await SetupCrdtProject(crdtFile, projectLookupService, projectId, projectsService, projectFolder, fwdataApi.ProjectId, config.Value.LexboxUrl);
+
+    var miniLcmApi = await services.OpenCrdtProject(crdtProject);
+    var crdtSyncService = services.GetRequiredService<CrdtSyncService>();
+    await crdtSyncService.Sync();
+
+
+    var result = await syncService.Sync(miniLcmApi, fwdataApi, dryRun);
+    logger.LogInformation("Sync result, CrdtChanges: {CrdtChanges}, FwdataChanges: {FwdataChanges}", result.CrdtChanges, result.FwdataChanges);
+    fwDataFactory.CloseProject(fwDataProject);
+
+    await crdtSyncService.Sync();
+    var srResult2 = srService.SendReceive(fwDataProject, projectCode);
+    logger.LogInformation("Send/Receive result after CRDT sync: {srResult2}", srResult2.Output);
+    return TypedResults.Ok(result);
+}
+
+static FwDataMiniLcmApi SetupFwData(FwDataProject fwDataProject,
+    SendReceiveService srService,
+    string projectCode,
+    ILogger<Program> logger,
+    FwDataFactory fwDataFactory)
+{
     if (File.Exists(fwDataProject.FilePath))
     {
         var srResult = srService.SendReceive(fwDataProject, projectCode);
@@ -87,15 +120,37 @@
         var srResult = srService.Clone(fwDataProject, projectCode);
         logger.LogInformation("Send/Receive result: {srResult}", srResult.Output);
     }
+
     var fwdataApi = fwDataFactory.GetFwDataMiniLcmApi(fwDataProject, true);
-    var crdtProject = File.Exists(crdtFile) ?
-        new CrdtProject("crdt", crdtFile) :
-        await projectsService.CreateProject(new("crdt", SeedNewProjectData: false, Path: projectFolder, FwProjectId: fwdataApi.ProjectId));
-    var miniLcmApi = await services.OpenCrdtProject(crdtProject);
-    var result = await syncService.Sync(miniLcmApi, fwdataApi, dryRun);
-    logger.LogInformation("Sync result, CrdtChanges: {CrdtChanges}, FwdataChanges: {FwdataChanges}", result.CrdtChanges, result.FwdataChanges);
-    var srResult2 = srService.SendReceive(fwDataProject, projectCode);
-    logger.LogInformation("Send/Receive result after CRDT sync: {srResult2}", srResult2.Output);
-    return TypedResults.Ok(result);
+    return fwdataApi;
+}
+
+static async Task<CrdtProject> SetupCrdtProject(string crdtFile,
+    ProjectLookupService projectLookupService,
+    Guid projectId,
+    ProjectsService projectsService,
+    string projectFolder,
+    Guid fwProjectId,
+    string lexboxUrl)
+{
+    if (File.Exists(crdtFile))
+    {
+        return new CrdtProject("crdt", crdtFile);
+    }
+    else
+    {
+        if (await projectLookupService.IsCrdtProject(projectId))
+        {
+            //todo determine what to do in this case, maybe we just download the project?
+            throw new InvalidOperationException("Project already exists, not sure why it's not on the server");
+        }
+        return await projectsService.CreateProject(new("crdt",
+            SeedNewProjectData: false,
+            Id: projectId,
+            Path: projectFolder,
+            FwProjectId: fwProjectId,
+            Domain: new Uri(lexboxUrl)));
+    }
+
 }
 

backend/FwHeadless/ProjectLookupService.cs

@@ -1,5 +1,6 @@
 using LexData;
 using Microsoft.EntityFrameworkCore;
+using SIL.Harmony.Core;
 
 namespace FwHeadless;
 
@@ -13,4 +14,9 @@ public class ProjectLookupService(LexBoxDbContext dbContext)
             .FirstOrDefaultAsync();
         return projectCode;
     }
+
+    public async Task<bool> IsCrdtProject(Guid projectId)
+    {
+        return await dbContext.Set<ServerCommit>().AnyAsync(c => c.ProjectId == projectId);
+    }
 }

backend/FwHeadless/appsettings.Development.json

@@ -12,7 +12,7 @@
   "Logging": {
     "LogLevel": {
       "Default": "Information",
-      "Microsoft.AspNetCore": "Warning"
+      "Microsoft.AspNetCore": "Information"
     }
   }
 }

backend/FwHeadless/appsettings.json

@@ -5,7 +5,7 @@
   "Logging": {
     "LogLevel": {
       "Default": "Information",
-      "Microsoft.AspNetCore": "Warning"
+      "Microsoft.AspNetCore": "Information"
     }
   },
   "AllowedHosts": "*"

backend/FwLite/FwDataMiniLcmBridge/FwDataFactory.cs

@@ -120,7 +120,7 @@ public void CloseCurrentProject()
         CloseProject(fwDataProject);
     }
 
-    private void CloseProject(FwDataProject project)
+    public void CloseProject(FwDataProject project)
     {
         // if we are shutting down, don't do anything because we want project dispose to be called as part of the shutdown process.
         if (_shuttingDown) return;

backend/FwLite/LcmCrdt/LcmCrdt.csproj

@@ -18,6 +18,8 @@
         <PackageReference Include="linq2db.SQLite" Version="5.4.1" />
         <PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" Version="8.0.0"/>
         <PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="8.0.0" />
+        <PackageReference Include="Refit" Version="7.1.2"/>
+        <PackageReference Include="Refit.HttpClientFactory" Version="7.1.2"/>
     </ItemGroup>
 
     <ItemGroup>

backend/FwLite/LcmCrdt/LcmCrdtKernel.cs

@@ -6,6 +6,7 @@
 using LcmCrdt.Changes;
 using LcmCrdt.Changes.Entries;
 using LcmCrdt.Objects;
+using LcmCrdt.RemoteSync;
 using LinqToDB;
 using LinqToDB.AspNet.Logging;
 using LinqToDB.Data;
@@ -14,6 +15,8 @@
 using Microsoft.EntityFrameworkCore;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using Refit;
 
 namespace LcmCrdt;
 
@@ -33,6 +36,17 @@ public static IServiceCollection AddLcmCrdtClient(this IServiceCollection servic
         services.AddScoped<CurrentProjectService>();
         services.AddSingleton<ProjectContext>();
         services.AddSingleton<ProjectsService>();
+
+        services.AddHttpClient();
+        services.AddSingleton<RefitSettings>(provider => new RefitSettings
+        {
+            ContentSerializer = new SystemTextJsonContentSerializer(new(JsonSerializerDefaults.Web)
+            {
+                TypeInfoResolver = provider.GetRequiredService<IOptions<CrdtConfig>>().Value
+                    .MakeJsonTypeResolver()
+            })
+        });
+        services.AddSingleton<CrdtHttpSyncService>();
         return services;
     }