From 417e6a83a5e81243a6b6ccae9fb4c9fd3a9fd072 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Mon, 4 Nov 2024 13:42:37 +0700 Subject: [PATCH 1/7] move crdt sync service into LcmCrdt --- backend/FwLite/LcmCrdt/LcmCrdt.csproj | 2 + backend/FwLite/LcmCrdt/LcmCrdtKernel.cs | 14 +++++++ .../RemoteSync}/CrdtHttpSyncService.cs | 37 +++++++------------ backend/FwLite/LocalWebApp/LocalAppKernel.cs | 10 ----- backend/FwLite/LocalWebApp/LocalWebApp.csproj | 2 - backend/FwLite/LocalWebApp/SyncService.cs | 22 ++++++++++- 6 files changed, 50 insertions(+), 37 deletions(-) rename backend/FwLite/{LocalWebApp => LcmCrdt/RemoteSync}/CrdtHttpSyncService.cs (71%) diff --git a/backend/FwLite/LcmCrdt/LcmCrdt.csproj b/backend/FwLite/LcmCrdt/LcmCrdt.csproj index c61cc2535..c2382ec8a 100644 --- a/backend/FwLite/LcmCrdt/LcmCrdt.csproj +++ b/backend/FwLite/LcmCrdt/LcmCrdt.csproj @@ -18,6 +18,8 @@ + + diff --git a/backend/FwLite/LcmCrdt/LcmCrdtKernel.cs b/backend/FwLite/LcmCrdt/LcmCrdtKernel.cs index 93ea9587d..2750fa531 100644 --- a/backend/FwLite/LcmCrdt/LcmCrdtKernel.cs +++ b/backend/FwLite/LcmCrdt/LcmCrdtKernel.cs @@ -6,6 +6,7 @@ using LcmCrdt.Changes; using LcmCrdt.Changes.Entries; using LcmCrdt.Objects; +using LcmCrdt.RemoteSync; using LinqToDB; using LinqToDB.AspNet.Logging; using LinqToDB.Data; @@ -14,6 +15,8 @@ using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Refit; namespace LcmCrdt; @@ -33,6 +36,17 @@ public static IServiceCollection AddLcmCrdtClient(this IServiceCollection servic services.AddScoped(); services.AddSingleton(); services.AddSingleton(); + + services.AddHttpClient(); + services.AddSingleton(provider => new RefitSettings + { + ContentSerializer = new SystemTextJsonContentSerializer(new(JsonSerializerDefaults.Web) + { + TypeInfoResolver = provider.GetRequiredService>().Value + .MakeJsonTypeResolver() + }) + }); + services.AddSingleton(); return services; } diff --git a/backend/FwLite/LocalWebApp/CrdtHttpSyncService.cs b/backend/FwLite/LcmCrdt/RemoteSync/CrdtHttpSyncService.cs similarity index 71% rename from backend/FwLite/LocalWebApp/CrdtHttpSyncService.cs rename to backend/FwLite/LcmCrdt/RemoteSync/CrdtHttpSyncService.cs index 85e06783e..d3c22087a 100644 --- a/backend/FwLite/LocalWebApp/CrdtHttpSyncService.cs +++ b/backend/FwLite/LcmCrdt/RemoteSync/CrdtHttpSyncService.cs @@ -1,13 +1,11 @@ -using SIL.Harmony.Core; -using SIL.Harmony; -using SIL.Harmony.Db; -using LcmCrdt; -using LocalWebApp.Auth; +using Microsoft.Extensions.Logging; using Refit; +using SIL.Harmony; +using SIL.Harmony.Core; -namespace LocalWebApp; +namespace LcmCrdt.RemoteSync; -public class CrdtHttpSyncService(AuthHelpersFactory authHelpersFactory, ILogger logger, RefitSettings refitSettings) +public class CrdtHttpSyncService(ILogger logger, RefitSettings refitSettings) { //todo replace with a IMemoryCache check private bool? _isHealthy; @@ -42,26 +40,19 @@ public async ValueTask ShouldSync(ISyncHttp syncHttp) return _isHealthy.Value; } - public async ValueTask CreateProjectSyncable(ProjectData project) + /// + /// Creates a Harmony sync client to represent a remote server + /// + /// project data, used to provide the projectId and clientId + /// should have the base url set to the remote server + /// + public async ValueTask CreateProjectSyncable(ProjectData project, HttpClient client) { - if (string.IsNullOrEmpty(project.OriginDomain)) - { - logger.LogWarning("Project {ProjectName} has no origin domain, unable to create http sync client", project.Name); - return NullSyncable.Instance; - } - - var client = await authHelpersFactory.GetHelper(project).CreateClient(); - if (client is null) - { - logger.LogWarning("Unable to create http client to sync project {ProjectName}, user is not authenticated to {OriginDomain}", project.Name, project.OriginDomain); - return NullSyncable.Instance; - } - - return new CrdtProjectSync(RestService.For(client, refitSettings), project.Id, project.ClientId , project.OriginDomain, this); + return new CrdtProjectSync(RestService.For(client, refitSettings), project.Id, project.ClientId, this); } } -public class CrdtProjectSync(ISyncHttp restSyncClient, Guid projectId, Guid clientId, string originDomain, CrdtHttpSyncService httpSyncService) : ISyncable +internal class CrdtProjectSync(ISyncHttp restSyncClient, Guid projectId, Guid clientId, CrdtHttpSyncService httpSyncService) : ISyncable { public ValueTask ShouldSync() { diff --git a/backend/FwLite/LocalWebApp/LocalAppKernel.cs b/backend/FwLite/LocalWebApp/LocalAppKernel.cs index b13345e35..f3e300034 100644 --- a/backend/FwLite/LocalWebApp/LocalAppKernel.cs +++ b/backend/FwLite/LocalWebApp/LocalAppKernel.cs @@ -40,16 +40,6 @@ public static IServiceCollection AddLocalAppServices(this IServiceCollection ser { jsonOptions.PayloadSerializerOptions.TypeInfoResolver = crdtConfig.Value.MakeJsonTypeResolver(); }); - services.AddHttpClient(); - services.AddSingleton(provider => new RefitSettings - { - ContentSerializer = new SystemTextJsonContentSerializer(new(JsonSerializerDefaults.Web) - { - TypeInfoResolver = provider.GetRequiredService>().Value - .MakeJsonTypeResolver() - }) - }); - services.AddSingleton(); return services; } diff --git a/backend/FwLite/LocalWebApp/LocalWebApp.csproj b/backend/FwLite/LocalWebApp/LocalWebApp.csproj index 40cfbee76..eedd69c3e 100644 --- a/backend/FwLite/LocalWebApp/LocalWebApp.csproj +++ b/backend/FwLite/LocalWebApp/LocalWebApp.csproj @@ -24,8 +24,6 @@ - - diff --git a/backend/FwLite/LocalWebApp/SyncService.cs b/backend/FwLite/LocalWebApp/SyncService.cs index ac8925a00..7c4be59f6 100644 --- a/backend/FwLite/LocalWebApp/SyncService.cs +++ b/backend/FwLite/LocalWebApp/SyncService.cs @@ -1,5 +1,6 @@ using SIL.Harmony; using LcmCrdt; +using LcmCrdt.RemoteSync; using LocalWebApp.Auth; using LocalWebApp.Services; using MiniLcm; @@ -11,7 +12,7 @@ namespace LocalWebApp; public class SyncService( DataModel dataModel, CrdtHttpSyncService remoteSyncServiceServer, - AuthHelpersFactory factory, + AuthHelpersFactory authHelpersFactory, CurrentProjectService currentProjectService, ChangeEventBus changeEventBus, IMiniLcmApi lexboxApi, @@ -19,7 +20,24 @@ public class SyncService( { public async Task ExecuteSync() { - var remoteModel = await remoteSyncServiceServer.CreateProjectSyncable(await currentProjectService.GetProjectData()); + var project = await currentProjectService.GetProjectData(); + if (string.IsNullOrEmpty(project.OriginDomain)) + { + logger.LogWarning("Project {ProjectName} has no origin domain, unable to create http sync client", + project.Name); + return new SyncResults([], [], false); + } + + var httpClient = await authHelpersFactory.GetHelper(project).CreateClient(); + if (httpClient is null) + { + logger.LogWarning( + "Unable to create http client to sync project {ProjectName}, user is not authenticated to {OriginDomain}", + project.Name, + project.OriginDomain); + return new SyncResults([], [], false); + } + var remoteModel = await remoteSyncServiceServer.CreateProjectSyncable(project, httpClient); var syncResults = await dataModel.SyncWith(remoteModel); //need to await this, otherwise the database connection will be closed before the notifications are sent await SendNotifications(syncResults); From 8328eeaec00add7a74c96337e40303df3d44abc6 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Mon, 4 Nov 2024 17:02:06 +0700 Subject: [PATCH 2/7] setup fw-headless to sync crdt changes back to lexbox --- backend/FwHeadless/CrdtSyncService.cs | 27 +++++++ backend/FwHeadless/FwHeadlessKernel.cs | 11 ++- backend/FwHeadless/HttpClientAuthHandler.cs | 73 ++++++++++++++++++ backend/FwHeadless/Program.cs | 74 ++++++++++++++++--- backend/FwHeadless/ProjectLookupService.cs | 6 ++ .../FwHeadless/appsettings.Development.json | 2 +- backend/FwHeadless/appsettings.json | 2 +- .../LcmCrdt/RemoteSync/CrdtHttpSyncService.cs | 11 +++ backend/LexBoxApi/Auth/AuthKernel.cs | 2 +- backend/LexCore/Auth/LexAuthConstants.cs | 1 + 10 files changed, 195 insertions(+), 14 deletions(-) create mode 100644 backend/FwHeadless/CrdtSyncService.cs create mode 100644 backend/FwHeadless/HttpClientAuthHandler.cs diff --git a/backend/FwHeadless/CrdtSyncService.cs b/backend/FwHeadless/CrdtSyncService.cs new file mode 100644 index 000000000..473227e11 --- /dev/null +++ b/backend/FwHeadless/CrdtSyncService.cs @@ -0,0 +1,27 @@ +using LcmCrdt; +using LcmCrdt.RemoteSync; +using SIL.Harmony; + +namespace FwHeadless; + +public class CrdtSyncService( + CrdtHttpSyncService httpSyncService, + IHttpClientFactory httpClientFactory, + CurrentProjectService currentProjectService, + DataModel dataModel, + ILogger logger) +{ + public async Task Sync() + { + var lexboxRemoteServer = await httpSyncService.CreateProjectSyncable( + currentProjectService.ProjectData, + httpClientFactory.CreateClient(FwHeadlessKernel.LexboxHttpClientName) + ); + var syncResults = await dataModel.SyncWith(lexboxRemoteServer); + if (!syncResults.IsSynced) throw new InvalidOperationException("Sync failed"); + logger.LogInformation( + "Synced with Lexbox, Downloaded changes: {MissingFromLocal}, Uploaded changes: {MissingFromRemote}", + syncResults.MissingFromLocal.Length, + syncResults.MissingFromRemote.Length); + } +} diff --git a/backend/FwHeadless/FwHeadlessKernel.cs b/backend/FwHeadless/FwHeadlessKernel.cs index 9a0d42b95..5021e57e5 100644 --- a/backend/FwHeadless/FwHeadlessKernel.cs +++ b/backend/FwHeadless/FwHeadlessKernel.cs @@ -1,11 +1,13 @@ using FwDataMiniLcmBridge; using FwLiteProjectSync; using LcmCrdt; +using Microsoft.Extensions.Options; namespace FwHeadless; public static class FwHeadlessKernel { + public const string LexboxHttpClientName = "LexboxHttpClient"; public static void AddFwHeadless(this IServiceCollection services) { services @@ -20,5 +22,12 @@ public static void AddFwHeadless(this IServiceCollection services) .AddLcmCrdtClient() .AddFwDataBridge() .AddFwLiteProjectSync(); + services.AddScoped(); + services.AddTransient(); + services.AddHttpClient(LexboxHttpClientName, + (provider, client) => + { + client.BaseAddress = new Uri(provider.GetRequiredService>().Value.LexboxUrl); + }).AddHttpMessageHandler(); } -}; +} diff --git a/backend/FwHeadless/HttpClientAuthHandler.cs b/backend/FwHeadless/HttpClientAuthHandler.cs new file mode 100644 index 000000000..cc3efa799 --- /dev/null +++ b/backend/FwHeadless/HttpClientAuthHandler.cs @@ -0,0 +1,73 @@ +using System.Net; +using LexCore; +using LexCore.Auth; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Options; + +namespace FwHeadless; + +public class HttpClientAuthHandler(IOptions config, IMemoryCache cache, ILogger logger) : DelegatingHandler +{ + protected override HttpResponseMessage Send(HttpRequestMessage request, CancellationToken cancellationToken) + { + throw new NotSupportedException("use async apis"); + } + + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + var lexboxUrl = new Uri(config.Value.LexboxUrl); + if (request.RequestUri?.Authority != lexboxUrl.Authority) + { + return await base.SendAsync(request, cancellationToken); + } + try + { + await SetAuthHeader(request, cancellationToken, lexboxUrl); + } + catch (Exception e) + { + throw new InvalidOperationException("Unable to set auth header", e); + } + return await base.SendAsync(request, cancellationToken); + } + + private async Task SetAuthHeader(HttpRequestMessage request, CancellationToken cancellationToken, Uri lexboxUrl) + { + var cookieContainer = new CookieContainer(); + cookieContainer.Add(new Cookie(LexAuthConstants.AuthCookieName, await GetToken(cancellationToken), null, lexboxUrl.Authority)); + request.Headers.Add("Cookie", cookieContainer.GetCookieHeader(lexboxUrl)); + } + + private async ValueTask GetToken(CancellationToken cancellationToken) + { + try + { + return await cache.GetOrCreateAsync("LexboxAuthToken", + async entry => + { + if (InnerHandler is null) throw new InvalidOperationException("InnerHandler is null"); + logger.LogInformation("Getting auth token"); + var client = new HttpClient(InnerHandler); + client.BaseAddress = new Uri(config.Value.LexboxUrl); + var response = await client.PostAsJsonAsync("/api/login", + new LoginRequest(config.Value.LexboxPassword, config.Value.LexboxUsername), + cancellationToken); + response.EnsureSuccessStatusCode(); + var cookies = response.Headers.GetValues("Set-Cookie"); + var cookieContainer = new CookieContainer(); + cookieContainer.SetCookies(response.RequestMessage!.RequestUri!, cookies.Single()); + var authCookie = cookieContainer.GetAllCookies() + .FirstOrDefault(c => c.Name == LexAuthConstants.AuthCookieName); + if (authCookie is null) throw new InvalidOperationException("Auth cookie not found"); + entry.SetValue(authCookie.Value); + entry.AbsoluteExpiration = authCookie.Expires; + logger.LogInformation("Got auth token: {AuthToken}", authCookie.Value); + return authCookie.Value; + }) ?? throw new NullReferenceException("unable to get the login token"); + } + catch (Exception e) + { + throw new InvalidOperationException("Unable to get auth token", e); + } + } +} diff --git a/backend/FwHeadless/Program.cs b/backend/FwHeadless/Program.cs index 261dedd9b..c34848d5b 100644 --- a/backend/FwHeadless/Program.cs +++ b/backend/FwHeadless/Program.cs @@ -1,7 +1,9 @@ using FwHeadless; using FwDataMiniLcmBridge; +using FwDataMiniLcmBridge.Api; using FwLiteProjectSync; using LcmCrdt; +using LcmCrdt.RemoteSync; using LexData; using Microsoft.AspNetCore.Http.HttpResults; using Microsoft.Extensions.Options; @@ -41,7 +43,7 @@ app.Run(); -static async Task, NotFound>> ExecuteMergeRequest( +static async Task, NotFound, ProblemHttpResult>> ExecuteMergeRequest( ILogger logger, IServiceProvider services, SendReceiveService srService, @@ -50,6 +52,8 @@ ProjectsService projectsService, ProjectLookupService projectLookupService, CrdtFwdataProjectSyncService syncService, + CrdtHttpSyncService crdtHttpSyncService, + IHttpClientFactory httpClientFactory, Guid projectId, bool dryRun = false) { @@ -67,6 +71,11 @@ return TypedResults.NotFound(); } logger.LogInformation("Project code is {projectCode}", projectCode); + //if we can't sync with lexbox fail fast + if (!await crdtHttpSyncService.TestAuth(httpClientFactory.CreateClient(FwHeadlessKernel.LexboxHttpClientName))) + { + return TypedResults.Problem("Unable to authenticate with Lexbox"); + } var projectFolder = Path.Join(config.Value.ProjectStorageRoot, $"{projectCode}-{projectId}"); if (!Directory.Exists(projectFolder)) Directory.CreateDirectory(projectFolder); @@ -77,6 +86,29 @@ logger.LogDebug("crdtFile: {crdtFile}", crdtFile); logger.LogDebug("fwDataFile: {fwDataFile}", fwDataProject.FilePath); + var fwdataApi = SetupFwData(fwDataProject, srService, projectCode, logger, fwDataFactory); + var crdtProject = await SetupCrdtProject(crdtFile, projectLookupService, projectId, projectsService, projectFolder, fwdataApi.ProjectId, config.Value.LexboxUrl); + + var miniLcmApi = await services.OpenCrdtProject(crdtProject); + var crdtSyncService = services.GetRequiredService(); + await crdtSyncService.Sync(); + + + var result = await syncService.Sync(miniLcmApi, fwdataApi, dryRun); + logger.LogInformation("Sync result, CrdtChanges: {CrdtChanges}, FwdataChanges: {FwdataChanges}", result.CrdtChanges, result.FwdataChanges); + + await crdtSyncService.Sync(); + var srResult2 = srService.SendReceive(fwDataProject, projectCode); + logger.LogInformation("Send/Receive result after CRDT sync: {srResult2}", srResult2.Output); + return TypedResults.Ok(result); +} + +static FwDataMiniLcmApi SetupFwData(FwDataProject fwDataProject, + SendReceiveService srService, + string projectCode, + ILogger logger, + FwDataFactory fwDataFactory) +{ if (File.Exists(fwDataProject.FilePath)) { var srResult = srService.SendReceive(fwDataProject, projectCode); @@ -87,15 +119,37 @@ var srResult = srService.Clone(fwDataProject, projectCode); logger.LogInformation("Send/Receive result: {srResult}", srResult.Output); } + var fwdataApi = fwDataFactory.GetFwDataMiniLcmApi(fwDataProject, true); - var crdtProject = File.Exists(crdtFile) ? - new CrdtProject("crdt", crdtFile) : - await projectsService.CreateProject(new("crdt", SeedNewProjectData: false, Path: projectFolder, FwProjectId: fwdataApi.ProjectId)); - var miniLcmApi = await services.OpenCrdtProject(crdtProject); - var result = await syncService.Sync(miniLcmApi, fwdataApi, dryRun); - logger.LogInformation("Sync result, CrdtChanges: {CrdtChanges}, FwdataChanges: {FwdataChanges}", result.CrdtChanges, result.FwdataChanges); - var srResult2 = srService.SendReceive(fwDataProject, projectCode); - logger.LogInformation("Send/Receive result after CRDT sync: {srResult2}", srResult2.Output); - return TypedResults.Ok(result); + return fwdataApi; +} + +static async Task SetupCrdtProject(string crdtFile, + ProjectLookupService projectLookupService, + Guid projectId, + ProjectsService projectsService, + string projectFolder, + Guid fwProjectId, + string lexboxUrl) +{ + if (File.Exists(crdtFile)) + { + return new CrdtProject("crdt", crdtFile); + } + else + { + if (await projectLookupService.IsCrdtProject(projectId)) + { + //todo determine what to do in this case, maybe we just download the project? + throw new InvalidOperationException("Project already exists, not sure why it's not on the server"); + } + return await projectsService.CreateProject(new("crdt", + SeedNewProjectData: false, + Id: projectId, + Path: projectFolder, + FwProjectId: fwProjectId, + Domain: new Uri(lexboxUrl))); + } + } diff --git a/backend/FwHeadless/ProjectLookupService.cs b/backend/FwHeadless/ProjectLookupService.cs index 9cb8cb971..ad3c6fa20 100644 --- a/backend/FwHeadless/ProjectLookupService.cs +++ b/backend/FwHeadless/ProjectLookupService.cs @@ -1,5 +1,6 @@ using LexData; using Microsoft.EntityFrameworkCore; +using SIL.Harmony.Core; namespace FwHeadless; @@ -13,4 +14,9 @@ public class ProjectLookupService(LexBoxDbContext dbContext) .FirstOrDefaultAsync(); return projectCode; } + + public async Task IsCrdtProject(Guid projectId) + { + return await dbContext.Set().AnyAsync(c => c.ProjectId == projectId); + } } diff --git a/backend/FwHeadless/appsettings.Development.json b/backend/FwHeadless/appsettings.Development.json index 22a96c2a5..c39164b02 100644 --- a/backend/FwHeadless/appsettings.Development.json +++ b/backend/FwHeadless/appsettings.Development.json @@ -12,7 +12,7 @@ "Logging": { "LogLevel": { "Default": "Information", - "Microsoft.AspNetCore": "Warning" + "Microsoft.AspNetCore": "Information" } } } diff --git a/backend/FwHeadless/appsettings.json b/backend/FwHeadless/appsettings.json index d56001975..cbe77f391 100644 --- a/backend/FwHeadless/appsettings.json +++ b/backend/FwHeadless/appsettings.json @@ -5,7 +5,7 @@ "Logging": { "LogLevel": { "Default": "Information", - "Microsoft.AspNetCore": "Warning" + "Microsoft.AspNetCore": "Information" } }, "AllowedHosts": "*" diff --git a/backend/FwLite/LcmCrdt/RemoteSync/CrdtHttpSyncService.cs b/backend/FwLite/LcmCrdt/RemoteSync/CrdtHttpSyncService.cs index d3c22087a..38028bc03 100644 --- a/backend/FwLite/LcmCrdt/RemoteSync/CrdtHttpSyncService.cs +++ b/backend/FwLite/LcmCrdt/RemoteSync/CrdtHttpSyncService.cs @@ -20,6 +20,10 @@ public async ValueTask ShouldSync(ISyncHttp syncHttp) { var responseMessage = await syncHttp.HealthCheck(); _isHealthy = responseMessage.IsSuccessStatusCode; + if (!_isHealthy.Value) + { + logger.LogWarning("Health check failed, response status code {StatusCode}", responseMessage.StatusCode); + } _lastHealthCheck = responseMessage.Headers.Date ?? DateTimeOffset.UtcNow; } catch (HttpRequestException e) @@ -50,6 +54,13 @@ public async ValueTask CreateProjectSyncable(ProjectData project, Htt { return new CrdtProjectSync(RestService.For(client, refitSettings), project.Id, project.ClientId, this); } + + public async ValueTask TestAuth(HttpClient client) + { + logger.LogInformation("Testing auth, client base url: {ClientBaseUrl}", client.BaseAddress); + var syncable = await CreateProjectSyncable(new ProjectData("test", Guid.Empty, null, Guid.Empty), client); + return await syncable.ShouldSync(); + } } internal class CrdtProjectSync(ISyncHttp restSyncClient, Guid projectId, Guid clientId, CrdtHttpSyncService httpSyncService) : ISyncable diff --git a/backend/LexBoxApi/Auth/AuthKernel.cs b/backend/LexBoxApi/Auth/AuthKernel.cs index 4cdd1193d..957c1a028 100644 --- a/backend/LexBoxApi/Auth/AuthKernel.cs +++ b/backend/LexBoxApi/Auth/AuthKernel.cs @@ -21,7 +21,7 @@ public static class AuthKernel { public const string DefaultScheme = "JwtOrCookie"; public const string JwtOverBasicAuthUsername = "bearer"; - public const string AuthCookieName = ".LexBoxAuth"; + public const string AuthCookieName = LexAuthConstants.AuthCookieName; public static void AddLexBoxAuth(IServiceCollection services, IConfigurationRoot configuration, diff --git a/backend/LexCore/Auth/LexAuthConstants.cs b/backend/LexCore/Auth/LexAuthConstants.cs index c5c4e398a..8150042e9 100644 --- a/backend/LexCore/Auth/LexAuthConstants.cs +++ b/backend/LexCore/Auth/LexAuthConstants.cs @@ -2,6 +2,7 @@ namespace LexCore.Auth; public static class LexAuthConstants { + public const string AuthCookieName = ".LexBoxAuth"; public const string RoleClaimType = "role"; public const string EmailClaimType = "email"; public const string UsernameClaimType = "user"; From 22d5afa613ca1a045b78aec4c8fbefa46bd14062 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Mon, 4 Nov 2024 17:03:07 +0700 Subject: [PATCH 3/7] close fwdata project once sync is done --- backend/FwHeadless/Program.cs | 1 + backend/FwLite/FwDataMiniLcmBridge/FwDataFactory.cs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/FwHeadless/Program.cs b/backend/FwHeadless/Program.cs index c34848d5b..0df8c4982 100644 --- a/backend/FwHeadless/Program.cs +++ b/backend/FwHeadless/Program.cs @@ -100,6 +100,7 @@ await crdtSyncService.Sync(); var srResult2 = srService.SendReceive(fwDataProject, projectCode); logger.LogInformation("Send/Receive result after CRDT sync: {srResult2}", srResult2.Output); + fwDataFactory.CloseProject(fwDataProject); return TypedResults.Ok(result); } diff --git a/backend/FwLite/FwDataMiniLcmBridge/FwDataFactory.cs b/backend/FwLite/FwDataMiniLcmBridge/FwDataFactory.cs index 3490145a8..fb085fefe 100644 --- a/backend/FwLite/FwDataMiniLcmBridge/FwDataFactory.cs +++ b/backend/FwLite/FwDataMiniLcmBridge/FwDataFactory.cs @@ -120,7 +120,7 @@ public void CloseCurrentProject() CloseProject(fwDataProject); } - private void CloseProject(FwDataProject project) + public void CloseProject(FwDataProject project) { // if we are shutting down, don't do anything because we want project dispose to be called as part of the shutdown process. if (_shuttingDown) return; From 166b4d2aa367aca17c9101bb65c52e8a9a2fcf71 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Mon, 4 Nov 2024 17:07:18 +0700 Subject: [PATCH 4/7] close fw project before trying to do S&R --- backend/FwHeadless/Program.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/FwHeadless/Program.cs b/backend/FwHeadless/Program.cs index 0df8c4982..261047d99 100644 --- a/backend/FwHeadless/Program.cs +++ b/backend/FwHeadless/Program.cs @@ -96,11 +96,11 @@ var result = await syncService.Sync(miniLcmApi, fwdataApi, dryRun); logger.LogInformation("Sync result, CrdtChanges: {CrdtChanges}, FwdataChanges: {FwdataChanges}", result.CrdtChanges, result.FwdataChanges); + fwDataFactory.CloseProject(fwDataProject); await crdtSyncService.Sync(); var srResult2 = srService.SendReceive(fwDataProject, projectCode); logger.LogInformation("Send/Receive result after CRDT sync: {srResult2}", srResult2.Output); - fwDataFactory.CloseProject(fwDataProject); return TypedResults.Ok(result); } From 935c565ed7e8692dffa061a23f6bfe82a3a9c40a Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Wed, 6 Nov 2024 11:14:41 +0700 Subject: [PATCH 5/7] change seeding data default when creating a new CRDT project --- backend/FwLite/FwLiteProjectSync.Tests/Fixtures/SyncFixture.cs | 2 +- backend/FwLite/LcmCrdt.Tests/OpenProjectTests.cs | 2 +- backend/FwLite/LcmCrdt/ProjectsService.cs | 2 +- backend/FwLite/LocalWebApp/Routes/ProjectRoutes.cs | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/SyncFixture.cs b/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/SyncFixture.cs index 944de1834..025203306 100644 --- a/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/SyncFixture.cs +++ b/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/SyncFixture.cs @@ -58,7 +58,7 @@ public async Task InitializeAsync() if (Path.Exists(crdtProjectsFolder)) Directory.Delete(crdtProjectsFolder, true); Directory.CreateDirectory(crdtProjectsFolder); var crdtProject = await _services.ServiceProvider.GetRequiredService() - .CreateProject(new(_projectName, FwProjectId: FwDataApi.ProjectId)); + .CreateProject(new(_projectName, FwProjectId: FwDataApi.ProjectId, SeedNewProjectData: true)); CrdtApi = (CrdtMiniLcmApi) await _services.ServiceProvider.OpenCrdtProject(crdtProject); } diff --git a/backend/FwLite/LcmCrdt.Tests/OpenProjectTests.cs b/backend/FwLite/LcmCrdt.Tests/OpenProjectTests.cs index e53ea52e8..cdaa72ab0 100644 --- a/backend/FwLite/LcmCrdt.Tests/OpenProjectTests.cs +++ b/backend/FwLite/LcmCrdt.Tests/OpenProjectTests.cs @@ -16,7 +16,7 @@ public async Task OpeningAProjectWorks() var services = host.Services; var asyncScope = services.CreateAsyncScope(); await asyncScope.ServiceProvider.GetRequiredService() - .CreateProject(new(Name: "OpeningAProjectWorks", Path: "")); + .CreateProject(new(Name: "OpeningAProjectWorks", Path: "", SeedNewProjectData: true)); var miniLcmApi = (CrdtMiniLcmApi)await asyncScope.ServiceProvider.OpenCrdtProject(new CrdtProject("OpeningAProjectWorks", sqliteConnectionString)); miniLcmApi.ProjectData.Name.Should().Be("OpeningAProjectWorks"); diff --git a/backend/FwLite/LcmCrdt/ProjectsService.cs b/backend/FwLite/LcmCrdt/ProjectsService.cs index 4a9950e8b..91c3c05cd 100644 --- a/backend/FwLite/LcmCrdt/ProjectsService.cs +++ b/backend/FwLite/LcmCrdt/ProjectsService.cs @@ -38,7 +38,7 @@ public record CreateProjectRequest( Guid? Id = null, Uri? Domain = null, Func? AfterCreate = null, - bool SeedNewProjectData = true, + bool SeedNewProjectData = false, string? Path = null, Guid? FwProjectId = null); diff --git a/backend/FwLite/LocalWebApp/Routes/ProjectRoutes.cs b/backend/FwLite/LocalWebApp/Routes/ProjectRoutes.cs index 1ad474bf1..1d945394c 100644 --- a/backend/FwLite/LocalWebApp/Routes/ProjectRoutes.cs +++ b/backend/FwLite/LocalWebApp/Routes/ProjectRoutes.cs @@ -69,7 +69,7 @@ public static IEndpointConventionBuilder MapProjectRoutes(this WebApplication ap return Results.BadRequest("Project already exists"); if (!ProjectName().IsMatch(name)) return Results.BadRequest("Only letters, numbers, '-' and '_' are allowed"); - await projectService.CreateProject(new(name, AfterCreate: AfterCreate)); + await projectService.CreateProject(new(name, AfterCreate: AfterCreate, SeedNewProjectData: true)); return TypedResults.Ok(); }); group.MapPost($"/upload/crdt/{{serverAuthority}}/{{{CrdtMiniLcmApiHub.ProjectRouteKey}}}", From 919efb8befca86b39d58a43a4cdb5460c027ef91 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Wed, 6 Nov 2024 11:22:23 +0700 Subject: [PATCH 6/7] defer closing the fw project with a using statement. Make SendReceive async --- backend/FwHeadless/Program.cs | 12 ++++++------ backend/FwHeadless/SendReceiveHelpers.cs | 16 ++++++++++------ backend/FwHeadless/SendReceiveService.cs | 8 ++++---- .../FwLite/FwDataMiniLcmBridge/FwDataFactory.cs | 6 ++++++ .../FwDataMiniLcmBridge.csproj | 1 + 5 files changed, 27 insertions(+), 16 deletions(-) diff --git a/backend/FwHeadless/Program.cs b/backend/FwHeadless/Program.cs index 261047d99..3d3484df2 100644 --- a/backend/FwHeadless/Program.cs +++ b/backend/FwHeadless/Program.cs @@ -86,7 +86,8 @@ logger.LogDebug("crdtFile: {crdtFile}", crdtFile); logger.LogDebug("fwDataFile: {fwDataFile}", fwDataProject.FilePath); - var fwdataApi = SetupFwData(fwDataProject, srService, projectCode, logger, fwDataFactory); + var fwdataApi = await SetupFwData(fwDataProject, srService, projectCode, logger, fwDataFactory); + using var deferCloseFwData = fwDataFactory.DeferClose(fwDataProject); var crdtProject = await SetupCrdtProject(crdtFile, projectLookupService, projectId, projectsService, projectFolder, fwdataApi.ProjectId, config.Value.LexboxUrl); var miniLcmApi = await services.OpenCrdtProject(crdtProject); @@ -96,15 +97,14 @@ var result = await syncService.Sync(miniLcmApi, fwdataApi, dryRun); logger.LogInformation("Sync result, CrdtChanges: {CrdtChanges}, FwdataChanges: {FwdataChanges}", result.CrdtChanges, result.FwdataChanges); - fwDataFactory.CloseProject(fwDataProject); await crdtSyncService.Sync(); - var srResult2 = srService.SendReceive(fwDataProject, projectCode); + var srResult2 = await srService.SendReceive(fwDataProject, projectCode); logger.LogInformation("Send/Receive result after CRDT sync: {srResult2}", srResult2.Output); return TypedResults.Ok(result); } -static FwDataMiniLcmApi SetupFwData(FwDataProject fwDataProject, +static async Task SetupFwData(FwDataProject fwDataProject, SendReceiveService srService, string projectCode, ILogger logger, @@ -112,12 +112,12 @@ static FwDataMiniLcmApi SetupFwData(FwDataProject fwDataProject, { if (File.Exists(fwDataProject.FilePath)) { - var srResult = srService.SendReceive(fwDataProject, projectCode); + var srResult = await srService.SendReceive(fwDataProject, projectCode); logger.LogInformation("Send/Receive result: {srResult}", srResult.Output); } else { - var srResult = srService.Clone(fwDataProject, projectCode); + var srResult = await srService.Clone(fwDataProject, projectCode); logger.LogInformation("Send/Receive result: {srResult}", srResult.Output); } diff --git a/backend/FwHeadless/SendReceiveHelpers.cs b/backend/FwHeadless/SendReceiveHelpers.cs index 9c8ac9494..078011bd8 100644 --- a/backend/FwHeadless/SendReceiveHelpers.cs +++ b/backend/FwHeadless/SendReceiveHelpers.cs @@ -17,10 +17,14 @@ public SendReceiveAuth(FwHeadlessConfig config) : this(config.LexboxUsername, co public record LfMergeBridgeResult(string Output, string ProgressMessages); - private static LfMergeBridgeResult CallLfMergeBridge(string method, IDictionary flexBridgeOptions) + private static async Task CallLfMergeBridge(string method, IDictionary flexBridgeOptions) { var progress = new StringBuilderProgress(); - LfMergeBridge.LfMergeBridge.Execute(method, progress, flexBridgeOptions.ToDictionary(), out var lfMergeBridgeOutputForClient); + var lfMergeBridgeOutputForClient = await Task.Run(() => + { + LfMergeBridge.LfMergeBridge.Execute(method, progress, flexBridgeOptions.ToDictionary(), out var output); + return output; + }); return new LfMergeBridgeResult(lfMergeBridgeOutputForClient, progress.ToString()); } @@ -45,7 +49,7 @@ private static Uri BuildSendReceiveUrl(string baseUrl, string projectCode, SendR return builder.Uri; } - public static LfMergeBridgeResult SendReceive(FwDataProject project, string? projectCode = null, string baseUrl = "http://localhost", SendReceiveAuth? auth = null, string fdoDataModelVersion = "7000072", string? commitMessage = null) + public static async Task SendReceive(FwDataProject project, string? projectCode = null, string baseUrl = "http://localhost", SendReceiveAuth? auth = null, string fdoDataModelVersion = "7000072", string? commitMessage = null) { projectCode ??= project.Name; var fwdataInfo = new FileInfo(project.FilePath); @@ -65,10 +69,10 @@ public static LfMergeBridgeResult SendReceive(FwDataProject project, string? pro { "user", "LexBox" }, }; if (commitMessage is not null) flexBridgeOptions["commitMessage"] = commitMessage; - return CallLfMergeBridge("Language_Forge_Send_Receive", flexBridgeOptions); + return await CallLfMergeBridge("Language_Forge_Send_Receive", flexBridgeOptions); } - public static LfMergeBridgeResult CloneProject(FwDataProject project, string? projectCode = null, string baseUrl = "http://localhost", SendReceiveAuth? auth = null, string fdoDataModelVersion = "7000072") + public static async Task CloneProject(FwDataProject project, string? projectCode = null, string baseUrl = "http://localhost", SendReceiveAuth? auth = null, string fdoDataModelVersion = "7000072") { projectCode ??= project.Name; var fwdataInfo = new FileInfo(project.FilePath); @@ -84,6 +88,6 @@ public static LfMergeBridgeResult CloneProject(FwDataProject project, string? pr { "languageDepotRepoUri", repoUrl.ToString() }, { "deleteRepoIfNoSuchBranch", "false" }, }; - return CallLfMergeBridge("Language_Forge_Clone", flexBridgeOptions); + return await CallLfMergeBridge("Language_Forge_Clone", flexBridgeOptions); } } diff --git a/backend/FwHeadless/SendReceiveService.cs b/backend/FwHeadless/SendReceiveService.cs index 37023dc8f..d924034f6 100644 --- a/backend/FwHeadless/SendReceiveService.cs +++ b/backend/FwHeadless/SendReceiveService.cs @@ -5,9 +5,9 @@ namespace FwHeadless; public class SendReceiveService(IOptions config) { - public SendReceiveHelpers.LfMergeBridgeResult SendReceive(FwDataProject project, string? projectCode, string? commitMessage = null) + public async Task SendReceive(FwDataProject project, string? projectCode, string? commitMessage = null) { - return SendReceiveHelpers.SendReceive( + return await SendReceiveHelpers.SendReceive( project: project, projectCode: projectCode, baseUrl: config.Value.HgWebUrl, @@ -17,9 +17,9 @@ public SendReceiveHelpers.LfMergeBridgeResult SendReceive(FwDataProject project, ); } - public SendReceiveHelpers.LfMergeBridgeResult Clone(FwDataProject project, string? projectCode) + public async Task Clone(FwDataProject project, string? projectCode) { - return SendReceiveHelpers.CloneProject( + return await SendReceiveHelpers.CloneProject( project: project, projectCode: projectCode, baseUrl: config.Value.HgWebUrl, diff --git a/backend/FwLite/FwDataMiniLcmBridge/FwDataFactory.cs b/backend/FwLite/FwDataMiniLcmBridge/FwDataFactory.cs index fb085fefe..78fc74092 100644 --- a/backend/FwLite/FwDataMiniLcmBridge/FwDataFactory.cs +++ b/backend/FwLite/FwDataMiniLcmBridge/FwDataFactory.cs @@ -1,5 +1,6 @@ using FwDataMiniLcmBridge.Api; using FwDataMiniLcmBridge.LcmUtils; +using LexCore.Utils; using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; @@ -130,4 +131,9 @@ public void CloseProject(FwDataProject project) if (lcmCache is null) return; cache.Remove(cacheKey); } + + public IDisposable DeferClose(FwDataProject project) + { + return Defer.Action(() => CloseProject(project)); + } } diff --git a/backend/FwLite/FwDataMiniLcmBridge/FwDataMiniLcmBridge.csproj b/backend/FwLite/FwDataMiniLcmBridge/FwDataMiniLcmBridge.csproj index 457f4d84e..2192d7724 100644 --- a/backend/FwLite/FwDataMiniLcmBridge/FwDataMiniLcmBridge.csproj +++ b/backend/FwLite/FwDataMiniLcmBridge/FwDataMiniLcmBridge.csproj @@ -28,6 +28,7 @@ + From bfa6c09335584b93eaa4639fa02c176eeda673a2 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Mon, 11 Nov 2024 13:09:23 +0700 Subject: [PATCH 7/7] change sync endpoint to 'api/crdt-sync' --- backend/FwHeadless/Program.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/FwHeadless/Program.cs b/backend/FwHeadless/Program.cs index 3d3484df2..51ffbac80 100644 --- a/backend/FwHeadless/Program.cs +++ b/backend/FwHeadless/Program.cs @@ -39,7 +39,7 @@ app.MapHealthChecks("/api/healthz"); -app.MapPost("/sync", ExecuteMergeRequest); +app.MapPost("/api/crdt-sync", ExecuteMergeRequest); app.Run();