Merge branch 'fp/flexible-schema-epic' into fp/flexible-schema

# Conflicts:
#	wrappers/realm-core

.editorconfig

@@ -282,7 +282,7 @@ dotnet_diagnostic.SA1637.severity = none
 
 dotnet_diagnostic.SX1101.severity = warning
 
-
+resharper_inconsistent_naming_highlighting = none
 
 
 ## Visual Studio generated .editorconfig file with C++ settings.

.github/build-workflows.ps1

@@ -0,0 +1,3 @@
+Get-ChildItem -Path './pkl-workflows' -Filter *.pkl -File -Name | ForEach-Object {
+    &pkl eval ./pkl-workflows/$_ -o "./workflows/$($_.Replace('pkl', 'yml'))"
+}

.github/pkl-workflows/Readme.md

@@ -14,6 +14,8 @@ brew install pkl
 
 ## Building the workflows
 
+Run `pwsh .github/build-workflows.ps1` or the following bash script if you don't have/use powershell:
+
 ```bash
 cd $SolutionDir/.github/pkl-workflows
 for file in *.pkl ; do pkl eval $file -o  ../workflows/$(echo $file | sed s/pkl/yml/) ; done

.github/pkl-workflows/helpers/Actions.pkl

@@ -22,3 +22,4 @@ const mergePR = "juliangruber/merge-pull-request-action@9234b8714dda9a08f3d1df5b
 const createPR = "peter-evans/create-pull-request@6d6857d36972b65feb161a90e484f2984215f83e" // 6.0.5
 const publishGithubRelease = "ncipollo/release-action@2c591bcc8ecdcd2db72b97d6147f871fcd833ba5" // 1.14.0
 const releaseToSlack = "realm/ci-actions/release-to-slack@6418e15ed9bbdb19b7d456a347e5623779f95cdf"
+const setupCmake = "jwlawson/actions-setup-cmake@802fa1a2c4e212495c05bf94dba2704a92a472be" // 2.0.2

.github/pkl-workflows/helpers/Package.pkl

@@ -116,4 +116,27 @@ local const function buildDocs(condition: String): Listing<gha.Step> = new {
     }
     run = "C:\\docfx\\docfx Docs/docfx.json"
   }
+  new {
+    name = "Update Improve this doc links"
+    `if` = condition
+    run = """
+      Get-ChildItem Docs/_site -Filter *.html -Recurse -File |
+      ForEach-Object {
+        $content = ($_ | Get-Content -Raw)
+        $content = $content -replace "/Docs/apispec/new\\?filename", "/Docs/apispec?filename"
+        Set-Content $_.FullName $content
+      }
+      """
+    shell = "pwsh"
+  }
+  new {
+    name = "Archive docs"
+    `if` = condition
+    run = """
+      Compress-Archive -Path Docs/_site -DestinationPath "Realm/packages/Docs.zip"
+      """
+  }
+  (Steps.uploadArtifacts("Docs.zip", "Realm/packages/Docs.zip")) {
+    `if` = condition
+  }
 }

.github/pkl-workflows/helpers/Steps.pkl

@@ -60,6 +60,14 @@ const function setupXcode(version: String): gha.Step = new {
   }
 }
 
+const function setupCmake(version: String): gha.Step = new {
+  name = "Setup Cmake"
+  uses = Actions.setupCmake
+  with {
+    ["cmake-version"] = version
+  }
+}
+
 const function setupMSVC(edition: String): gha.Step = new {
   name = "Setup MSVC"
   run = "Start-Process \"C:\\Program Files (x86)\\Microsoft Visual Studio\\Installer\\vs_installer.exe\" -ArgumentList 'modify --installPath \"C:\\Program Files\\Microsoft Visual Studio\\2022\\Enterprise\" --quiet --add Microsoft.VisualStudio.Component.VC.14.35.17.5.\(edition)' -Wait -PassThru"

.github/pkl-workflows/publish-release.pkl

@@ -25,7 +25,7 @@ jobs {
       ...uploadToNuGet()
       ...Steps.uploadToNPM("latest")
       ...mergeReleasePR()
-      publishGithubReleasee()
+      publishGithubRelease()
       ...updateChangelogForvNext()
       postReleaseToSlack()
     }
@@ -77,7 +77,7 @@ local function mergePR(numberExpression: String): Step = new {
   }
 }
 
-local function publishGithubReleasee(): Step = new {
+local function publishGithubRelease(): Step = new {
   name = "Publish Github Release"
   uses = Actions.publishGithubRelease
   with {
@@ -125,6 +125,7 @@ local function updateChangelogForvNext(): Listing<Step> = new {
       ["delete-branch"] = true
       ["base"] = Common.mainBranch
       ["commit-message"] = "Prepare for vNext"
+      ["labels"] = "no-jira-ticket"
     }
   }
   mergePR("${{ steps.\(create_pr_step).outputs.pull-request-number }}")

.github/pkl-workflows/wrappers.pkl

@@ -52,13 +52,19 @@ jobs {
   } |> wrappersJob(new JobConfig {
       cmd = "./wrappers/build-macos.sh"
       platform = "macos"
+      intermediateSteps {
+        Steps.setupCmake("3.29.0")
+      }
     })
   ["catalyst"] = new Job {
     `runs-on` = "macos-12"
     name = "Catalyst"
   } |> wrappersJob(new JobConfig {
       cmd = "pwsh ./wrappers/build-apple-platform.ps1 Catalyst"
       platform = "catalyst"
+      intermediateSteps {
+        Steps.setupCmake("3.29.0")
+      }
     })
   ["apple-platform"] = new MatrixJob {
     strategy {
@@ -75,6 +81,7 @@ jobs {
       platform = "${{ matrix.platform }}-${{ matrix.target }}"
       intermediateSteps {
         Steps.setupXcode("14")
+        Steps.setupCmake("3.29.0")
       }
       ifCondition = "\(Common.applePlatformTargets((platform, target) -> "needs.\(job_CheckCache).outputs.wrappers-\(platform)-\(target) != 'true'").join(" || "))"
     })

.github/workflows/main.yml

@@ -274,6 +274,27 @@ jobs:
       env:
         DOCFX_SOURCE_BRANCH_NAME: ${{ github.head_ref }}
       run: C:\docfx\docfx Docs/docfx.json
+    - name: Update Improve this doc links
+      if: contains(github.head_ref, 'release')
+      run: |-
+        Get-ChildItem Docs/_site -Filter *.html -Recurse -File |
+        ForEach-Object {
+          $content = ($_ | Get-Content -Raw)
+          $content = $content -replace "/Docs/apispec/new\?filename", "/Docs/apispec?filename"
+          Set-Content $_.FullName $content
+        }
+      shell: pwsh
+    - name: Archive docs
+      if: contains(github.head_ref, 'release')
+      run: Compress-Archive -Path Docs/_site -DestinationPath "Realm/packages/Docs.zip"
+    - name: Store artifacts for Docs.zip
+      if: contains(github.head_ref, 'release')
+      uses: actions/upload-artifact@v4
+      with:
+        name: Docs.zip
+        path: Realm/packages/Docs.zip
+        retention-days: ${{ github.event_name != 'pull_request' && 30 || 1 }}
+        if-no-files-found: error
   build-unity:
     name: Package Unity
     needs:

.github/workflows/pr.yml

@@ -274,6 +274,27 @@ jobs:
       env:
         DOCFX_SOURCE_BRANCH_NAME: ${{ github.head_ref }}
       run: C:\docfx\docfx Docs/docfx.json
+    - name: Update Improve this doc links
+      if: contains(github.head_ref, 'release')
+      run: |-
+        Get-ChildItem Docs/_site -Filter *.html -Recurse -File |
+        ForEach-Object {
+          $content = ($_ | Get-Content -Raw)
+          $content = $content -replace "/Docs/apispec/new\?filename", "/Docs/apispec?filename"
+          Set-Content $_.FullName $content
+        }
+      shell: pwsh
+    - name: Archive docs
+      if: contains(github.head_ref, 'release')
+      run: Compress-Archive -Path Docs/_site -DestinationPath "Realm/packages/Docs.zip"
+    - name: Store artifacts for Docs.zip
+      if: contains(github.head_ref, 'release')
+      uses: actions/upload-artifact@v4
+      with:
+        name: Docs.zip
+        path: Realm/packages/Docs.zip
+        retention-days: ${{ github.event_name != 'pull_request' && 30 || 1 }}
+        if-no-files-found: error
   build-unity:
     name: Package Unity
     needs:

.github/workflows/prepare-release.yml

@@ -56,3 +56,4 @@ jobs:
           body: An automated PR for next release.
           commit-message: Prepare for ${{ steps.update-changelog.outputs.new-version }}
           token: ${{ secrets.REALM_CI_PAT }}
+          labels: no-jira-ticket

.github/workflows/publish-release.yml

@@ -107,6 +107,7 @@ jobs:
         delete-branch: true
         base: main
         commit-message: Prepare for vNext
+        labels: no-jira-ticket
     - name: Merge Pull Request
       uses: juliangruber/merge-pull-request-action@9234b8714dda9a08f3d1df5b2a6a3abd7b695353
       with:

.github/workflows/wrappers.yml

@@ -405,6 +405,11 @@ jobs:
       with:
         path: ./wrappers/build/**
         key: wrappers-macos-Release-${{hashFiles('./wrappers/**')}}
+    - name: Setup Cmake
+      if: steps.check-cache.outputs.cache-hit != 'true'
+      uses: jwlawson/actions-setup-cmake@802fa1a2c4e212495c05bf94dba2704a92a472be
+      with:
+        cmake-version: 3.29.0
     - name: Build Wrappers
       if: steps.check-cache.outputs.cache-hit != 'true'
       run: ./wrappers/build-macos.sh --configuration=Release -DCMAKE_INTERPROCEDURAL_OPTIMIZATION='ON'
@@ -434,6 +439,11 @@ jobs:
       with:
         path: ./wrappers/build/**
         key: wrappers-catalyst-Release-${{hashFiles('./wrappers/**')}}
+    - name: Setup Cmake
+      if: steps.check-cache.outputs.cache-hit != 'true'
+      uses: jwlawson/actions-setup-cmake@802fa1a2c4e212495c05bf94dba2704a92a472be
+      with:
+        cmake-version: 3.29.0
     - name: Build Wrappers
       if: steps.check-cache.outputs.cache-hit != 'true'
       run: pwsh ./wrappers/build-apple-platform.ps1 Catalyst -Configuration Release -EnableLTO
@@ -468,6 +478,11 @@ jobs:
       uses: maxim-lobanov/setup-xcode@60606e260d2fc5762a71e64e74b2174e8ea3c8bd
       with:
         xcode-version: '14'
+    - name: Setup Cmake
+      if: steps.check-cache.outputs.cache-hit != 'true'
+      uses: jwlawson/actions-setup-cmake@802fa1a2c4e212495c05bf94dba2704a92a472be
+      with:
+        cmake-version: 3.29.0
     - name: Build Wrappers
       if: steps.check-cache.outputs.cache-hit != 'true'
       run: 'pwsh ./wrappers/build-apple-platform.ps1 ${{ matrix.platform }} -Targets ${{ matrix.target }} -Configuration Release '

CHANGELOG.md

@@ -1,21 +1,78 @@
 ## vNext (TBD)
 
+### Enhancements
+* None
+
+### Fixed
+* None
+
+### Compatibility
+* Realm Studio: 15.0.0 or later.
+
+### Internal
+* Using Core x.y.z.
+
+## 12.3.0 (2024-07-17)
+
+### Deprecations
+* The `Logger` has been deprecated in favor of `RealmLogger`, which `Logger` currently derives from. (PR [#3634](https://github.com/realm/realm-dotnet/pull/3634))
+  * The `Logger.LogLevel` `set` and `get` accessors have been deprecated. Please use `RealmLogger.SetLogLevel()` and `RealmLogger.GetLogLevel()` (see **Enhancements** below).
+  * The `Logger.Function(Action<LogLevel, string> logFunction)` have been deprecated. Please use `RealmLogger.Function(Action<LogLevel, LogCategory, string> logFunction)` (see **Enhancements** below).
+
 ### Enhancements
 * Allow `ShouldCompactOnLaunch` to be set on `SyncConfiguration`, not only `RealmConfiguration`. (Issue [#3617](https://github.com/realm/realm-dotnet/issues/3617))
+* Reduce the size of the local transaction log produced by creating objects, improving the performance of insertion-heavy transactions (Core 14.10.0).
+* Performance has been improved for range queries on integers and timestamps. Requires that you use the "BETWEEN" operation in `Realm.All<T>().Filter(...)`. (Core 14.10.1)
+* Allowed `ShouldCompactOnLaunch` to be set on `SyncConfiguration`, not only `RealmConfiguration`. (Issue [#3617](https://github.com/realm/realm-dotnet/issues/3617))
+* Introduced a `LogCategory` and allowed for more control over which category of messages should be logged and at which criticality level. (PR [#3634](https://github.com/realm/realm-dotnet/pull/3634))
+  * Allowed setting and getting a `LogLevel` for a given `LogCategory`. The hierarchy of categories starts at `LogCategory.Realm`.
+    ```csharp
+    RealmLogger.SetLogLevel(LogLevel.Warn, LogCategory.Realm.Sync);
+    RealmLogger.GetLogLevel(LogCategory.Realm.Sync.Client.Session); // LogLevel.Warn
+    ```
+  * Added a function logger that accepts a callback that will receive the `LogLevel`, `LogCategory`, and the message when invoked.
+    ```csharp
+    RealmLogger.Default = RealmLogger.Function((level, category, message) => /* custom implementation */);
+    ```
+  * Added a `RealmLogger.Log()` overload taking a category. The pre-existing `Log()` API will implicitly log at `LogCategory.Realm.SDK`.
+    ```csharp
+    RealmLogger.Default.Log(LogLevel.Warn, LogCategory.Realm, "A warning message");
+    ```
+* On Windows devices Device Sync will additionally look up SSL certificates in the Windows Trusted Root Certification Authorities certificate store when establishing a connection. (Core 14.11.0)
+* Role and permissions changes no longer require a client reset to update the local realm. (Core 14.11.0)
 
 ### Fixed
 * A `ForCurrentlyOutstandingWork` progress notifier would not immediately call its callback after registration. Instead you would have to wait for some data to be received to get your first update - if you were already caught up when you registered the notifier you could end up waiting a long time for the server to deliver a download that would call/expire your notifier. (Core 14.8.0)
 * After compacting, a file upgrade would be triggered. This could cause loss of data if `ShouldDeleteIfMigrationNeeded` is set to `true`. (Issue [#3583](https://github.com/realm/realm-dotnet/issues/3583), Core 14.9.0)
+* Passing in a deleted object as a substitution argument to `.Filter()` would throw a confusing error with a message starting with `invalid RQL for table`. It now throws a more descriptive error instead. (Issue [#3619](https://github.com/realm/realm-dotnet/issues/3619))
+* Fix some client resets (such as migrating to flexible sync) potentially failing with AutoClientResetFailed if a new client reset condition (such as rolling back a flexible sync migration) occurred before the first one completed. (Core 14.10.0)
+* Encrypted files on Windows had a maximum size of 2GB even on x64 due to internal usage of `off_t`, which is a 32-bit type on 64-bit Windows. (Core 14.10.0)
+* The encryption code no longer behaves differently depending on the system page size, which should entirely eliminate a recurring source of bugs related to copying encrypted Realm files between platforms with different page sizes. One known outstanding bug was ([RNET-1141](https://github.com/realm/realm-dotnet/issues/3592)), where opening files on a system with a larger page size than the writing system would attempt to read sections of the file which had never been written to. (Core 14.10.0)
+* There were several complicated scenarios which could result in stale reads from encrypted files in multiprocess scenarios. These were very difficult to hit and would typically lead to a crash, either due to an assertion failure or DecryptionFailure being thrown. (Core 14.10.0)
+* Encrypted files have some benign data races where we can memcpy a block of memory while another thread is writing to a limited range of it. It is logically impossible to ever read from that range when this happens, but Thread Sanitizer quite reasonably complains about this. We now perform a slower operations when running with TSan which avoids this benign race. (Core 14.10.0)
+* Tokenizing strings for full-text search could pass values outside the range [-1, 255] to `isspace()`, which is undefined behavior. (Core 14.10.0)
+* Opening an Flexible Sync Realm asynchronously may not wait to download all data. (Core 14.10.1)
+* Clearing a List of `RealmValue` in an upgraded file would lead to an assertion failing. (Core 14.10.1)
+* You could get unexpected merge results when assigning to a nested collection. (Core 14.10.2)
+* Fixed removing backlinks from the wrong objects if the link came from a nested list, nested dictionary, top-level dictionary, or list of `RealmValue`, and the source table had more than 256 objects. This could manifest as `array_backlink.cpp:112: Assertion failed: int64_t(value >> 1) == key.value` when removing an object. (Core 14.10.3)
+* Fixed the collapse/rejoin of clusters which contained nested collections with links. This could manifest as `array.cpp:319: Array::move() Assertion failed: begin <= end [2, 1]` when removing an object. (Core 14.10.3)
+* `Session.WaitForUpload()` was inconsistent in how it handled commits which did not produce any changesets to upload. Previously it would sometimes complete immediately if all commits waiting to be uploaded were empty, and at other times it would wait for a server roundtrip. It will now always complete immediately. (Core 14.10.3)
+* When a property is remapped, calling `.Filter("... SORT/DISTINCT(mapped-to-name)")` with the internal name could throw an error like `Cannot sort on key path 'NAME': property 'PersonObject.NAME' does not exist`. (Core 14.10.4)
+* Sync client can crash if a session is resumed while the session is being suspended. (Core 14.11.0)
+* If a sync session is interrupted by a disconnect or restart while downloading a bootstrap, stale data from the previous bootstrap may be included when the session reconnects and downloads the bootstrap. This can lead to objects stored in the database that do not match the actual state of the server and potentially leading to compensating writes. (Core 14.11.0)
+* Fixed unnecessary server roundtrips when there is no download to acknowledge. (Core 14.11.0)
+
 
 ### Compatibility
 * Realm Studio: 15.0.0 or later.
 
 ### Internal
-* Using Core 14.9.0.
+* Using Core v14.10.3.
 
 ## 12.2.0 (2024-05-22)
 
 ### Enhancements
+* Improved calculation of `SyncProgress.ProgressEstimate`. (Issue [#3580](https://github.com/realm/realm-dotnet/issues/3580]))
 * Added support for `Migration.FindInNewRealm` which is a helper that allows you to lookup the object in the post-migration Realm that corresponds to an object from the pre-migration Realm. (Issue [#3600](https://github.com/realm/realm-dotnet/issues/3600))
 * Added `[System.Reflection.Obfuscation]` on the generated `RealmSchema` field to improve compatibility with obfuscation tools that change field and property names of generated classes. (Issue [#3574](https://github.com/realm/realm-dotnet/issues/3574))
 * Added support for list and dictionaries of `RealmValue` (`IList<RealmValue>` and `IDictionary<string, RealmValue>`) to be contained in a `RealmValue`. Lists and dictionaries can contain an arbitrary number of collections themselves. It is possible to convert an existing collection to a `RealmValue` using the new static methods `RealmValue.List` and `RealmValue.Dictionary` or using the implicit operators if converting from common types like `List`, `RealmValue[]` or `Dictionary`. Finally, it is possible to obtain the contained collections by using the new conversion method `AsList` and `AsDictionary`. For example:

Realm/AssemblyInfo.props

@@ -1,7 +1,7 @@
 <Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <PropertyGroup>
     <Product Condition="'$(Product)' == ''">Realm .NET</Product>
-    <VersionPrefix>12.2.0</VersionPrefix>
+    <VersionPrefix>12.3.0</VersionPrefix>
     <Description Condition="'$(Description)' == ''">Realm is a mobile database: a replacement for SQLite</Description>
     <Company>Realm Inc.</Company>
     <Copyright>Copyright © $([System.DateTime]::Now.ToString(yyyy)) Realm Inc.</Copyright>

Realm/Realm.Unity/package.json

@@ -1,6 +1,6 @@
 {
   "name": "io.realm.unity",
-  "version": "12.2.0",
+  "version": "12.3.0",
   "displayName": "Realm",
   "description": "Realm is an embedded, object-oriented database that lets you build real-time, always-on applications. With Realm, data is directly exposed as objects and queryable by code, removing the need for ORM's riddled with performance & maintenance issues. Additionally, objects and collections in Realm are always live, meaning that they always reflect the latest data stored in the database. You can subscribe to changes, letting you keep your UI consistently up to date.\nThe .NET Realm SDK also provide access to Atlas App Services, a secure backend that can sync data between devices, authenticate and manage users, and run serverless JavaScript functions.",
   "unity": "2021.1",

Realm/Realm.UnityUtils/Initializer.cs

@@ -37,7 +37,7 @@ public static void Initialize()
                 Platform.DeviceInfo = new UnityDeviceInfo();
                 Platform.BundleId = Application.productName;
                 InteropConfig.AddPotentialStorageFolder(FileHelper.GetStorageFolder());
-                Realms.Logging.Logger.Console = new UnityLogger();
+                Realms.Logging.RealmLogger.Console = new UnityLogger();
                 Application.quitting += () =>
                 {
                     NativeCommon.CleanupNativeResources("Application is exiting");

Realm/Realm.UnityUtils/UnityLogger.cs

@@ -20,11 +20,14 @@
 
 namespace UnityUtils
 {
-    public class UnityLogger : Logger
+    /// <summary>
+    /// A <see cref="RealmLogger"/> that outputs messages via UnityEngine.
+    /// </summary>
+    public class UnityLogger : RealmLogger
     {
-        protected override void LogImpl(LogLevel level, string message)
+        protected override void LogImpl(LogLevel level, LogCategory category, string message)
         {
-            var toLog = FormatLog(level, message);
+            var toLog = FormatLog(level, category!, message);
             switch (level)
             {
                 case LogLevel.Fatal: