diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/CrossPartition/HybridSearch/HybridSearchCrossPartitionQueryPipelineStage.cs b/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/CrossPartition/HybridSearch/HybridSearchCrossPartitionQueryPipelineStage.cs index 7b58adbaf5..8ef594fcea 100644 --- a/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/CrossPartition/HybridSearch/HybridSearchCrossPartitionQueryPipelineStage.cs +++ b/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/CrossPartition/HybridSearch/HybridSearchCrossPartitionQueryPipelineStage.cs @@ -118,9 +118,6 @@ TryCatch ComponentPipelineFactory(QueryInfo rewrittenQueryI { QueryExecutionOptions queryExecutionOptions = new QueryExecutionOptions(pageSizeHint: maxItemCount); - // TODO: Remove this once the FullTextWordCount is fixed in the backend - queryInfo.GlobalStatisticsQuery = queryInfo.GlobalStatisticsQuery.Replace("_FullTextWordCount", "_FullText_WordCount"); - SqlQuerySpec globalStatisticsQuerySpec = new SqlQuerySpec( queryInfo.GlobalStatisticsQuery, sqlQuerySpec.Parameters); @@ -293,6 +290,7 @@ private async ValueTask MoveNextAsync_DrainSingletonComponentAsync(ITrace foreach (CosmosElement cosmosElement in page.Documents) { HybridSearchQueryResult hybridSearchQueryResult = HybridSearchQueryResult.Create(cosmosElement); + HybridSearchDebugTraceHelpers.TraceQueryResult(hybridSearchQueryResult); documents.Add(hybridSearchQueryResult.Payload); } @@ -877,7 +875,7 @@ private static class Placeholders private static class HybridSearchDebugTraceHelpers { - private const bool Enabled = true; + private const bool Enabled = false; #pragma warning disable CS0162 // Unreachable code detected [Conditional("DEBUG")] @@ -926,6 +924,18 @@ public static void TraceQueryResultsWithRanks(IReadOnlyList documents = documentsArray.Select(document => document.ToString()); - await this.CreateIngestQueryDeleteAsync( - connectionModes: ConnectionModes.Direct, // | ConnectionModes.Gateway, - collectionTypes: CollectionTypes.MultiPartition, // | CollectionTypes.SinglePartition, - documents: documents, - query: RunSanityTests, + await this.CreateIngestQueryDeleteAsync( + connectionModes: ConnectionModes.Direct, // | ConnectionModes.Gateway, + collectionTypes: CollectionTypes.MultiPartition, // | CollectionTypes.SinglePartition, + documents: documents, + query: RunSanityTests, indexingPolicy: CompositeIndexPolicy); } @@ -43,50 +43,56 @@ private static async Task RunSanityTests(Container container, IReadOnlyList{ 2, 57, 85 }), + new List>{ new List{ 2, 57, 85 }, new List{ 2, 85, 57 } }), MakeSanityTest(@" SELECT TOP 10 c.index AS Index, c.title AS Title, c.text AS Text FROM c WHERE FullTextContains(c.title, 'John') OR FullTextContains(c.text, 'John') ORDER BY RANK FullTextScore(c.title, ['John'])", - new List{ 2, 57, 85 }), + new List>{ new List{ 2, 57, 85 }, new List{ 2, 85, 57 } }), MakeSanityTest(@" SELECT c.index AS Index, c.title AS Title, c.text AS Text FROM c WHERE FullTextContains(c.title, 'John') OR FullTextContains(c.text, 'John') ORDER BY RANK FullTextScore(c.title, ['John']) OFFSET 1 LIMIT 5", - new List{ 57, 85 }), + new List>{ new List{ 57, 85 }, new List{ 85, 57 } }), MakeSanityTest(@" SELECT c.index AS Index, c.title AS Title, c.text AS Text FROM c WHERE FullTextContains(c.title, 'John') OR FullTextContains(c.text, 'John') OR FullTextContains(c.text, 'United States') ORDER BY RANK RRF(FullTextScore(c.title, ['John']), FullTextScore(c.text, ['United States']))", - new List{ 61, 51, 49, 54, 75, 24, 77, 76, 80, 25, 22, 2, 66, 57, 85 }), + new List>{ + new List{ 61, 51, 49, 54, 75, 24, 77, 76, 80, 25, 22, 2, 66, 57, 85 }, + new List{ 61, 51, 49, 54, 75, 24, 77, 76, 80, 25, 22, 2, 66, 85, 57 }, + }), MakeSanityTest(@" SELECT TOP 10 c.index AS Index, c.title AS Title, c.text AS Text FROM c WHERE FullTextContains(c.title, 'John') OR FullTextContains(c.text, 'John') OR FullTextContains(c.text, 'United States') ORDER BY RANK RRF(FullTextScore(c.title, ['John']), FullTextScore(c.text, ['United States']))", - new List{ 61, 51, 49, 54, 75, 24, 77, 76, 80, 25 }), + new List>{ new List{ 61, 51, 49, 54, 75, 24, 77, 76, 80, 25 } }), MakeSanityTest(@" SELECT c.index AS Index, c.title AS Title, c.text AS Text FROM c WHERE FullTextContains(c.title, 'John') OR FullTextContains(c.text, 'John') OR FullTextContains(c.text, 'United States') ORDER BY RANK RRF(FullTextScore(c.title, ['John']), FullTextScore(c.text, ['United States'])) OFFSET 5 LIMIT 10", - new List{ 24, 77, 76, 80, 25, 22, 2, 66, 57, 85 }), + new List>{ + new List{ 24, 77, 76, 80, 25, 22, 2, 66, 57, 85 }, + new List{ 24, 77, 76, 80, 25, 22, 2, 66, 85, 57 }, + }), MakeSanityTest(@" SELECT TOP 10 c.index AS Index, c.title AS Title, c.text AS Text FROM c ORDER BY RANK RRF(FullTextScore(c.title, ['John']), FullTextScore(c.text, ['United States']))", - new List{ 61, 51, 49, 54, 75, 24, 77, 76, 80, 25 }), + new List>{new List{ 61, 51, 49, 54, 75, 24, 77, 76, 80, 25 } }), MakeSanityTest(@" SELECT c.index AS Index, c.title AS Title, c.text AS Text FROM c ORDER BY RANK RRF(FullTextScore(c.title, ['John']), FullTextScore(c.text, ['United States'])) OFFSET 0 LIMIT 13", - new List{ 61, 51, 49, 54, 75, 24, 77, 76, 80, 25, 22, 2, 66 }), + new List>{ new List{ 61, 51, 49, 54, 75, 24, 77, 76, 80, 25, 22, 2, 66 } }), }; foreach (SanityTestCase testCase in testCases) @@ -98,12 +104,27 @@ ORDER BY RANK RRF(FullTextScore(c.title, ['John']), FullTextScore(c.text, ['Unit queryDrainingMode: QueryDrainingMode.HoldState); IEnumerable actual = result.Select(document => document.Index); - if (!testCase.ExpectedIndices.SequenceEqual(actual)) + + bool match = false; + foreach (IReadOnlyList expectedIndices in testCase.ExpectedIndices) + { + if (expectedIndices.SequenceEqual(actual)) + { + match = true; + break; + } + } + + if (!match) { Trace.WriteLine($"Query: {testCase.Query}"); - Trace.WriteLine($"Expected: {string.Join(", ", testCase.ExpectedIndices)}"); Trace.WriteLine($"Actual: {string.Join(", ", actual)}"); - Assert.Fail("The query results did not match the expected results."); + + string errorMessage = @"The query results did not match any of the expected results." + + "Please set HybridSearchCrossPartitionQueryPipelineStage.HybridSearchDebugTraceHelpers.Enabled = true to debug." + + "Usually, the failure may be due to some swaps in the results that have equal scores. You can see this in the debug output." + + "The solution is to add another expected result that matches the actual results (provided the scores are in decresing order)."; + Assert.Fail(errorMessage); } } } @@ -119,21 +140,21 @@ private static async Task LoadDocuments() return items; } - private static IndexingPolicy CreateIndexingPolicy() - { - IndexingPolicy policy = new IndexingPolicy(); - - policy.IncludedPaths.Add(new IncludedPath { Path = IndexingPolicy.DefaultPath }); - policy.CompositeIndexes.Add(new Collection - { - new CompositePath { Path = $"/index" }, - new CompositePath { Path = $"/mixedTypefield" }, - }); - - return policy; + private static IndexingPolicy CreateIndexingPolicy() + { + IndexingPolicy policy = new IndexingPolicy(); + + policy.IncludedPaths.Add(new IncludedPath { Path = IndexingPolicy.DefaultPath }); + policy.CompositeIndexes.Add(new Collection + { + new CompositePath { Path = $"/index" }, + new CompositePath { Path = $"/mixedTypefield" }, + }); + + return policy; } - private static SanityTestCase MakeSanityTest(string query, IReadOnlyList expectedIndices) + private static SanityTestCase MakeSanityTest(string query, IReadOnlyList> expectedIndices) { return new SanityTestCase { @@ -146,7 +167,7 @@ private sealed class SanityTestCase { public string Query { get; init; } - public IReadOnlyList ExpectedIndices { get; init; } + public IReadOnlyList> ExpectedIndices { get; init; } } private sealed class TextDocument @@ -166,5 +187,5 @@ private static class FieldNames public const string Text = "text"; public const string Rid = "_rid"; } - } + } } diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Query/Pipeline/NonStreamingOrderByQueryTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Query/Pipeline/NonStreamingOrderByQueryTests.cs index 2ee65e13ab..8a55546f30 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Query/Pipeline/NonStreamingOrderByQueryTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Query/Pipeline/NonStreamingOrderByQueryTests.cs @@ -24,6 +24,9 @@ namespace Microsoft.Azure.Cosmos.Tests.Query.Pipeline using System; using System.Linq; using Microsoft.Azure.Cosmos.CosmosElements.Numbers; + using Microsoft.Azure.Cosmos.Query.Core.QueryPlan; + using Microsoft.Azure.Cosmos.Query.Core.QueryClient; + using Microsoft.Azure.Cosmos.Query.Core.Pipeline.Distinct; [TestClass] public class NonStreamingOrderByQueryTests @@ -48,8 +51,12 @@ public class NonStreamingOrderByQueryTests private const string Payload = "payload"; + private const string ComponentScores = "componentScores"; + private const string Item = "item"; + private const string Text = "text"; + private const string Index = "index"; private const string IndexString = "indexString"; @@ -241,6 +248,75 @@ FROM c await RunParityTests(testCases); } + [TestMethod] + public async Task HybridSearchTests() + { + IReadOnlyList ranges = new List + { + new FeedRangeEpk(new Documents.Routing.Range(string.Empty, "AA", true, false)), + new FeedRangeEpk(new Documents.Routing.Range("AA", "BB", true, false)), + new FeedRangeEpk(new Documents.Routing.Range("BB", "CC", true, false)), + new FeedRangeEpk(new Documents.Routing.Range("CC", "DD", true, false)), + new FeedRangeEpk(new Documents.Routing.Range("DD", "EE", true, false)), + new FeedRangeEpk(new Documents.Routing.Range("EE", "FF", true, false)), + }; + + // TODO: parameterize this test by pulling out the constants as parameters + // e.g. leafPageCount, backendPageSize, pageSize, take + // we can easily add a skip parameter to simulate offset/limit + // Similarly, we can increase/decrease the number of component queries + // Only marginally more involved is to add validation for the global statistics query: + // Make the MockDocumentContainer recognize when it receives the statistics query and return a simple hard coded result + // Finally, it would be cool if we could make the scores independent of each other + + int feedRangeCount = ranges.Count; + int leafPageCount = 4; + int backendPageSize = 10; + + int documentCount = feedRangeCount * leafPageCount * backendPageSize; + + int take = 100; + + IEnumerable expectedIndices = Enumerable + .Range(0, documentCount) + .Reverse() + .Take(take); + + MockDocumentContainer nonStreamingDocumentContainer = MockDocumentContainer.Create( + ranges, + PartitionedFeedMode.NonStreamingReversed, + componentCount: 2, + leafPageCount: leafPageCount, + backendPageSize: backendPageSize); + + (IReadOnlyList results, double requestCharge) = await CreateAndRunHybridSearchQueryPipelineStage( + documentContainer: nonStreamingDocumentContainer, + ranges: ranges, + pageSize: 1000, + take: take); + + Assert.AreEqual(expectedIndices.Count(), results.Count); + + List actual = new List(results.Count); + foreach (CosmosElement result in results) + { + CosmosObject cosmosObject = result as CosmosObject; + CosmosNumber cosmosNumber = cosmosObject[Index] as CosmosNumber; + Assert.IsTrue(cosmosNumber != null && cosmosNumber.Value.IsInteger); + actual.Add((int)Number64.ToLong(cosmosNumber.Value)); + } + + if (!expectedIndices.SequenceEqual(actual)) + { + System.Diagnostics.Trace.WriteLine("Mismatch in query results"); + System.Diagnostics.Trace.WriteLine($"Expected: {string.Join(", ", expectedIndices)}"); + System.Diagnostics.Trace.WriteLine($"Actual: {string.Join(", ", actual)}"); + Assert.Fail(); + } + + Assert.AreEqual(nonStreamingDocumentContainer.TotalRequestCharge, requestCharge); + } + private static Task RunParityTests( IDocumentContainer documentContainer, IDocumentContainer nonStreamingDocumentContainer, @@ -295,6 +371,30 @@ private static async Task RunParityTests( } } + private static Task<(IReadOnlyList, double)> CreateAndRunHybridSearchQueryPipelineStage( + IDocumentContainer documentContainer, + IReadOnlyList ranges, + int pageSize, + int take) + { + TryCatch tryCreatePipeline = PipelineFactory.MonadicCreate( + documentContainer, + Create2ItemSqlQuerySpec(), + ranges, + partitionKey: null, + queryInfo: null, + Create2ItemHybridSearchQueryInfo(requiresGlobalStatistics: false, take), + maxItemCount: pageSize, + new ContainerQueryProperties(), + ranges, + isContinuationExpected: true, + maxConcurrency: MaxConcurrency, + requestContinuationToken: null); + + Assert.IsTrue(tryCreatePipeline.Succeeded); + return RunPipelineStage(tryCreatePipeline.Result, pageSize); + } + private static Task<(IReadOnlyList, double)> CreateAndRunPipelineStage( IDocumentContainer documentContainer, IReadOnlyList ranges, @@ -313,7 +413,7 @@ private static async Task RunParityTests( MaxConcurrency); } - private static async Task<(IReadOnlyList, double)> CreateAndRunPipelineStage( + private static Task<(IReadOnlyList, double)> CreateAndRunPipelineStage( IDocumentContainer documentContainer, IReadOnlyList ranges, string queryText, @@ -337,8 +437,12 @@ private static async Task RunParityTests( Assert.IsTrue(pipelineStage.Succeeded); + return RunPipelineStage(pipelineStage.Result, pageSize); + } + + private static async Task<(IReadOnlyList, double)> RunPipelineStage(IQueryPipelineStage stage, int pageSize) + { double totalRequestCharge = 0; - IQueryPipelineStage stage = pipelineStage.Result; List documents = new List(); while (await stage.MoveNextAsync(NoOpTrace.Singleton, default)) { @@ -806,32 +910,75 @@ public static void TracePage(QueryPage page) private class MockDocumentContainer : IDocumentContainer { - private readonly IReadOnlyDictionary>> pages; + private readonly IReadOnlyList>>> pages; private readonly bool streaming; - public double TotalRequestCharge { get; } + private readonly Func componentSelector; + + private readonly double totalRequestCharge; + + private int queryCount; + + public double TotalRequestCharge + { + get + { + if (this.totalRequestCharge > 0) + { + return this.totalRequestCharge; + } + else + { + int queryCount = Interlocked.CompareExchange(ref this.queryCount, 0, 0); + return queryCount * QueryCharge; + } + } + } + + public static MockDocumentContainer Create( + IReadOnlyList feedRanges, + PartitionedFeedMode feedMode, + int componentCount, + int leafPageCount, + int backendPageSize) + { + IReadOnlyList>>> pages = CreateHybridSearchPartitionedFeed( + componentCount, + feedRanges, + feedMode, + leafPageCount, + backendPageSize); + return new MockDocumentContainer(pages, !feedMode.HasFlag(PartitionedFeedMode.NonStreaming), GetOrderByScoreKind, 0); + } public static MockDocumentContainer Create(IReadOnlyList feedRanges, PartitionedFeedMode feedMode, DocumentCreationMode documentCreationMode) { - IReadOnlyDictionary>> pages = CreatePartitionedFeed( + IReadOnlyList>>> pages = CreatePartitionedFeed( feedRanges, LeafPageCount, PageSize, feedMode, (index) => CreateDocument(index, documentCreationMode)); double totalRequestCharge = feedRanges.Count * LeafPageCount * QueryCharge; - return new MockDocumentContainer(pages, !feedMode.HasFlag(PartitionedFeedMode.NonStreaming), totalRequestCharge); + + return new MockDocumentContainer( + pages, + !feedMode.HasFlag(PartitionedFeedMode.NonStreaming), + _ => 0, + totalRequestCharge); } private MockDocumentContainer( - IReadOnlyDictionary>> pages, + IReadOnlyList>>> pages, bool streaming, + Func componentSelector, double totalRequestCharge) { this.pages = pages ?? throw new ArgumentNullException(nameof(pages)); this.streaming = streaming; - this.TotalRequestCharge = totalRequestCharge; + this.componentSelector = componentSelector; + this.totalRequestCharge = totalRequestCharge; } public Task ChangeFeedAsync(FeedRangeState feedRangeState, ChangeFeedExecutionOptions changeFeedPaginationOptions, ITrace trace, CancellationToken cancellationToken) @@ -851,7 +998,7 @@ public Task> GetChildRangeAsync(FeedRangeInternal feedRange, public Task> GetFeedRangesAsync(ITrace trace, CancellationToken cancellationToken) { - return Task.FromResult(this.pages.Keys.Cast().ToList()); + return Task.FromResult(this.pages[0].Keys.Cast().ToList()); } public Task GetResourceIdentifierAsync(ITrace trace, CancellationToken cancellationToken) @@ -881,7 +1028,7 @@ public Task>> MonadicGetChildRangeAsync(FeedRangeInt public Task>> MonadicGetFeedRangesAsync(ITrace trace, CancellationToken cancellationToken) { - return Task.FromResult(TryCatch>.FromResult(this.pages.Keys.Cast().ToList())); + return Task.FromResult(TryCatch>.FromResult(this.pages[0].Keys.Cast().ToList())); } public Task> MonadicGetResourceIdentifierAsync(ITrace trace, CancellationToken cancellationToken) @@ -896,7 +1043,8 @@ public Task MonadicMergeAsync(FeedRangeInternal feedRange1, FeedRangeI public Task> MonadicQueryAsync(SqlQuerySpec sqlQuerySpec, FeedRangeState feedRangeState, QueryExecutionOptions queryPaginationOptions, ITrace trace, CancellationToken cancellationToken) { - IReadOnlyList> feedRangePages = this.pages[feedRangeState.FeedRange]; + int componentIndex = this.componentSelector(sqlQuerySpec); + IReadOnlyList> feedRangePages = this.pages[componentIndex][feedRangeState.FeedRange]; int index = feedRangeState.State == null ? 0 : int.Parse(((CosmosString)feedRangeState.State.Value).Value); IReadOnlyList documents = feedRangePages[index]; @@ -912,6 +1060,9 @@ public Task> MonadicQueryAsync(SqlQuerySpec sqlQuerySpec, Fe state: state, streaming: this.streaming); + DebugTraceHelpers.TraceBackendResponse(queryPage); + Interlocked.Increment(ref this.queryCount); + return Task.FromResult(TryCatch.FromResult(queryPage)); } @@ -973,12 +1124,79 @@ enum PartitionedFeedMode NonStreamingReversed = NonStreaming | Reversed, } - private static IReadOnlyDictionary>> CreatePartitionedFeed( + private static int GetOrderByScoreKind(SqlQuerySpec sqlQuerySpec) + { + string queryText = sqlQuerySpec.QueryText; + if (queryText.Contains(@"ORDER BY _FullTextScore(c.text")) + { + return 0; + } + else if (queryText.Contains(@"ORDER BY _FullTextScore(c.abstract")) + { + return 1; + } + else if (queryText.Contains(@"ORDER BY _FullTextScore(c.image")) + { + return 2; + } + else + { + throw new ArgumentException("Unknown query text"); + } + } + + private static IReadOnlyList>>> CreatePartitionedFeed( IReadOnlyList feedRanges, int leafPageCount, int pageSize, PartitionedFeedMode mode, Func createDocument) + { + IReadOnlyDictionary>> pages = CreatePartitionedFeed( + feedRanges, + leafPageCount, + pageSize, + mode, + componentIndex: 0, + (_, index) => createDocument(index)); + + return new List>>> + { + pages + }; + } + + private static IReadOnlyList>>> CreateHybridSearchPartitionedFeed( + int componentCount, + IReadOnlyList feedRanges, + PartitionedFeedMode feedMode, + int leafPageCount, + int pageSize) + { + List>>> componentPages = new List>>>(componentCount); + for (int componentIndex = 0; componentIndex < componentCount; ++componentIndex) + { + IReadOnlyDictionary>> pages = CreatePartitionedFeed( + feedRanges, + leafPageCount, + pageSize, + feedMode, + componentIndex, + (componentIndex, index) => CreateHybridSearchDocument(componentCount, index, componentIndex)); + + componentPages.Add(pages); + } + + return componentPages; + } + + private static IReadOnlyDictionary>> CreatePartitionedFeed( + IReadOnlyList feedRanges, + int leafPageCount, + int pageSize, + PartitionedFeedMode mode, + int componentIndex, + Func createDocument) { int feedRangeIndex = 0; Dictionary>> pages = new Dictionary>>(); @@ -991,7 +1209,7 @@ private static IReadOnlyDictionary documents = new List(pageSize); for (int documentCount = 0; documentCount < pageSize; ++documentCount) { - documents.Add(createDocument(index)); + documents.Add(createDocument(componentIndex, index)); index += feedRanges.Count; } @@ -1030,6 +1248,51 @@ enum DocumentCreationMode MultiItemSwapped = MultiItem | Swapped, } + private static CosmosElement CreateHybridSearchDocument(int componentCount, int index, int componentIndex) + { + CosmosElement indexElement = CosmosNumber64.Create(index); + CosmosElement indexStringElement = CosmosString.Create(index.ToString("D4")); + + double[] scores = new double[componentCount]; + double delta = 0.1; + for (int scoreIndex = 0; scoreIndex < componentCount; ++scoreIndex) + { + scores[scoreIndex] = index + ((1 + scoreIndex) * delta); + } + + List orderByItems = new List + { + CosmosObject.Create(new Dictionary + { + [Item] = CosmosNumber64.Create(scores[componentIndex]) + }) + }; + + Dictionary payload = new Dictionary + { + [Payload] = CosmosObject.Create(new Dictionary + { + [Text] = indexStringElement, + [Index] = indexElement, + }), + [ComponentScores] = CosmosArray.Create(scores.Select(score => CosmosNumber64.Create(score))), + }; + + Documents.ResourceId resourceId = Documents.ResourceId.NewCollectionChildResourceId( + CollectionRid, + (ulong)index, + Documents.ResourceType.Document); + + CosmosElement document = CosmosObject.Create(new Dictionary + { + [RId] = CosmosString.Create(resourceId.ToString()), + [OrderByItems] = CosmosArray.Create(orderByItems), + [Payload] = CosmosObject.Create(payload) + }); + + return document; + } + private static CosmosElement CreateDocument(int index, DocumentCreationMode mode) { CosmosElement indexElement = CosmosNumber64.Create(index); @@ -1093,6 +1356,114 @@ private static void FischerYatesShuffle(IList list) } } + private static HybridSearchQueryInfo Create2ItemHybridSearchQueryInfo(bool requiresGlobalStatistics, int? take) + { + return new HybridSearchQueryInfo + { + GlobalStatisticsQuery = @" + SELECT + COUNT(1) AS documentCount, + [ + { + totalWordCount: SUM(_FullTextWordCount(c.text)), + hitCounts: [ + COUNTIF(FullTextContains(c.text, ""swim"")), + COUNTIF(FullTextContains(c.text, ""run"")) + ] + }, + { + totalWordCount: SUM(_FullTextWordCount(c.abstract)), + hitCounts: [ + COUNTIF(FullTextContains(c.abstract, ""energy"")) + ] + } + ] AS fullTextStatistics + FROM c", + + ComponentQueryInfos = new List + { + new QueryInfo + { + DistinctType = DistinctQueryType.None, + Top = 200, + OrderBy = new List{ SortOrder.Descending }, + OrderByExpressions = new List + { + "_FullTextScore(c.text, [\"swim\", \"run\"], {documentdb-formattablehybridsearchquery-totaldocumentcount}, {documentdb-formattablehybridsearchquery-totalwordcount-0}, {documentdb-formattablehybridsearchquery-hitcountsarray-0})", + }, + HasSelectValue = false, + RewrittenQuery = @" + SELECT TOP 200 + c._rid, + [ + { + item: _FullTextScore(c.text, [""swim"", ""run""], {documentdb-formattablehybridsearchquery-totaldocumentcount}, {documentdb-formattablehybridsearchquery-totalwordcount-0}, {documentdb-formattablehybridsearchquery-hitcountsarray-0}) + } + ] AS orderByItems, + { + payload: { + text: c.text, + abstract: c.abstract + }, + componentScores: [ + _FullTextScore(c.text, [""swim"", ""run""], {documentdb-formattablehybridsearchquery-totaldocumentcount}, {documentdb-formattablehybridsearchquery-totalwordcount-0}, {documentdb-formattablehybridsearchquery-hitcountsarray-0}), + _FullTextScore(c.abstract, [""energy""], {documentdb-formattablehybridsearchquery-totaldocumentcount}, {documentdb-formattablehybridsearchquery-totalwordcount-1}, {documentdb-formattablehybridsearchquery-hitcountsarray-1}) + ] + } AS payload + FROM c + WHERE {documentdb-formattableorderbyquery-filter} + ORDER BY _FullTextScore(c.text, [""swim"", ""run""], {documentdb-formattablehybridsearchquery-totaldocumentcount}, {documentdb-formattablehybridsearchquery-totalwordcount-0}, {documentdb-formattablehybridsearchquery-hitcountsarray-0}) DESC", + HasNonStreamingOrderBy = true, + }, + + new QueryInfo + { + DistinctType = DistinctQueryType.None, + Top = 200, + OrderBy = new List{ SortOrder.Descending }, + OrderByExpressions = new List + { + "_FullTextScore(c.abstract, [\"energy\"], {documentdb-formattablehybridsearchquery-totaldocumentcount}, {documentdb-formattablehybridsearchquery-totalwordcount-1}, {documentdb-formattablehybridsearchquery-hitcountsarray-1})", + }, + HasSelectValue = false, + RewrittenQuery = @" + SELECT TOP 200 + c._rid, + [ + { + item: _FullTextScore(c.abstract, [""energy""], {documentdb-formattablehybridsearchquery-totaldocumentcount}, {documentdb-formattablehybridsearchquery-totalwordcount-1}, {documentdb-formattablehybridsearchquery-hitcountsarray-1}) + } + ] AS orderByItems, + { + payload: { + text: c.text, + abstract: c.abstract + }, + componentScores: [ + _FullTextScore(c.text, [""swim"", ""run""], {documentdb-formattablehybridsearchquery-totaldocumentcount}, {documentdb-formattablehybridsearchquery-totalwordcount-0}, {documentdb-formattablehybridsearchquery-hitcountsarray-0}), + _FullTextScore(c.abstract, [""energy""], {documentdb-formattablehybridsearchquery-totaldocumentcount}, {documentdb-formattablehybridsearchquery-totalwordcount-1}, {documentdb-formattablehybridsearchquery-hitcountsarray-1}) + ] + } AS payload + FROM c + WHERE {documentdb-formattableorderbyquery-filter} + ORDER BY _FullTextScore(c.abstract, [""energy""], {documentdb-formattablehybridsearchquery-totaldocumentcount}, {documentdb-formattablehybridsearchquery-totalwordcount-1}, {documentdb-formattablehybridsearchquery-hitcountsarray-1}) DESC", + HasNonStreamingOrderBy = true, + }, + }, + + Take = take, + RequiresGlobalStatistics = requiresGlobalStatistics, + }; + } + + private static SqlQuerySpec Create2ItemSqlQuerySpec() + { + return new SqlQuerySpec(@" + SELECT TOP 100 c.text, c.abstract + FROM c + ORDER BY RANK RRF(FullTextScore(c.text, ['swim', 'run']), FullTextScore(c.abstract, ['energy']))"); + } + private static async Task CreateDocumentContainerAsync(int documentCount) { Documents.PartitionKeyDefinition partitionKeyDefinition = new Documents.PartitionKeyDefinition()