diff --git a/Microsoft.Azure.Cosmos/src/ClientRetryPolicy.cs b/Microsoft.Azure.Cosmos/src/ClientRetryPolicy.cs index 7cef90d541..147977e130 100644 --- a/Microsoft.Azure.Cosmos/src/ClientRetryPolicy.cs +++ b/Microsoft.Azure.Cosmos/src/ClientRetryPolicy.cs @@ -246,9 +246,9 @@ private async Task ShouldRetryInternalAsync( if (statusCode == HttpStatusCode.NotFound && subStatusCode == SubStatusCodes.ReadSessionNotAvailable) { - return this.ShouldRetryOnSessionNotAvailable(); + return this.ShouldRetryOnSessionNotAvailable(this.documentServiceRequest); } - + // Received 503 due to client connect timeout or Gateway if (statusCode == HttpStatusCode.ServiceUnavailable) { @@ -330,7 +330,7 @@ private async Task ShouldRetryOnEndpointFailureAsync( return ShouldRetryResult.RetryAfter(retryDelay); } - private ShouldRetryResult ShouldRetryOnSessionNotAvailable() + private ShouldRetryResult ShouldRetryOnSessionNotAvailable(DocumentServiceRequest request) { this.sessionTokenRetryCount++; @@ -343,7 +343,7 @@ private ShouldRetryResult ShouldRetryOnSessionNotAvailable() { if (this.canUseMultipleWriteLocations) { - ReadOnlyCollection endpoints = this.isReadRequest ? this.globalEndpointManager.ReadEndpoints : this.globalEndpointManager.WriteEndpoints; + ReadOnlyCollection endpoints = this.globalEndpointManager.GetApplicableEndpoints(request, this.isReadRequest); if (this.sessionTokenRetryCount > endpoints.Count) { diff --git a/Microsoft.Azure.Cosmos/src/Handler/RequestMessage.cs b/Microsoft.Azure.Cosmos/src/Handler/RequestMessage.cs index 6748e79921..67d3dbdc27 100644 --- a/Microsoft.Azure.Cosmos/src/Handler/RequestMessage.cs +++ b/Microsoft.Azure.Cosmos/src/Handler/RequestMessage.cs @@ -282,7 +282,8 @@ internal DocumentServiceRequest ToDocumentServiceRequest() { this.DocumentServiceRequest.RouteTo(this.PartitionKeyRangeId); } - + + this.DocumentServiceRequest.RequestContext.ExcludeRegions = this.RequestOptions?.ExcludeRegions; this.OnBeforeRequestHandler(this.DocumentServiceRequest); return this.DocumentServiceRequest; } @@ -299,6 +300,7 @@ private static Headers CreateHeaders() private void OnBeforeRequestHandler(DocumentServiceRequest serviceRequest) { + serviceRequest.RequestContext.ExcludeRegions = this.RequestOptions?.ExcludeRegions; this.OnBeforeSendRequestActions?.Invoke(serviceRequest); } diff --git a/Microsoft.Azure.Cosmos/src/RequestOptions/QueryRequestOptions.cs b/Microsoft.Azure.Cosmos/src/RequestOptions/QueryRequestOptions.cs index 2b4aba5ac0..a4557da6d2 100644 --- a/Microsoft.Azure.Cosmos/src/RequestOptions/QueryRequestOptions.cs +++ b/Microsoft.Azure.Cosmos/src/RequestOptions/QueryRequestOptions.cs @@ -5,7 +5,6 @@ namespace Microsoft.Azure.Cosmos { using System; - using System.ComponentModel; using System.Text; using Microsoft.Azure.Cosmos.CosmosElements; using Microsoft.Azure.Cosmos.Query; diff --git a/Microsoft.Azure.Cosmos/src/RequestOptions/RequestOptions.cs b/Microsoft.Azure.Cosmos/src/RequestOptions/RequestOptions.cs index e8d4c602ff..dd24f3e45e 100644 --- a/Microsoft.Azure.Cosmos/src/RequestOptions/RequestOptions.cs +++ b/Microsoft.Azure.Cosmos/src/RequestOptions/RequestOptions.cs @@ -69,6 +69,13 @@ public class RequestOptions /// public CosmosThresholdOptions CosmosThresholdOptions { get; set; } + /// + /// List of regions to be excluded routing the request to. + /// This can be used to route a request to a specific region by excluding all other regions. + /// If all regions are excluded, then the request will be routed to the primary/hub region. + /// + public List ExcludeRegions { get; set; } + /// /// Gets or sets the boolean to use effective partition key routing in the cosmos db request. /// diff --git a/Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs b/Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs index bcaae809f4..e12d99255d 100644 --- a/Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs +++ b/Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs @@ -362,6 +362,11 @@ public string GetLocation(Uri endpoint) return this.locationCache.GetLocation(endpoint); } + public ReadOnlyCollection GetApplicableEndpoints(DocumentServiceRequest request, bool isReadRequest) + { + return this.locationCache.GetApplicableEndpoints(request, isReadRequest); + } + public bool TryGetLocationForGatewayDiagnostics(Uri endpoint, out string regionName) { return this.locationCache.TryGetLocationForGatewayDiagnostics(endpoint, out regionName); diff --git a/Microsoft.Azure.Cosmos/src/Routing/LocationCache.cs b/Microsoft.Azure.Cosmos/src/Routing/LocationCache.cs index 4480df1826..88a369bd2d 100644 --- a/Microsoft.Azure.Cosmos/src/Routing/LocationCache.cs +++ b/Microsoft.Azure.Cosmos/src/Routing/LocationCache.cs @@ -31,6 +31,7 @@ internal sealed class LocationCache private readonly TimeSpan unavailableLocationsExpirationTime; private readonly int connectionLimit; private readonly ConcurrentDictionary locationUnavailablityInfoByEndpoint; + private readonly RegionNameMapper regionNameMapper; private DatabaseAccountLocationsInfo locationInfo; private DateTime lastCacheUpdateTimestamp; @@ -54,6 +55,7 @@ public LocationCache( this.lastCacheUpdateTimestamp = DateTime.MinValue; this.enableMultipleWriteLocations = false; this.unavailableLocationsExpirationTime = TimeSpan.FromSeconds(LocationCache.DefaultUnavailableLocationsExpirationTimeInSeconds); + this.regionNameMapper = new RegionNameMapper(); #if !(NETSTANDARD15 || NETSTANDARD16) #if NETSTANDARD20 @@ -281,7 +283,7 @@ public Uri ResolveServiceEndpoint(DocumentServiceRequest request) } else { - ReadOnlyCollection endpoints = request.OperationType.IsWriteOperation() ? this.WriteEndpoints : this.ReadEndpoints; + ReadOnlyCollection endpoints = this.GetApplicableEndpoints(request, !request.OperationType.IsWriteOperation()); locationEndpointToRoute = endpoints[locationIndex % endpoints.Count]; } @@ -289,6 +291,64 @@ public Uri ResolveServiceEndpoint(DocumentServiceRequest request) return locationEndpointToRoute; } + public ReadOnlyCollection GetApplicableEndpoints(DocumentServiceRequest request, bool isReadRequest) + { + ReadOnlyCollection endpoints = isReadRequest ? this.ReadEndpoints : this.WriteEndpoints; + + if (request.RequestContext.ExcludeRegions == null || request.RequestContext.ExcludeRegions.Count == 0) + { + return endpoints; + } + + return this.GetApplicableEndpoints( + endpoints, + isReadRequest ? this.locationInfo.AvailableReadEndpointByLocation : this.locationInfo.AvailableWriteEndpointByLocation, + this.defaultEndpoint, + request.RequestContext.ExcludeRegions); + } + + /// + /// Gets applicable endpoints for a request, if there are no applicable endpoints, returns the fallback endpoint + /// + /// + /// + /// + /// + /// a list of applicable endpoints for a request + private ReadOnlyCollection GetApplicableEndpoints( + IReadOnlyList endpoints, + ReadOnlyDictionary regionNameByEndpoint, + Uri fallbackEndpoint, + IReadOnlyList excludeRegions) + { + List applicableEndpoints = new List(endpoints.Count); + HashSet excludeUris = new HashSet(); + + foreach (string region in excludeRegions) + { + string normalizedRegionName = this.regionNameMapper.GetCosmosDBRegionName(region); + if (regionNameByEndpoint.ContainsKey(normalizedRegionName)) + { + excludeUris.Add(regionNameByEndpoint[normalizedRegionName]); + } + } + + foreach (Uri endpoint in endpoints) + { + if (!excludeUris.Contains(endpoint)) + { + applicableEndpoints.Add(endpoint); + } + } + + if (applicableEndpoints.Count == 0) + { + applicableEndpoints.Add(fallbackEndpoint); + } + + return new ReadOnlyCollection(applicableEndpoints); + } + public bool ShouldRefreshEndpoints(out bool canRefreshInBackground) { canRefreshInBackground = true; diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetSDKAPI.json b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetSDKAPI.json index 8c8d7d6ae1..46bdf1c170 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetSDKAPI.json +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetSDKAPI.json @@ -7986,6 +7986,18 @@ "Attributes": [], "MethodInfo": "System.Collections.Generic.IReadOnlyDictionary`2[System.String,System.Object] Properties;CanRead:True;CanWrite:True;System.Collections.Generic.IReadOnlyDictionary`2[System.String,System.Object] get_Properties();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;Void set_Properties(System.Collections.Generic.IReadOnlyDictionary`2[System.String,System.Object]);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, + "System.Collections.Generic.List`1[System.String] ExcludeRegions": { + "Type": "Property", + "Attributes": [], + "MethodInfo": "System.Collections.Generic.List`1[System.String] ExcludeRegions;CanRead:True;CanWrite:True;System.Collections.Generic.List`1[System.String] get_ExcludeRegions();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;Void set_ExcludeRegions(System.Collections.Generic.List`1[System.String]);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Collections.Generic.List`1[System.String] get_ExcludeRegions()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { + "Type": "Method", + "Attributes": [ + "CompilerGeneratedAttribute" + ], + "MethodInfo": "System.Collections.Generic.List`1[System.String] get_ExcludeRegions();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, "System.String get_IfMatchEtag()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { "Type": "Method", "Attributes": [ @@ -8029,6 +8041,13 @@ ], "MethodInfo": "Void set_CosmosThresholdOptions(Microsoft.Azure.Cosmos.CosmosThresholdOptions);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, + "Void set_ExcludeRegions(System.Collections.Generic.List`1[System.String])[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { + "Type": "Method", + "Attributes": [ + "CompilerGeneratedAttribute" + ], + "MethodInfo": "Void set_ExcludeRegions(System.Collections.Generic.List`1[System.String]);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, "Void set_IfMatchEtag(System.String)[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { "Type": "Method", "Attributes": [ diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/LocationCacheTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/LocationCacheTests.cs index b93107b302..06ed0d4026 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/LocationCacheTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/LocationCacheTests.cs @@ -770,11 +770,180 @@ await BackoffRetryUtility.ExecuteAsync( } } + [TestMethod] + [DataRow(true, true, true, DisplayName = "Read request - Multi master - with preferred locations")] + [DataRow(true, true, false, DisplayName = "Read request - Multi master - no preferred locations")] + [DataRow(true, false, true, DisplayName = "Read request - Single master - with preferred locations")] + [DataRow(true, false, false, DisplayName = "Read request - Single master - no preferred locations")] + [DataRow(false, true, true, DisplayName = "Write request - Multi master - with preferred locations")] + [DataRow(false, true, false, DisplayName = "Write request - Multi master - no preferred locations")] + [DataRow(false, false, true, DisplayName = "Write request - Single master - with preferred locations")] + [DataRow(false, false, false, DisplayName = "Write request - Single master - no preferred locations")] + public void VerifyRegionExcludedTest( + bool isReadRequest, + bool useMultipleWriteLocations, + bool usesPreferredLocations) + { + bool enableEndpointDiscovery = true; + + ReadOnlyCollection preferredList = usesPreferredLocations ? + isReadRequest ? + new List { + "location4", + "location2", + "location1" + }.AsReadOnly() : + new List { + "location3", + "location2", + "location1" + }.AsReadOnly() : + isReadRequest ? + new List() { + "default", + "location1", + "location2", + "location4" + }.AsReadOnly() : + new List() { + "default", + "location1", + "location2", + "location3" + }.AsReadOnly(); + + List> excludeRegionCases = isReadRequest ? + new List>() + { + new List {"default"}, new List {"default", "location1"}, new List {"default", "location2"}, new List {"default", "location4"}, + new List {"default", "location1", "location2"}, new List {"default", "location1", "location4"}, new List {"default", "location2", "location4"}, + new List {"default", "location1", "location2", "location4"}, new List { "location1" }, new List {"location1", "location2"}, + new List {"location1", "location4"}, new List {"location1", "location2", "location4"},new List { "location2" }, + new List {"location2", "location4"},new List { "location4" }, + } : + new List>() + { + new List {"default" }, new List {"default", "location1"}, new List {"default", "location2"}, new List {"default", "location3"}, + new List {"default", "location1", "location2"}, new List {"default", "location1", "location3"}, new List {"default", "location2", "location3"}, + new List {"default", "location1", "location2", "location3"}, new List { "location1" }, new List {"location1", "location2"}, + new List {"location1", "location3"}, new List {"location1", "location2", "location3"},new List { "location2" }, + new List {"location2", "location3"},new List { "location3" }, + }; + + foreach (List excludeRegions in excludeRegionCases) + { + using GlobalEndpointManager endpointManager = this.Initialize( + useMultipleWriteLocations: useMultipleWriteLocations, + enableEndpointDiscovery: enableEndpointDiscovery, + isPreferredLocationsListEmpty: false, + preferedRegionListOverride: preferredList, + enforceSingleMasterSingleWriteLocation: true, + isExcludeRegionsTest: true); + + endpointManager.InitializeAccountPropertiesAndStartBackgroundRefresh(this.databaseAccount); + ClientRetryPolicy retryPolicy = this.CreateClientRetryPolicy(enableEndpointDiscovery: true, partitionLevelFailoverEnabled: false, endpointManager: endpointManager); + + using (DocumentServiceRequest request = this.CreateRequest(isReadRequest: isReadRequest, isMasterResourceType: false)) + { + request.RequestContext.ExcludeRegions = excludeRegions; + + ReadOnlyCollection applicableEndpoints = endpointManager.GetApplicableEndpoints(request, isReadRequest); + + Uri endpoint = endpointManager.ResolveServiceEndpoint(request); + ReadOnlyCollection applicableRegions = this.GetApplicableRegions(isReadRequest, useMultipleWriteLocations, usesPreferredLocations, excludeRegions); + + Assert.AreEqual(applicableRegions.Count, applicableEndpoints.Count); + for(int i = 0; i < applicableRegions.Count; i++) + { + Assert.AreEqual(applicableRegions[i], applicableEndpoints[i]); + } + + Assert.AreEqual(applicableRegions[0], endpoint); + } + } + + } + + private ReadOnlyCollection GetApplicableRegions(bool isReadRequest, bool useMultipleWriteLocations, bool usesPreferredLocations, List excludeRegions) + { + if(!isReadRequest && !useMultipleWriteLocations) + { + return new List() {LocationCacheTests.DefaultEndpoint }.AsReadOnly(); + } + + Dictionary readWriteLocations = usesPreferredLocations ? + isReadRequest ? + new Dictionary() + { + {"location4", LocationCacheTests.Location4Endpoint }, + {"location2", LocationCacheTests.Location2Endpoint }, + {"location1", LocationCacheTests.Location1Endpoint }, + } : + useMultipleWriteLocations ? + new Dictionary() + { + {"location3", LocationCacheTests.Location3Endpoint }, + {"location2", LocationCacheTests.Location2Endpoint }, + {"location1", LocationCacheTests.Location1Endpoint }, + } : + new Dictionary() + { + {"default", LocationCacheTests.DefaultEndpoint }, + } : + isReadRequest ? + new Dictionary() + { + {"default", LocationCacheTests.DefaultEndpoint }, + {"location1", LocationCacheTests.Location1Endpoint }, + {"location2", LocationCacheTests.Location2Endpoint }, + {"location4", LocationCacheTests.Location4Endpoint }, + } : + useMultipleWriteLocations ? + new Dictionary() + { + {"default", LocationCacheTests.DefaultEndpoint }, + {"location1", LocationCacheTests.Location1Endpoint }, + {"location2", LocationCacheTests.Location2Endpoint }, + {"location3", LocationCacheTests.Location3Endpoint }, + } : + new Dictionary() + { + {"default", LocationCacheTests.DefaultEndpoint} + }; + + List applicableRegions = new List(); + + foreach (string region in readWriteLocations.Keys) + { + if(!excludeRegions.Contains(region)) + { + applicableRegions.Add(readWriteLocations[region]); + } + } + + if (applicableRegions.Count == 0) + { + applicableRegions.Add(LocationCacheTests.DefaultEndpoint); + } + + return applicableRegions.AsReadOnly(); + } + private static AccountProperties CreateDatabaseAccount( bool useMultipleWriteLocations, - bool enforceSingleMasterSingleWriteLocation) + bool enforceSingleMasterSingleWriteLocation, + bool isExcludeRegionsTest = false) { - Collection writeLocations = new Collection() + Collection writeLocations = isExcludeRegionsTest ? + + new Collection() + { + { new AccountRegion() { Name = "default", Endpoint = LocationCacheTests.DefaultEndpoint.ToString() } }, + { new AccountRegion() { Name = "location1", Endpoint = LocationCacheTests.Location1Endpoint.ToString() } }, + { new AccountRegion() { Name = "location2", Endpoint = LocationCacheTests.Location2Endpoint.ToString() } }, + { new AccountRegion() { Name = "location3", Endpoint = LocationCacheTests.Location3Endpoint.ToString() } }, + } : + new Collection() { { new AccountRegion() { Name = "location1", Endpoint = LocationCacheTests.Location1Endpoint.ToString() } }, { new AccountRegion() { Name = "location2", Endpoint = LocationCacheTests.Location2Endpoint.ToString() } }, @@ -786,21 +955,34 @@ private static AccountProperties CreateDatabaseAccount( { // Some pre-existing tests depend on the account having multiple write locations even on single master setup // Newer tests can correctly define a single master account (single write region) without breaking existing tests - writeLocations = new Collection() - { - { new AccountRegion() { Name = "location1", Endpoint = LocationCacheTests.Location1Endpoint.ToString() } } - }; + writeLocations = isExcludeRegionsTest ? + new Collection() + { + { new AccountRegion() { Name = "default", Endpoint = LocationCacheTests.DefaultEndpoint.ToString() } } + } : + new Collection() + { + { new AccountRegion() { Name = "location1", Endpoint = LocationCacheTests.Location1Endpoint.ToString() } } + } ; } AccountProperties databaseAccount = new AccountProperties() { EnableMultipleWriteLocations = useMultipleWriteLocations, - ReadLocationsInternal = new Collection() - { - { new AccountRegion() { Name = "location1", Endpoint = LocationCacheTests.Location1Endpoint.ToString() } }, - { new AccountRegion() { Name = "location2", Endpoint = LocationCacheTests.Location2Endpoint.ToString() } }, - { new AccountRegion() { Name = "location4", Endpoint = LocationCacheTests.Location4Endpoint.ToString() } }, - }, + ReadLocationsInternal = isExcludeRegionsTest ? + new Collection() + { + { new AccountRegion() { Name = "default", Endpoint = LocationCacheTests.DefaultEndpoint.ToString() } }, + { new AccountRegion() { Name = "location1", Endpoint = LocationCacheTests.Location1Endpoint.ToString() } }, + { new AccountRegion() { Name = "location2", Endpoint = LocationCacheTests.Location2Endpoint.ToString() } }, + { new AccountRegion() { Name = "location4", Endpoint = LocationCacheTests.Location4Endpoint.ToString() } }, + } : + new Collection() + { + { new AccountRegion() { Name = "location1", Endpoint = LocationCacheTests.Location1Endpoint.ToString() } }, + { new AccountRegion() { Name = "location2", Endpoint = LocationCacheTests.Location2Endpoint.ToString() } }, + { new AccountRegion() { Name = "location4", Endpoint = LocationCacheTests.Location4Endpoint.ToString() } }, + }, WriteLocationsInternal = writeLocations }; @@ -813,11 +995,13 @@ private GlobalEndpointManager Initialize( bool isPreferredLocationsListEmpty, bool enforceSingleMasterSingleWriteLocation = false, // Some tests depend on the Initialize to create an account with multiple write locations, even when not multi master ReadOnlyCollection preferedRegionListOverride = null, - bool enablePartitionLevelFailover = false) + bool enablePartitionLevelFailover = false, + bool isExcludeRegionsTest = false) { this.databaseAccount = LocationCacheTests.CreateDatabaseAccount( useMultipleWriteLocations, - enforceSingleMasterSingleWriteLocation); + enforceSingleMasterSingleWriteLocation, + isExcludeRegionsTest); if (isPreferredLocationsListEmpty) { @@ -826,7 +1010,7 @@ private GlobalEndpointManager Initialize( else { // Allow for override at the test method level if needed - this.preferredLocations = preferedRegionListOverride != null ? preferedRegionListOverride : new List() + this.preferredLocations = preferedRegionListOverride ?? new List() { "location1", "location2", @@ -844,8 +1028,8 @@ private GlobalEndpointManager Initialize( this.cache.OnDatabaseAccountRead(this.databaseAccount); this.mockedClient = new Mock(); - mockedClient.Setup(owner => owner.ServiceEndpoint).Returns(LocationCacheTests.DefaultEndpoint); - mockedClient.Setup(owner => owner.GetDatabaseAccountInternalAsync(It.IsAny(), It.IsAny())).ReturnsAsync(this.databaseAccount); + this.mockedClient.Setup(owner => owner.ServiceEndpoint).Returns(LocationCacheTests.DefaultEndpoint); + this.mockedClient.Setup(owner => owner.GetDatabaseAccountInternalAsync(It.IsAny(), It.IsAny())).ReturnsAsync(this.databaseAccount); ConnectionPolicy connectionPolicy = new ConnectionPolicy() { @@ -860,14 +1044,9 @@ private GlobalEndpointManager Initialize( GlobalEndpointManager endpointManager = new GlobalEndpointManager(this.mockedClient.Object, connectionPolicy); - if (enablePartitionLevelFailover) - { - this.partitionKeyRangeLocationCache = new GlobalPartitionEndpointManagerCore(endpointManager); - } - else - { - this.partitionKeyRangeLocationCache = GlobalPartitionEndpointManagerNoOp.Instance; - } + this.partitionKeyRangeLocationCache = enablePartitionLevelFailover + ? new GlobalPartitionEndpointManagerCore(endpointManager) + : GlobalPartitionEndpointManagerNoOp.Instance; return endpointManager; }