From b405feaf8b9cfa4ea37c4dfb92eb4a03c70ce082 Mon Sep 17 00:00:00 2001 From: Nalu Tripician <27316859+NaluTripician@users.noreply.github.com> Date: Tue, 26 Nov 2024 17:48:10 -0500 Subject: [PATCH] Hedging: Adds support for writes on multi region accounts (#4706) # Pull Request Template ## Description Adds support to use hedging on write requests for multimaster accounts. Note that this does come with the caveat that there will be more 409 errors thrown by the SDK. This is expected and applications that adapt this feature should be prepared to handle these exceptions. This feature will also be an opt in feature for Availability Strategies and users will need to especially specify that in the availability strategy definition. ``` ItemRequestOptions requestOptions = new ItemRequestOptions { AvailabilityStrategy = new CrossRegionHedgingAvailabilityStrategy( threshold: TimeSpan.FromMilliseconds(100), thresholdStep: TimeSpan.FromMilliseconds(50), multiMasterWritesEnabled: true) }; ``` ## Type of change Please delete options that are not relevant. - [] New feature (non-breaking change which adds functionality) ## Closing issues To automatically close an issue: closes #4457 --------- Co-authored-by: Fabian Meiswinkel Co-authored-by: Kevin Pilch --- .../src/ClientRetryPolicy.cs | 2 +- Microsoft.Azure.Cosmos/src/DocumentClient.cs | 4 - .../src/Resource/ClientContextCore.cs | 1 - .../AvailabilityStrategy.cs | 12 +- .../CrossRegionHedgingAvailabilityStrategy.cs | 27 +- .../src/Routing/GlobalEndpointManager.cs | 13 +- .../src/Routing/IGlobalEndpointManager.cs | 2 +- .../CosmosAvailabilityStrategyTests.cs | 655 ++++++++++++++---- .../Contracts/DotNetPreviewSDKAPI.json | 4 +- 9 files changed, 556 insertions(+), 164 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/ClientRetryPolicy.cs b/Microsoft.Azure.Cosmos/src/ClientRetryPolicy.cs index 6b3cd3f882..11391ddcbd 100644 --- a/Microsoft.Azure.Cosmos/src/ClientRetryPolicy.cs +++ b/Microsoft.Azure.Cosmos/src/ClientRetryPolicy.cs @@ -193,7 +193,7 @@ public void OnBeforeSendRequest(DocumentServiceRequest request) this.canUseMultipleWriteLocations = this.globalEndpointManager.CanUseMultipleWriteLocations(request); this.documentServiceRequest = request; this.isMultiMasterWriteRequest = !this.isReadRequest - && (this.globalEndpointManager?.CanSupportMultipleWriteLocations(request) ?? false); + && (this.globalEndpointManager?.CanSupportMultipleWriteLocations(request.ResourceType, request.OperationType) ?? false); // clear previous location-based routing directive request.RequestContext.ClearRouteToLocation(); diff --git a/Microsoft.Azure.Cosmos/src/DocumentClient.cs b/Microsoft.Azure.Cosmos/src/DocumentClient.cs index 668513b6b7..5db147fc10 100644 --- a/Microsoft.Azure.Cosmos/src/DocumentClient.cs +++ b/Microsoft.Azure.Cosmos/src/DocumentClient.cs @@ -115,7 +115,6 @@ internal partial class DocumentClient : IDisposable, IAuthorizationTokenProvider private readonly bool IsLocalQuorumConsistency = false; private readonly bool isReplicaAddressValidationEnabled; - private readonly AvailabilityStrategy availabilityStrategy; //Fault Injection private readonly IChaosInterceptorFactory chaosInterceptorFactory; @@ -441,7 +440,6 @@ internal DocumentClient(Uri serviceEndpoint, /// /// This delegate responsible for validating the third party certificate. /// This is distributed tracing flag - /// This is the availability strategy for the client" /// This is the chaos interceptor used for fault injection /// /// The service endpoint can be obtained from the Azure Management Portal. @@ -471,7 +469,6 @@ internal DocumentClient(Uri serviceEndpoint, string cosmosClientId = null, RemoteCertificateValidationCallback remoteCertificateValidationCallback = null, CosmosClientTelemetryOptions cosmosClientTelemetryOptions = null, - AvailabilityStrategy availabilityStrategy = null, IChaosInterceptorFactory chaosInterceptorFactory = null) { if (sendingRequestEventArgs != null) @@ -495,7 +492,6 @@ internal DocumentClient(Uri serviceEndpoint, this.transportClientHandlerFactory = transportClientHandlerFactory; this.IsLocalQuorumConsistency = isLocalQuorumConsistency; this.initTaskCache = new AsyncCacheNonBlocking(cancellationToken: this.cancellationTokenSource.Token); - this.availabilityStrategy = availabilityStrategy; this.chaosInterceptorFactory = chaosInterceptorFactory; this.chaosInterceptor = chaosInterceptorFactory?.CreateInterceptor(this); diff --git a/Microsoft.Azure.Cosmos/src/Resource/ClientContextCore.cs b/Microsoft.Azure.Cosmos/src/Resource/ClientContextCore.cs index 37fe60f89e..cebe0666ed 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/ClientContextCore.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/ClientContextCore.cs @@ -84,7 +84,6 @@ internal static CosmosClientContext Create( cosmosClientId: cosmosClient.Id, remoteCertificateValidationCallback: ClientContextCore.SslCustomValidationCallBack(clientOptions.GetServerCertificateCustomValidationCallback()), cosmosClientTelemetryOptions: clientOptions.CosmosClientTelemetryOptions, - availabilityStrategy: clientOptions.AvailabilityStrategy, chaosInterceptorFactory: clientOptions.ChaosInterceptorFactory); return ClientContextCore.Create( diff --git a/Microsoft.Azure.Cosmos/src/Routing/AvailabilityStrategy/AvailabilityStrategy.cs b/Microsoft.Azure.Cosmos/src/Routing/AvailabilityStrategy/AvailabilityStrategy.cs index 7cb155737c..346226cea3 100644 --- a/Microsoft.Azure.Cosmos/src/Routing/AvailabilityStrategy/AvailabilityStrategy.cs +++ b/Microsoft.Azure.Cosmos/src/Routing/AvailabilityStrategy/AvailabilityStrategy.cs @@ -39,11 +39,17 @@ internal static AvailabilityStrategy DisabledStrategy() /// /// how long before SDK begins hedging /// Period of time between first hedge and next hedging attempts + /// Whether hedging for write requests on accounts with multi-region writes are enabled + /// Note that this does come with the caveat that there will be more 409 / 412 errors thrown by the SDK. + /// This is expected and applications that adopt this feature should be prepared to handle these exceptions. + /// Application might not be able to be deterministic on Create vs Replace in the case of Upsert Operations /// something - public static AvailabilityStrategy CrossRegionHedgingStrategy(TimeSpan threshold, - TimeSpan? thresholdStep) + public static AvailabilityStrategy CrossRegionHedgingStrategy( + TimeSpan threshold, + TimeSpan? thresholdStep, + bool enableMultiWriteRegionHedge = false) { - return new CrossRegionHedgingAvailabilityStrategy(threshold, thresholdStep); + return new CrossRegionHedgingAvailabilityStrategy(threshold, thresholdStep, enableMultiWriteRegionHedge); } } } \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos/src/Routing/AvailabilityStrategy/CrossRegionHedgingAvailabilityStrategy.cs b/Microsoft.Azure.Cosmos/src/Routing/AvailabilityStrategy/CrossRegionHedgingAvailabilityStrategy.cs index a6ed3bfea7..b2358ed897 100644 --- a/Microsoft.Azure.Cosmos/src/Routing/AvailabilityStrategy/CrossRegionHedgingAvailabilityStrategy.cs +++ b/Microsoft.Azure.Cosmos/src/Routing/AvailabilityStrategy/CrossRegionHedgingAvailabilityStrategy.cs @@ -36,14 +36,24 @@ internal class CrossRegionHedgingAvailabilityStrategy : AvailabilityStrategyInte /// public TimeSpan ThresholdStep { get; private set; } + /// + /// Whether hedging for write requests on accounts with multi-region writes is enabled. + /// Note that this does come with the caveat that there will be more 409 / 412 errors thrown by the SDK. + /// This is expected and applications that adopt this feature should be prepared to handle these exceptions. + /// Application might not be able to be deterministic on Create vs Replace in the case of Upsert Operations + /// + public bool EnableMultiWriteRegionHedge { get; private set; } + /// /// Constructor for hedging availability strategy /// /// /// + /// public CrossRegionHedgingAvailabilityStrategy( TimeSpan threshold, - TimeSpan? thresholdStep) + TimeSpan? thresholdStep, + bool enableMultiWriteRegionHedge = false) { if (threshold <= TimeSpan.Zero) { @@ -57,6 +67,7 @@ public CrossRegionHedgingAvailabilityStrategy( this.Threshold = threshold; this.ThresholdStep = thresholdStep ?? TimeSpan.FromMilliseconds(-1); + this.EnableMultiWriteRegionHedge = enableMultiWriteRegionHedge; } /// @@ -70,8 +81,9 @@ internal override bool Enabled() /// This availability strategy can only be used if the request is a read-only request on a document request. /// /// + /// /// whether the request should be a hedging request. - internal bool ShouldHedge(RequestMessage request) + internal bool ShouldHedge(RequestMessage request, CosmosClient client) { //Only use availability strategy for document point operations if (request.ResourceType != ResourceType.Document) @@ -79,9 +91,14 @@ internal bool ShouldHedge(RequestMessage request) return false; } - //check to see if it is a not a read-only request + //check to see if it is a not a read-only request/ if multimaster writes are enabled if (!OperationTypeExtensions.IsReadOperation(request.OperationType)) { + if (this.EnableMultiWriteRegionHedge + && client.DocumentClient.GlobalEndpointManager.CanSupportMultipleWriteLocations(request.ResourceType, request.OperationType)) + { + return true; + } return false; } @@ -102,7 +119,7 @@ internal override async Task ExecuteAvailabilityStrategyAsync( RequestMessage request, CancellationToken cancellationToken) { - if (!this.ShouldHedge(request) + if (!this.ShouldHedge(request, client) || client.DocumentClient.GlobalEndpointManager.ReadEndpoints.Count == 1) { return await sender(request, cancellationToken); @@ -113,7 +130,7 @@ internal override async Task ExecuteAvailabilityStrategyAsync( using (CancellationTokenSource cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken)) { using (CloneableStream clonedBody = (CloneableStream)(request.Content == null - ? null//new CloneableStream(new MemoryStream()) + ? null : await StreamExtension.AsClonableStreamAsync(request.Content))) { IReadOnlyCollection hedgeRegions = client.DocumentClient.GlobalEndpointManager diff --git a/Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs b/Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs index 4cd873c0c7..db3e7dbcb1 100644 --- a/Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs +++ b/Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs @@ -546,14 +546,17 @@ public virtual async Task RefreshLocationAsync(bool forceRefresh = false) /// Determines whether the current configuration and state of the service allow for supporting multiple write locations. /// This method returns True is the AvailableWriteLocations in LocationCache is more than 1. Otherwise, it returns False. /// - /// The document service request for which the write location support is being evaluated. - /// A boolean flag indicating if the available write locations are more than one. - public bool CanSupportMultipleWriteLocations(DocumentServiceRequest request) + /// resource type of the request + /// operation type of the request + /// A boolean flag indicating if the available write locations are more than one. + public bool CanSupportMultipleWriteLocations( + ResourceType resourceType, + OperationType operationType) { return this.locationCache.CanUseMultipleWriteLocations() && this.locationCache.GetAvailableAccountLevelWriteLocations()?.Count > 1 - && (request.ResourceType == ResourceType.Document || - (request.ResourceType == ResourceType.StoredProcedure && request.OperationType == OperationType.Execute)); + && (resourceType == ResourceType.Document || + (resourceType == ResourceType.StoredProcedure && operationType == OperationType.Execute)); } #pragma warning disable VSTHRD100 // Avoid async void methods diff --git a/Microsoft.Azure.Cosmos/src/Routing/IGlobalEndpointManager.cs b/Microsoft.Azure.Cosmos/src/Routing/IGlobalEndpointManager.cs index fe93027aec..0857ac48bb 100644 --- a/Microsoft.Azure.Cosmos/src/Routing/IGlobalEndpointManager.cs +++ b/Microsoft.Azure.Cosmos/src/Routing/IGlobalEndpointManager.cs @@ -37,6 +37,6 @@ internal interface IGlobalEndpointManager : IDisposable ReadOnlyDictionary GetAvailableReadEndpointsByLocation(); - bool CanSupportMultipleWriteLocations(DocumentServiceRequest request); + bool CanSupportMultipleWriteLocations(ResourceType resourceType, OperationType operationType); } } \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosAvailabilityStrategyTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosAvailabilityStrategyTests.cs index 8ce5741509..bfbadc745a 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosAvailabilityStrategyTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosAvailabilityStrategyTests.cs @@ -22,9 +22,6 @@ namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests [TestClass] public class CosmosAvailabilityStrategyTests { - private const string centralUS = "Central US"; - private const string northCentralUS = "North Central US"; - private const string eastUs = "East US"; private const string dbName = "availabilityStrategyTestDb"; private const string containerName = "availabilityStrategyTestContainer"; private const string changeFeedContainerName = "availabilityStrategyTestChangeFeedContainer"; @@ -35,110 +32,32 @@ public class CosmosAvailabilityStrategyTests private Container changeFeedContainer; private CosmosSystemTextJsonSerializer cosmosSystemTextJsonSerializer; private string connectionString; - - [TestCleanup] - public void TestCleanup() - { - //Do not delete the resources, georeplication is slow and we want to reuse the resources - this.client?.Dispose(); - } + private static string region1; + private static string region2; + private static string region3; - private static readonly FaultInjectionCondition readConditon = new FaultInjectionConditionBuilder() - .WithRegion("Central US") - .WithConnectionType(FaultInjectionConnectionType.Direct) - .WithOperationType(FaultInjectionOperationType.ReadItem) - .Build(); - private static readonly FaultInjectionCondition queryConditon = new FaultInjectionConditionBuilder() - .WithRegion("Central US") - .WithConnectionType(FaultInjectionConnectionType.Direct) - .WithOperationType(FaultInjectionOperationType.QueryItem) - .Build(); - private static readonly FaultInjectionCondition readManyCondition = new FaultInjectionConditionBuilder() - .WithRegion("Central US") - .WithConnectionType(FaultInjectionConnectionType.Direct) - .WithOperationType(FaultInjectionOperationType.QueryItem) - .Build(); - private static readonly FaultInjectionCondition changeFeedCondtion = new FaultInjectionConditionBuilder() - .WithRegion("Central US") - .WithConnectionType(FaultInjectionConnectionType.Direct) - .WithOperationType(FaultInjectionOperationType.All) - .Build(); - - private static readonly FaultInjectionCondition readConditonStep = new FaultInjectionConditionBuilder() - .WithRegion("North Central US") - .WithConnectionType(FaultInjectionConnectionType.Direct) - .WithOperationType(FaultInjectionOperationType.ReadItem) - .Build(); - private static readonly FaultInjectionCondition queryConditonStep = new FaultInjectionConditionBuilder() - .WithRegion("North Central US") - .WithConnectionType(FaultInjectionConnectionType.Direct) - .WithOperationType(FaultInjectionOperationType.QueryItem) - .Build(); - private static readonly FaultInjectionCondition readManyConditionStep = new FaultInjectionConditionBuilder() - .WithRegion("North Central US") - .WithConnectionType(FaultInjectionConnectionType.Direct) - .WithOperationType(FaultInjectionOperationType.QueryItem) - .Build(); - private static readonly FaultInjectionCondition changeFeedCondtionStep = new FaultInjectionConditionBuilder() - .WithRegion("North Central US") - .WithConnectionType(FaultInjectionConnectionType.Direct) - .WithOperationType(FaultInjectionOperationType.ReadFeed) - .Build(); - - private static readonly IFaultInjectionResult goneResult = FaultInjectionResultBuilder - .GetResultBuilder(FaultInjectionServerErrorType.Gone) - .Build(); - private static readonly IFaultInjectionResult retryWithResult = FaultInjectionResultBuilder - .GetResultBuilder(FaultInjectionServerErrorType.RetryWith) - .Build(); - private static readonly IFaultInjectionResult internalServerErrorResult = FaultInjectionResultBuilder - .GetResultBuilder(FaultInjectionServerErrorType.InternalServerError) - .Build(); - private static readonly IFaultInjectionResult readSessionNotAvailableResult = FaultInjectionResultBuilder - .GetResultBuilder(FaultInjectionServerErrorType.ReadSessionNotAvailable) - .Build(); - private static readonly IFaultInjectionResult timeoutResult = FaultInjectionResultBuilder - .GetResultBuilder(FaultInjectionServerErrorType.Timeout) - .Build(); - private static readonly IFaultInjectionResult partitionIsSplittingResult = FaultInjectionResultBuilder - .GetResultBuilder(FaultInjectionServerErrorType.PartitionIsSplitting) - .Build(); - private static readonly IFaultInjectionResult partitionIsMigratingResult = FaultInjectionResultBuilder - .GetResultBuilder(FaultInjectionServerErrorType.PartitionIsMigrating) - .Build(); - private static readonly IFaultInjectionResult serviceUnavailableResult = FaultInjectionResultBuilder - .GetResultBuilder(FaultInjectionServerErrorType.ServiceUnavailable) - .Build(); - private static readonly IFaultInjectionResult responseDelayResult = FaultInjectionResultBuilder - .GetResultBuilder(FaultInjectionServerErrorType.ResponseDelay) - .WithDelay(TimeSpan.FromMilliseconds(4000)) - .Build(); - - private readonly Dictionary conditions = new Dictionary() - { - { "Read", readConditon }, - { "Query", queryConditon }, - { "ReadMany", readManyCondition }, - { "ChangeFeed", changeFeedCondtion }, - { "ReadStep", readConditonStep }, - { "QueryStep", queryConditonStep }, - { "ReadManyStep", readManyConditionStep }, - { "ChangeFeedStep", changeFeedCondtionStep} - }; - - private readonly Dictionary results = new Dictionary() - { - { "Gone", goneResult }, - { "RetryWith", retryWithResult }, - { "InternalServerError", internalServerErrorResult }, - { "ReadSessionNotAvailable", readSessionNotAvailableResult }, - { "Timeout", timeoutResult }, - { "PartitionIsSplitting", partitionIsSplittingResult }, - { "PartitionIsMigrating", partitionIsMigratingResult }, - { "ServiceUnavailable", serviceUnavailableResult }, - { "ResponseDelay", responseDelayResult } - }; + private static FaultInjectionCondition readConditon; + private static FaultInjectionCondition queryConditon; + private static FaultInjectionCondition readManyCondition; + private static FaultInjectionCondition changeFeedCondtion; + + private static FaultInjectionCondition readConditonStep; + private static FaultInjectionCondition queryConditonStep; + private static FaultInjectionCondition readManyConditionStep; + private static FaultInjectionCondition changeFeedCondtionStep; + + private static IFaultInjectionResult retryWithResult; + private static IFaultInjectionResult internalServerErrorResult; + private static IFaultInjectionResult readSessionNotAvailableResult; + private static IFaultInjectionResult timeoutResult; + private static IFaultInjectionResult partitionIsSplittingResult; + private static IFaultInjectionResult partitionIsMigratingResult; + private static IFaultInjectionResult serviceUnavailableResult; + private static IFaultInjectionResult responseDelayResult; + + private Dictionary conditions; + private Dictionary results; [TestInitialize] public async Task TestInitAsync() @@ -163,6 +82,120 @@ public async Task TestInitAsync() }); (this.database, this.container, this.changeFeedContainer) = await MultiRegionSetupHelpers.GetOrCreateMultiRegionDatabaseAndContainers(this.client); + + IDictionary readRegions = this.client.DocumentClient.GlobalEndpointManager.GetAvailableReadEndpointsByLocation(); + Assert.IsTrue(readRegions.Count() >= 3); + + region1 = readRegions.Keys.ElementAt(0); + region2 = readRegions.Keys.ElementAt(1); + region3 = readRegions.Keys.ElementAt(2); + + this.CreateRules(); + } + + [TestCleanup] + public void TestCleanup() + { + try + { + this.container.DeleteItemAsync("deleteMe", new PartitionKey("MMWrite")); + } + catch (CosmosException ex) when (ex.StatusCode == HttpStatusCode.NotFound) + { + // Ignore + } + finally + { + //Do not delete the resources (except MM Write test object), georeplication is slow and we want to reuse the resources + this.client?.Dispose(); + } + } + + private void CreateRules() + { + readConditon = new FaultInjectionConditionBuilder() + .WithRegion(region1) + .WithOperationType(FaultInjectionOperationType.ReadItem) + .Build(); + queryConditon = new FaultInjectionConditionBuilder() + .WithRegion(region1) + .WithOperationType(FaultInjectionOperationType.QueryItem) + .Build(); + readManyCondition = new FaultInjectionConditionBuilder() + .WithRegion(region1) + .WithOperationType(FaultInjectionOperationType.QueryItem) + .Build(); + changeFeedCondtion = new FaultInjectionConditionBuilder() + .WithRegion(region1) + .WithOperationType(FaultInjectionOperationType.ReadFeed) + .Build(); + + readConditonStep = new FaultInjectionConditionBuilder() + .WithRegion(region2) + .WithOperationType(FaultInjectionOperationType.ReadItem) + .Build(); + queryConditonStep = new FaultInjectionConditionBuilder() + .WithRegion(region2) + .WithOperationType(FaultInjectionOperationType.QueryItem) + .Build(); + readManyConditionStep = new FaultInjectionConditionBuilder() + .WithRegion(region2) + .WithOperationType(FaultInjectionOperationType.QueryItem) + .Build(); + changeFeedCondtionStep = new FaultInjectionConditionBuilder() + .WithRegion(region2) + .WithOperationType(FaultInjectionOperationType.ReadFeed) + .Build(); + + retryWithResult = FaultInjectionResultBuilder + .GetResultBuilder(FaultInjectionServerErrorType.RetryWith) + .Build(); + internalServerErrorResult = FaultInjectionResultBuilder + .GetResultBuilder(FaultInjectionServerErrorType.InternalServerError) + .Build(); + readSessionNotAvailableResult = FaultInjectionResultBuilder + .GetResultBuilder(FaultInjectionServerErrorType.ReadSessionNotAvailable) + .Build(); + timeoutResult = FaultInjectionResultBuilder + .GetResultBuilder(FaultInjectionServerErrorType.Timeout) + .Build(); + partitionIsSplittingResult = FaultInjectionResultBuilder + .GetResultBuilder(FaultInjectionServerErrorType.PartitionIsSplitting) + .Build(); + partitionIsMigratingResult = FaultInjectionResultBuilder + .GetResultBuilder(FaultInjectionServerErrorType.PartitionIsMigrating) + .Build(); + serviceUnavailableResult = FaultInjectionResultBuilder + .GetResultBuilder(FaultInjectionServerErrorType.ServiceUnavailable) + .Build(); + responseDelayResult = FaultInjectionResultBuilder + .GetResultBuilder(FaultInjectionServerErrorType.ResponseDelay) + .WithDelay(TimeSpan.FromMilliseconds(4000)) + .Build(); + + this.conditions = new Dictionary() + { + { "Read", readConditon }, + { "Query", queryConditon }, + { "ReadMany", readManyCondition }, + { "ChangeFeed", changeFeedCondtion }, + { "ReadStep", readConditonStep }, + { "QueryStep", queryConditonStep }, + { "ReadManyStep", readManyConditionStep }, + { "ChangeFeedStep", changeFeedCondtionStep} + }; + + this.results = new Dictionary() + { + { "RetryWith", retryWithResult }, + { "InternalServerError", internalServerErrorResult }, + { "ReadSessionNotAvailable", readSessionNotAvailableResult }, + { "Timeout", timeoutResult }, + { "PartitionIsSplitting", partitionIsSplittingResult }, + { "PartitionIsMigrating", partitionIsMigratingResult }, + { "ServiceUnavailable", serviceUnavailableResult }, + { "ResponseDelay", responseDelayResult } + }; } [TestMethod] @@ -175,7 +208,7 @@ public async Task AvailabilityStrategyNoTriggerTest(bool isPreferredLocationsEmp id: "responseDely", condition: new FaultInjectionConditionBuilder() - .WithRegion("Central US") + .WithRegion(region1) .WithOperationType(FaultInjectionOperationType.ReadItem) .Build(), result: @@ -189,7 +222,7 @@ public async Task AvailabilityStrategyNoTriggerTest(bool isPreferredLocationsEmp id: "responseDely", condition: new FaultInjectionConditionBuilder() - .WithRegion("North Central US") + .WithRegion(region2) .WithOperationType(FaultInjectionOperationType.ReadItem) .Build(), result: @@ -207,7 +240,7 @@ public async Task AvailabilityStrategyNoTriggerTest(bool isPreferredLocationsEmp CosmosClientOptions clientOptions = new CosmosClientOptions() { ConnectionMode = ConnectionMode.Direct, - ApplicationPreferredRegions = isPreferredLocationsEmpty ? new List() : new List() { "Central US", "North Central US" }, + ApplicationPreferredRegions = isPreferredLocationsEmpty ? new List() : new List() { region1, region2 }, AvailabilityStrategy = AvailabilityStrategy.CrossRegionHedgingStrategy( threshold: TimeSpan.FromMilliseconds(300), thresholdStep: TimeSpan.FromMilliseconds(50)), @@ -228,7 +261,7 @@ public async Task AvailabilityStrategyNoTriggerTest(bool isPreferredLocationsEmp Assert.IsNotNull(traceDiagnostic); traceDiagnostic.Value.Data.TryGetValue("Response Region", out object responseRegion); Assert.IsNotNull(responseRegion); - Assert.AreEqual(CosmosAvailabilityStrategyTests.centralUS, (string)responseRegion); + Assert.AreEqual(region1, (string)responseRegion); //Should send out hedge request but original should be returned traceDiagnostic.Value.Data.TryGetValue("Hedge Context", out object hedgeContext); @@ -239,15 +272,15 @@ public async Task AvailabilityStrategyNoTriggerTest(bool isPreferredLocationsEmp if (isPreferredLocationsEmpty) { Assert.AreEqual(3, hedgeContextList.Count); - Assert.IsTrue(hedgeContextList.Contains(CosmosAvailabilityStrategyTests.centralUS)); - Assert.IsTrue(hedgeContextList.Contains(CosmosAvailabilityStrategyTests.northCentralUS)); - Assert.IsTrue(hedgeContextList.Contains(CosmosAvailabilityStrategyTests.eastUs)); + Assert.IsTrue(hedgeContextList.Contains(region1)); + Assert.IsTrue(hedgeContextList.Contains(region2)); + Assert.IsTrue(hedgeContextList.Contains(region3)); } else { Assert.AreEqual(2, hedgeContextList.Count); - Assert.IsTrue(hedgeContextList.Contains(CosmosAvailabilityStrategyTests.centralUS)); - Assert.IsTrue(hedgeContextList.Contains(CosmosAvailabilityStrategyTests.northCentralUS)); + Assert.IsTrue(hedgeContextList.Contains(region1)); + Assert.IsTrue(hedgeContextList.Contains(region2)); } }; } @@ -262,7 +295,7 @@ public async Task AvailabilityStrategyRequestOptionsTriggerTest(bool isPreferred id: "responseDely", condition: new FaultInjectionConditionBuilder() - .WithRegion("Central US") + .WithRegion(region1) .WithOperationType(FaultInjectionOperationType.ReadItem) .Build(), result: @@ -280,7 +313,7 @@ public async Task AvailabilityStrategyRequestOptionsTriggerTest(bool isPreferred CosmosClientOptions clientOptions = new CosmosClientOptions() { ConnectionMode = ConnectionMode.Direct, - ApplicationPreferredRegions = isPreferredLocationsEmpty? new List() : new List() { "Central US", "North Central US" }, + ApplicationPreferredRegions = isPreferredLocationsEmpty? new List() : new List() { region1, region2 }, Serializer = this.cosmosSystemTextJsonSerializer }; @@ -308,7 +341,7 @@ public async Task AvailabilityStrategyRequestOptionsTriggerTest(bool isPreferred Assert.IsNotNull(traceDiagnostic); traceDiagnostic.Value.Data.TryGetValue("Response Region", out object hedgeContext); Assert.IsNotNull(hedgeContext); - Assert.AreEqual(CosmosAvailabilityStrategyTests.northCentralUS, (string)hedgeContext); + Assert.AreEqual(region2, (string)hedgeContext); } } @@ -322,7 +355,7 @@ public async Task AvailabilityStrategyDisableOverideTest(bool isPreferredLocatio id: "responseDely", condition: new FaultInjectionConditionBuilder() - .WithRegion("Central US") + .WithRegion(region1) .WithOperationType(FaultInjectionOperationType.ReadItem) .Build(), result: @@ -341,7 +374,7 @@ public async Task AvailabilityStrategyDisableOverideTest(bool isPreferredLocatio CosmosClientOptions clientOptions = new CosmosClientOptions() { ConnectionMode = ConnectionMode.Direct, - ApplicationPreferredRegions = isPreferredLocationsEmpty ? new List() : new List() { "Central US", "North Central US" }, + ApplicationPreferredRegions = isPreferredLocationsEmpty ? new List() : new List() { region1, region2 }, AvailabilityStrategy = AvailabilityStrategy.CrossRegionHedgingStrategy( threshold: TimeSpan.FromMilliseconds(100), thresholdStep: TimeSpan.FromMilliseconds(50)), @@ -375,7 +408,6 @@ public async Task AvailabilityStrategyDisableOverideTest(bool isPreferredLocatio [DataTestMethod] [TestCategory("MultiRegion")] - [DataRow("Read", "Read", "Gone", false, DisplayName = "Read | Gone | With Preferred Regions")] [DataRow("Read", "Read", "RetryWith", false, DisplayName = "Read | RetryWith | With Preferred Regions")] [DataRow("Read", "Read", "InternalServerError", false, DisplayName = "Read | InternalServerError | With Preferred Regions")] [DataRow("Read", "Read", "ReadSessionNotAvailable", false, DisplayName = "Read | ReadSessionNotAvailable | With Preferred Regions")] @@ -384,7 +416,6 @@ public async Task AvailabilityStrategyDisableOverideTest(bool isPreferredLocatio [DataRow("Read", "Read", "PartitionIsMigrating", false, DisplayName = "Read | PartitionIsMigrating | With Preferred Regions")] [DataRow("Read", "Read", "ServiceUnavailable", false, DisplayName = "Read | ServiceUnavailable | With Preferred Regions")] [DataRow("Read", "Read", "ResponseDelay", false, DisplayName = "Read | ResponseDelay | With Preferred Regions")] - [DataRow("SinglePartitionQuery", "Query", "Gone", false, DisplayName = "SinglePartitionQuery | Gone | With Preferred Regions")] [DataRow("SinglePartitionQuery", "Query", "RetryWith", false, DisplayName = "SinglePartitionQuery | RetryWith | With Preferred Regions")] [DataRow("SinglePartitionQuery", "Query", "InternalServerError", false, DisplayName = "SinglePartitionQuery | InternalServerError | With Preferred Regions")] [DataRow("SinglePartitionQuery", "Query", "ReadSessionNotAvailable", false, DisplayName = "SinglePartitionQuery | ReadSessionNotAvailable | With Preferred Regions")] @@ -393,7 +424,6 @@ public async Task AvailabilityStrategyDisableOverideTest(bool isPreferredLocatio [DataRow("SinglePartitionQuery", "Query", "PartitionIsMigrating", false, DisplayName = "SinglePartitionQuery | PartitionIsMigrating | With Preferred Regions")] [DataRow("SinglePartitionQuery", "Query", "ServiceUnavailable", false, DisplayName = "SinglePartitionQuery | ServiceUnavailable | With Preferred Regions")] [DataRow("SinglePartitionQuery", "Query", "ResponseDelay", false, DisplayName = "SinglePartitionQuery | ResponseDelay | With Preferred Regions")] - [DataRow("CrossPartitionQuery", "Query", "Gone", false, DisplayName = "CrossPartitionQuery | Gone | With Preferred Regions")] [DataRow("CrossPartitionQuery", "Query", "RetryWith", false, DisplayName = "CrossPartitionQuery | RetryWith | With Preferred Regions")] [DataRow("CrossPartitionQuery", "Query", "InternalServerError", false, DisplayName = "CrossPartitionQuery | InternalServerError | With Preferred Regions")] [DataRow("CrossPartitionQuery", "Query", "ReadSessionNotAvailable", false, DisplayName = "CrossPartitionQuery | ReadSessionNotAvailable | With Preferred Regions")] @@ -402,7 +432,6 @@ public async Task AvailabilityStrategyDisableOverideTest(bool isPreferredLocatio [DataRow("CrossPartitionQuery", "Query", "PartitionIsMigrating", false, DisplayName = "CrossPartitionQuery | PartitionIsMigrating | With Preferred Regions")] [DataRow("CrossPartitionQuery", "Query", "ServiceUnavailable", false, DisplayName = "CrossPartitionQuery | ServiceUnavailable | With Preferred Regions")] [DataRow("CrossPartitionQuery", "Query", "ResponseDelay", false, DisplayName = "CrossPartitionQuery | ResponseDelay | With Preferred Regions")] - [DataRow("ReadMany", "ReadMany", "Gone", false, DisplayName = "ReadMany | Gone | With Preferred Regions")] [DataRow("ReadMany", "ReadMany", "RetryWith", false, DisplayName = "ReadMany | RetryWith | With Preferred Regions")] [DataRow("ReadMany", "ReadMany", "InternalServerError", false, DisplayName = "ReadMany | InternalServerError | With Preferred Regions")] [DataRow("ReadMany", "ReadMany", "ReadSessionNotAvailable", false, DisplayName = "ReadMany | ReadSessionNotAvailable | With Preferred Regions")] @@ -411,7 +440,6 @@ public async Task AvailabilityStrategyDisableOverideTest(bool isPreferredLocatio [DataRow("ReadMany", "ReadMany", "PartitionIsMigrating", false, DisplayName = "ReadMany | PartitionIsMigrating | With Preferred Regions")] [DataRow("ReadMany", "ReadMany", "ServiceUnavailable", false, DisplayName = "ReadMany | ServiceUnavailable | With Preferred Regions")] [DataRow("ReadMany", "ReadMany", "ResponseDelay", false, DisplayName = "ReadMany | ResponseDelay | With Preferred Regions")] - [DataRow("ChangeFeed", "ChangeFeed", "Gone", false, DisplayName = "ChangeFeed | Gone | With Preferred Regions")] [DataRow("ChangeFeed", "ChangeFeed", "RetryWith", false, DisplayName = "ChangeFeed | RetryWith | With Preferred Regions")] [DataRow("ChangeFeed", "ChangeFeed", "InternalServerError", false, DisplayName = "ChangeFeed | InternalServerError | With Preferred Regions")] [DataRow("ChangeFeed", "ChangeFeed", "ReadSessionNotAvailable", false, DisplayName = "ChangeFeed | ReadSessionNotAvailable | With Preferred Regions")] @@ -420,7 +448,6 @@ public async Task AvailabilityStrategyDisableOverideTest(bool isPreferredLocatio [DataRow("ChangeFeed", "ChangeFeed", "PartitionIsMigrating", false, DisplayName = "ChangeFeed | PartitionIsMigrating | With Preferred Regions")] [DataRow("ChangeFeed", "ChangeFeed", "ServiceUnavailable", false, DisplayName = "ChangeFeed | ServiceUnavailable | With Preferred Regions")] [DataRow("ChangeFeed", "ChangeFeed", "ResponseDelay", false, DisplayName = "ChangeFeed | ResponseDelay | With Preferred Regions")] - [DataRow("Read", "Read", "Gone", true, DisplayName = "Read | Gone | W/O Preferred Regions")] [DataRow("Read", "Read", "RetryWith", true, DisplayName = "Read | RetryWith | W/O Preferred Regions")] [DataRow("Read", "Read", "InternalServerError", true, DisplayName = "Read | InternalServerError | W/O Preferred Regions")] [DataRow("Read", "Read", "ReadSessionNotAvailable", true, DisplayName = "Read | ReadSessionNotAvailable | W/O Preferred Regions")] @@ -429,7 +456,6 @@ public async Task AvailabilityStrategyDisableOverideTest(bool isPreferredLocatio [DataRow("Read", "Read", "PartitionIsMigrating", true, DisplayName = "Read | PartitionIsMigrating | W/O Preferred Regions")] [DataRow("Read", "Read", "ServiceUnavailable", true, DisplayName = "Read | ServiceUnavailable | W/O Preferred Regions")] [DataRow("Read", "Read", "ResponseDelay", true, DisplayName = "Read | ResponseDelay | W/O Preferred Regions")] - [DataRow("SinglePartitionQuery", "Query", "Gone", true, DisplayName = "SinglePartitionQuery | Gone | W/O Preferred Regions")] [DataRow("SinglePartitionQuery", "Query", "RetryWith", true, DisplayName = "SinglePartitionQuery | RetryWith | W/O Preferred Regions")] [DataRow("SinglePartitionQuery", "Query", "InternalServerError", true, DisplayName = "SinglePartitionQuery | InternalServerError | W/O Preferred Regions")] [DataRow("SinglePartitionQuery", "Query", "ReadSessionNotAvailable", true, DisplayName = "SinglePartitionQuery | ReadSessionNotAvailable | W/O Preferred Regions")] @@ -438,7 +464,6 @@ public async Task AvailabilityStrategyDisableOverideTest(bool isPreferredLocatio [DataRow("SinglePartitionQuery", "Query", "PartitionIsMigrating", true, DisplayName = "SinglePartitionQuery | PartitionIsMigrating | W/O Preferred Regions")] [DataRow("SinglePartitionQuery", "Query", "ServiceUnavailable", true, DisplayName = "SinglePartitionQuery | ServiceUnavailable | W/O Preferred Regions")] [DataRow("SinglePartitionQuery", "Query", "ResponseDelay", true, DisplayName = "SinglePartitionQuery | ResponseDelay | W/O Preferred Regions")] - [DataRow("CrossPartitionQuery", "Query", "Gone", true, DisplayName = "CrossPartitionQuery | Gone | W/O Preferred Regions")] [DataRow("CrossPartitionQuery", "Query", "RetryWith", true, DisplayName = "CrossPartitionQuery | RetryWith | W/O Preferred Regions")] [DataRow("CrossPartitionQuery", "Query", "InternalServerError", true, DisplayName = "CrossPartitionQuery | InternalServerError | W/O Preferred Regions")] [DataRow("CrossPartitionQuery", "Query", "ReadSessionNotAvailable", true, DisplayName = "CrossPartitionQuery | ReadSessionNotAvailable | W/O Preferred Regions")] @@ -447,7 +472,6 @@ public async Task AvailabilityStrategyDisableOverideTest(bool isPreferredLocatio [DataRow("CrossPartitionQuery", "Query", "PartitionIsMigrating", true, DisplayName = "CrossPartitionQuery | PartitionIsMigrating | W/O Preferred Regions")] [DataRow("CrossPartitionQuery", "Query", "ServiceUnavailable", true, DisplayName = "CrossPartitionQuery | ServiceUnavailable | W/O Preferred Regions")] [DataRow("CrossPartitionQuery", "Query", "ResponseDelay", true, DisplayName = "CrossPartitionQuery | ResponseDelay | W/O Preferred Regions")] - [DataRow("ReadMany", "ReadMany", "Gone", true, DisplayName = "ReadMany | Gone | W/O Preferred Regions")] [DataRow("ReadMany", "ReadMany", "RetryWith", true, DisplayName = "ReadMany | RetryWith | W/O Preferred Regions")] [DataRow("ReadMany", "ReadMany", "InternalServerError", true, DisplayName = "ReadMany | InternalServerError | W/O Preferred Regions")] [DataRow("ReadMany", "ReadMany", "ReadSessionNotAvailable", true, DisplayName = "ReadMany | ReadSessionNotAvailable | W/O Preferred Regions")] @@ -456,7 +480,6 @@ public async Task AvailabilityStrategyDisableOverideTest(bool isPreferredLocatio [DataRow("ReadMany", "ReadMany", "PartitionIsMigrating", true, DisplayName = "ReadMany | PartitionIsMigrating | W/O Preferred Regions")] [DataRow("ReadMany", "ReadMany", "ServiceUnavailable", true, DisplayName = "ReadMany | ServiceUnavailable | W/O Preferred Regions")] [DataRow("ReadMany", "ReadMany", "ResponseDelay", true, DisplayName = "ReadMany | ResponseDelay | W/O Preferred Regions")] - [DataRow("ChangeFeed", "ChangeFeed", "Gone", true, DisplayName = "ChangeFeed | Gone | W/O Preferred Regions")] [DataRow("ChangeFeed", "ChangeFeed", "RetryWith", true, DisplayName = "ChangeFeed | RetryWith | W/O Preferred Regions")] [DataRow("ChangeFeed", "ChangeFeed", "InternalServerError", true, DisplayName = "ChangeFeed | InternalServerError | W/O Preferred Regions")] [DataRow("ChangeFeed", "ChangeFeed", "ReadSessionNotAvailable", true, DisplayName = "ChangeFeed | ReadSessionNotAvailable | W/O Preferred Regions")] @@ -485,7 +508,7 @@ public async Task AvailabilityStrategyAllFaultsTests(string operation, string co CosmosClientOptions clientOptions = new CosmosClientOptions() { ConnectionMode = ConnectionMode.Direct, - ApplicationPreferredRegions = isPreferredLocationsEmpty ? new List() : new List() { "Central US", "North Central US" }, + ApplicationPreferredRegions = isPreferredLocationsEmpty ? new List() :new List() { region1, region2 }, AvailabilityStrategy = AvailabilityStrategy.CrossRegionHedgingStrategy( threshold: TimeSpan.FromMilliseconds(100), thresholdStep: TimeSpan.FromMilliseconds(50)), @@ -524,7 +547,7 @@ public async Task AvailabilityStrategyAllFaultsTests(string operation, string co Assert.IsNotNull(traceDiagnostic); traceDiagnostic.Value.Data.TryGetValue("Response Region", out hedgeContext); Assert.IsNotNull(hedgeContext); - Assert.AreEqual(CosmosAvailabilityStrategyTests.northCentralUS, (string)hedgeContext); + Assert.AreEqual(region2, (string)hedgeContext); break; @@ -556,7 +579,7 @@ public async Task AvailabilityStrategyAllFaultsTests(string operation, string co Assert.IsNotNull(traceDiagnostic); traceDiagnostic.Value.Data.TryGetValue("Response Region", out hedgeContext); Assert.IsNotNull(hedgeContext); - Assert.AreEqual(CosmosAvailabilityStrategyTests.northCentralUS, (string)hedgeContext); + Assert.AreEqual(region2, (string)hedgeContext); } break; @@ -587,7 +610,7 @@ public async Task AvailabilityStrategyAllFaultsTests(string operation, string co Assert.IsNotNull(traceDiagnostic); traceDiagnostic.Value.Data.TryGetValue("Response Region", out hedgeContext); Assert.IsNotNull(hedgeContext); - Assert.AreEqual(CosmosAvailabilityStrategyTests.northCentralUS, (string)hedgeContext); + Assert.AreEqual(region2, (string)hedgeContext); } break; @@ -617,7 +640,7 @@ public async Task AvailabilityStrategyAllFaultsTests(string operation, string co Assert.IsNotNull(traceDiagnostic); traceDiagnostic.Value.Data.TryGetValue("Response Region", out hedgeContext); Assert.IsNotNull(hedgeContext); - Assert.AreEqual(CosmosAvailabilityStrategyTests.northCentralUS, (string)hedgeContext); + Assert.AreEqual(region2, (string)hedgeContext); break; @@ -702,7 +725,7 @@ public async Task AvailabilityStrategyStepTests(string operation, string condito CosmosClientOptions clientOptions = new CosmosClientOptions() { ConnectionMode = ConnectionMode.Direct, - ApplicationPreferredRegions = isPreferredRegionsEmpty ? new List() : new List() { "Central US", "North Central US", "East US" }, + ApplicationPreferredRegions = isPreferredRegionsEmpty ? new List() : new List() { region1, region2, region3 }, AvailabilityStrategy = AvailabilityStrategy.CrossRegionHedgingStrategy( threshold: TimeSpan.FromMilliseconds(100), thresholdStep: TimeSpan.FromMilliseconds(50)), @@ -733,7 +756,7 @@ public async Task AvailabilityStrategyStepTests(string operation, string condito Assert.IsNotNull(traceDiagnostic); traceDiagnostic.Value.Data.TryGetValue("Response Region", out hedgeContext); Assert.IsNotNull(hedgeContext); - Assert.AreEqual(CosmosAvailabilityStrategyTests.eastUs, (string)hedgeContext); + Assert.AreEqual(region3, (string)hedgeContext); break; @@ -760,7 +783,7 @@ public async Task AvailabilityStrategyStepTests(string operation, string condito Assert.IsNotNull(traceDiagnostic); traceDiagnostic.Value.Data.TryGetValue("Response Region", out hedgeContext); Assert.IsNotNull(hedgeContext); - Assert.AreEqual(CosmosAvailabilityStrategyTests.eastUs, (string)hedgeContext); + Assert.AreEqual(region3, (string)hedgeContext); } break; @@ -781,7 +804,7 @@ public async Task AvailabilityStrategyStepTests(string operation, string condito Assert.IsNotNull(traceDiagnostic); traceDiagnostic.Value.Data.TryGetValue("Response Region", out hedgeContext); Assert.IsNotNull(hedgeContext); - Assert.AreEqual(CosmosAvailabilityStrategyTests.eastUs, (string)hedgeContext); + Assert.AreEqual(region3, (string)hedgeContext); } break; @@ -803,7 +826,7 @@ public async Task AvailabilityStrategyStepTests(string operation, string condito Assert.IsNotNull(traceDiagnostic); traceDiagnostic.Value.Data.TryGetValue("Response Region", out hedgeContext); Assert.IsNotNull(hedgeContext); - Assert.AreEqual(CosmosAvailabilityStrategyTests.eastUs, (string)hedgeContext); + Assert.AreEqual(region3, (string)hedgeContext); break; @@ -848,6 +871,354 @@ public async Task AvailabilityStrategyStepTests(string operation, string condito } } + [TestMethod] + [TestCategory("MultiMaster")] + public async Task AvailabilityStrategyMultiMasterWriteBeforeTest() + { + FaultInjectionRule sendDelay = new FaultInjectionRuleBuilder( + id: "sendDelay", + condition: + new FaultInjectionConditionBuilder() + .WithRegion(region1) + .WithOperationType(FaultInjectionOperationType.CreateItem) + .Build(), + result: + FaultInjectionResultBuilder.GetResultBuilder(FaultInjectionServerErrorType.SendDelay) + .WithDelay(TimeSpan.FromMilliseconds(6000)) + .Build()) + .WithDuration(TimeSpan.FromMinutes(90)) + .Build(); + + List rules = new List() { sendDelay }; + FaultInjector faultInjector = new FaultInjector(rules); + + sendDelay.Disable(); + + CosmosClientOptions clientOptions = new CosmosClientOptions() + { + ConnectionMode = ConnectionMode.Direct, + ApplicationPreferredRegions = new List() { region1, region2 }, + Serializer = this.cosmosSystemTextJsonSerializer + }; + + using (CosmosClient faultInjectionClient = new CosmosClient( + connectionString: this.connectionString, + clientOptions: faultInjector.GetFaultInjectionClientOptions(clientOptions))) + { + Database database = faultInjectionClient.GetDatabase(CosmosAvailabilityStrategyTests.dbName); + Container container = database.GetContainer(CosmosAvailabilityStrategyTests.containerName); + + sendDelay.Enable(); + + ItemRequestOptions requestOptions = new ItemRequestOptions + { + AvailabilityStrategy = new CrossRegionHedgingAvailabilityStrategy( + threshold: TimeSpan.FromMilliseconds(100), + thresholdStep: TimeSpan.FromMilliseconds(50), + enableMultiWriteRegionHedge: true) + }; + + AvailabilityStrategyTestObject availabilityStrategyTestObject = new AvailabilityStrategyTestObject + { + Id = "deleteMe", + Pk = "MMWrite", + Other = "test" + }; + + ItemResponse ir = await container.CreateItemAsync( + availabilityStrategyTestObject, + requestOptions: requestOptions); + + sendDelay.Disable(); + + CosmosTraceDiagnostics traceDiagnostic = ir.Diagnostics as CosmosTraceDiagnostics; + Assert.IsNotNull(traceDiagnostic); + traceDiagnostic.Value.Data.TryGetValue("Response Region", out object hedgeContext); + Assert.IsNotNull(hedgeContext); + Assert.AreEqual(region2, (string)hedgeContext); + } + } + + [TestMethod] + [TestCategory("MultiMaster")] + public async Task AvailabilityStrategyMultiMasterWriteAfterTest() + { + FaultInjectionRule responseDelay = new FaultInjectionRuleBuilder( + id: "responseDelay", + condition: + new FaultInjectionConditionBuilder() + .WithRegion(region1) + .WithOperationType(FaultInjectionOperationType.CreateItem) + .Build(), + result: + FaultInjectionResultBuilder.GetResultBuilder(FaultInjectionServerErrorType.ResponseDelay) + .WithDelay(TimeSpan.FromMilliseconds(6000)) + .Build()) + .WithDuration(TimeSpan.FromMinutes(90)) + .Build(); + + List rules = new List() { responseDelay }; + FaultInjector faultInjector = new FaultInjector(rules); + + responseDelay.Disable(); + + CosmosClientOptions clientOptions = new CosmosClientOptions() + { + ConnectionMode = ConnectionMode.Direct, + ApplicationPreferredRegions = new List() { region1, region2 }, + Serializer = this.cosmosSystemTextJsonSerializer + }; + + using (CosmosClient faultInjectionClient = new CosmosClient( + connectionString: this.connectionString, + clientOptions: faultInjector.GetFaultInjectionClientOptions(clientOptions))) + { + Database database = faultInjectionClient.GetDatabase(CosmosAvailabilityStrategyTests.dbName); + Container container = database.GetContainer(CosmosAvailabilityStrategyTests.containerName); + + responseDelay.Enable(); + + ItemRequestOptions requestOptions = new ItemRequestOptions + { + AvailabilityStrategy = new CrossRegionHedgingAvailabilityStrategy( + threshold: TimeSpan.FromMilliseconds(100), + thresholdStep: TimeSpan.FromMilliseconds(50), + enableMultiWriteRegionHedge: true) + }; + + AvailabilityStrategyTestObject availabilityStrategyTestObject = new AvailabilityStrategyTestObject + { + Id = "deleteMe", + Pk = "MMWrite", + Other = "test" + }; + + try + { + ItemResponse ir = await container.CreateItemAsync( + availabilityStrategyTestObject, + requestOptions: requestOptions); + } + catch (CosmosException ex) + { + Assert.AreEqual(HttpStatusCode.Conflict, ex.StatusCode); + + CosmosTraceDiagnostics traceDiagnostic = ex.Diagnostics as CosmosTraceDiagnostics; + Assert.IsNotNull(traceDiagnostic); + traceDiagnostic.Value.Data.TryGetValue("Response Region", out object hedgeContext); + Assert.IsNotNull(hedgeContext); + Assert.AreEqual(region2, (string)hedgeContext); + } + finally + { + responseDelay.Disable(); + } + } + } + + [TestMethod] + [TestCategory("MultiMaster")] + public async Task AvailabilityStrategyMultiMasterWriteBeforeStepTest() + { + FaultInjectionRule sendDelay = new FaultInjectionRuleBuilder( + id: "sendDelay", + condition: + new FaultInjectionConditionBuilder() + .WithRegion(region1) + .WithOperationType(FaultInjectionOperationType.CreateItem) + .Build(), + result: + FaultInjectionResultBuilder.GetResultBuilder(FaultInjectionServerErrorType.SendDelay) + .WithDelay(TimeSpan.FromMilliseconds(6000)) + .Build()) + .WithDuration(TimeSpan.FromMinutes(90)) + .Build(); + + FaultInjectionRule sendDelay2 = new FaultInjectionRuleBuilder( + id: "sendDelay2", + condition: + new FaultInjectionConditionBuilder() + .WithRegion(region2) + .WithOperationType(FaultInjectionOperationType.CreateItem) + .Build(), + result: + FaultInjectionResultBuilder.GetResultBuilder(FaultInjectionServerErrorType.SendDelay) + .WithDelay(TimeSpan.FromMilliseconds(6000)) + .Build()) + .WithDuration(TimeSpan.FromMinutes(90)) + .Build(); + + List rules = new List() { sendDelay, sendDelay2 }; + FaultInjector faultInjector = new FaultInjector(rules); + + sendDelay.Disable(); + sendDelay2.Disable(); + + CosmosClientOptions clientOptions = new CosmosClientOptions() + { + ConnectionMode = ConnectionMode.Direct, + ApplicationPreferredRegions = new List() { region1, region2, region3 }, + Serializer = this.cosmosSystemTextJsonSerializer + }; + + using (CosmosClient faultInjectionClient = new CosmosClient( + connectionString: this.connectionString, + clientOptions: faultInjector.GetFaultInjectionClientOptions(clientOptions))) + { + Database database = faultInjectionClient.GetDatabase(CosmosAvailabilityStrategyTests.dbName); + Container container = database.GetContainer(CosmosAvailabilityStrategyTests.containerName); + + + + ItemRequestOptions requestOptions = new ItemRequestOptions + { + AvailabilityStrategy = new CrossRegionHedgingAvailabilityStrategy( + threshold: TimeSpan.FromMilliseconds(100), + thresholdStep: TimeSpan.FromMilliseconds(50), + enableMultiWriteRegionHedge: true) + }; + + AvailabilityStrategyTestObject availabilityStrategyTestObject = new AvailabilityStrategyTestObject + { + Id = "deleteMe", + Pk = "MMWrite", + Other = "test" + }; + + try + { + await this.container.DeleteItemAsync( + availabilityStrategyTestObject.Id, + new PartitionKey(availabilityStrategyTestObject.Pk)); + } + catch (Exception) + { + // Ignore + } + + sendDelay.Enable(); + sendDelay2.Enable(); + + ItemResponse ir = await container.CreateItemAsync( + availabilityStrategyTestObject, + requestOptions: requestOptions); + + sendDelay.Disable(); + sendDelay2.Disable(); + + CosmosTraceDiagnostics traceDiagnostic = ir.Diagnostics as CosmosTraceDiagnostics; + Assert.IsNotNull(traceDiagnostic); + traceDiagnostic.Value.Data.TryGetValue("Response Region", out object hedgeContext); + Assert.IsNotNull(hedgeContext); + Assert.AreEqual(region3, (string)hedgeContext); + } + } + + [TestMethod] + [TestCategory("MultiMaster")] + public async Task AvailabilityStrategyMultiMasterWriteAfterStepTest() + { + FaultInjectionRule responseDelay = new FaultInjectionRuleBuilder( + id: "responseDelay", + condition: + new FaultInjectionConditionBuilder() + .WithRegion(region1) + .WithOperationType(FaultInjectionOperationType.CreateItem) + .Build(), + result: + FaultInjectionResultBuilder.GetResultBuilder(FaultInjectionServerErrorType.ResponseDelay) + .WithDelay(TimeSpan.FromMilliseconds(6000)) + .Build()) + .WithDuration(TimeSpan.FromMinutes(90)) + .Build(); + + FaultInjectionRule responseDelay2 = new FaultInjectionRuleBuilder( + id: "responseDelay2", + condition: + new FaultInjectionConditionBuilder() + .WithRegion(region2) + .WithOperationType(FaultInjectionOperationType.CreateItem) + .Build(), + result: + FaultInjectionResultBuilder.GetResultBuilder(FaultInjectionServerErrorType.ResponseDelay) + .WithDelay(TimeSpan.FromMilliseconds(6000)) + .Build()) + .WithDuration(TimeSpan.FromMinutes(90)) + .Build(); + + List rules = new List() { responseDelay, responseDelay2 }; + FaultInjector faultInjector = new FaultInjector(rules); + + responseDelay.Disable(); + responseDelay2.Disable(); + + CosmosClientOptions clientOptions = new CosmosClientOptions() + { + ConnectionMode = ConnectionMode.Direct, + ApplicationPreferredRegions = new List() { region1, region2, region3 }, + Serializer = this.cosmosSystemTextJsonSerializer + }; + + using (CosmosClient faultInjectionClient = new CosmosClient( + connectionString: this.connectionString, + clientOptions: faultInjector.GetFaultInjectionClientOptions(clientOptions))) + { + Database database = faultInjectionClient.GetDatabase(CosmosAvailabilityStrategyTests.dbName); + Container container = database.GetContainer(CosmosAvailabilityStrategyTests.containerName); + + ItemRequestOptions requestOptions = new ItemRequestOptions + { + AvailabilityStrategy = new CrossRegionHedgingAvailabilityStrategy( + threshold: TimeSpan.FromMilliseconds(100), + thresholdStep: TimeSpan.FromMilliseconds(50), + enableMultiWriteRegionHedge: true) + }; + + AvailabilityStrategyTestObject availabilityStrategyTestObject = new AvailabilityStrategyTestObject + { + Id = "deleteMe", + Pk = "MMWrite", + Other = "test" + }; + + try + { + await this.container.DeleteItemAsync( + availabilityStrategyTestObject.Id, + new PartitionKey(availabilityStrategyTestObject.Pk)); + } + catch (Exception) + { + // Ignore + } + + responseDelay.Enable(); + responseDelay2.Enable(); + + try + { + ItemResponse ir = await container.CreateItemAsync( + availabilityStrategyTestObject, + requestOptions: requestOptions); + } + catch (CosmosException ex) + { + Assert.AreEqual(HttpStatusCode.Conflict, ex.StatusCode); + + CosmosTraceDiagnostics traceDiagnostic = ex.Diagnostics as CosmosTraceDiagnostics; + Assert.IsNotNull(traceDiagnostic); + traceDiagnostic.Value.Data.TryGetValue("Response Region", out object hedgeContext); + Assert.IsNotNull(hedgeContext); + Assert.AreEqual(region3, (string)hedgeContext); + } + finally + { + responseDelay.Disable(); + responseDelay2.Disable(); + } + } + } + private static async Task HandleChangesAsync( ChangeFeedProcessorContext context, IReadOnlyCollection changes, @@ -862,7 +1233,7 @@ private static async Task HandleChangesAsync( Assert.IsNotNull(traceDiagnostic); traceDiagnostic.Value.Data.TryGetValue("Response Region", out object hedgeContext); Assert.IsNotNull(hedgeContext); - Assert.AreNotEqual(CosmosAvailabilityStrategyTests.centralUS, (string)hedgeContext); + Assert.AreNotEqual(region1, (string)hedgeContext); await Task.Delay(1); } @@ -880,8 +1251,8 @@ private static async Task HandleChangesStepAsync( Assert.IsNotNull(traceDiagnostic); traceDiagnostic.Value.Data.TryGetValue("Response Region", out object hedgeContext); Assert.IsNotNull(hedgeContext); - Assert.AreNotEqual(CosmosAvailabilityStrategyTests.centralUS, (string)hedgeContext); - Assert.AreNotEqual(CosmosAvailabilityStrategyTests.northCentralUS, (string)hedgeContext); + Assert.AreNotEqual(region1, (string)hedgeContext); + Assert.AreNotEqual(region2, (string)hedgeContext); await Task.Delay(1); } diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetPreviewSDKAPI.json b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetPreviewSDKAPI.json index f68d7b8ecd..54a159590b 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetPreviewSDKAPI.json +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetPreviewSDKAPI.json @@ -3,10 +3,10 @@ "Microsoft.Azure.Cosmos.AvailabilityStrategy;System.Object;IsAbstract:True;IsSealed:False;IsInterface:False;IsEnum:False;IsClass:True;IsValueType:False;IsNested:False;IsGenericType:False;IsSerializable:False": { "Subclasses": {}, "Members": { - "Microsoft.Azure.Cosmos.AvailabilityStrategy CrossRegionHedgingStrategy(System.TimeSpan, System.Nullable`1[System.TimeSpan])": { + "Microsoft.Azure.Cosmos.AvailabilityStrategy CrossRegionHedgingStrategy(System.TimeSpan, System.Nullable`1[System.TimeSpan], Boolean)": { "Type": "Method", "Attributes": [], - "MethodInfo": "Microsoft.Azure.Cosmos.AvailabilityStrategy CrossRegionHedgingStrategy(System.TimeSpan, System.Nullable`1[System.TimeSpan]);IsAbstract:False;IsStatic:True;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + "MethodInfo": "Microsoft.Azure.Cosmos.AvailabilityStrategy CrossRegionHedgingStrategy(System.TimeSpan, System.Nullable`1[System.TimeSpan], Boolean);IsAbstract:False;IsStatic:True;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" } }, "NestedTypes": {}