Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Hedging: Adds support for writes on multi region accounts #4706

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
42 commits
Select commit Hold shift + click to select a range
a2415ef
add mm hedging support
NaluTripician Sep 17, 2024
59a98f8
Merge branch 'master' into users/nalutripician/hedgingMMWrites
NaluTripician Sep 18, 2024
1fe29f3
Merge branch 'master' into users/nalutripician/hedgingMMWrites
NaluTripician Sep 18, 2024
4596c90
test optimizations and fixes
NaluTripician Oct 10, 2024
ef0772d
Merge branch 'master' into users/nalutripician/hedgingMMWrites
NaluTripician Oct 10, 2024
410cd9c
Merge branch 'master' into users/nalutripician/hedgingMMWrites
NaluTripician Oct 10, 2024
a795572
Adds comments
NaluTripician Oct 15, 2024
70bbb12
Merge branch 'master' into users/nalutripician/hedgingMMWrites
NaluTripician Oct 15, 2024
7f12531
Merge branch 'master' into users/nalutripician/hedgingMMWrites
NaluTripician Oct 15, 2024
fd1ae82
Merge branch 'master' into users/nalutripician/hedgingMMWrites
NaluTripician Oct 17, 2024
2687058
fixed from merge
NaluTripician Oct 17, 2024
08bd1f0
Merge branch 'master' into users/nalutripician/hedgingMMWrites
NaluTripician Oct 18, 2024
538e096
API Change
NaluTripician Oct 24, 2024
8d41961
fixed conflicts
NaluTripician Oct 24, 2024
de4d794
Merge branch 'master' into users/nalutripician/hedgingMMWrites
NaluTripician Oct 24, 2024
e79f616
fixed rename bug
NaluTripician Oct 24, 2024
a736cda
Merge branch 'users/nalutripician/hedgingMMWrites' of https://github.…
NaluTripician Oct 24, 2024
1c1aea2
Merge branch 'master' into users/nalutripician/hedgingMMWrites
NaluTripician Oct 25, 2024
28d8012
Merge branch 'master' into users/nalutripician/hedgingMMWrites
NaluTripician Oct 31, 2024
1bea3c1
api updates
NaluTripician Nov 1, 2024
7565422
Merge branch 'master' into users/nalutripician/hedgingMMWrites
NaluTripician Nov 1, 2024
7b26e9d
fix contracts
NaluTripician Nov 1, 2024
51116d1
Update DotNetPreviewSDKAPI.json
NaluTripician Nov 1, 2024
e9c2f4d
Merge branch 'master' into users/nalutripician/hedgingMMWrites
NaluTripician Nov 6, 2024
9a22f3a
Merge branch 'master' into users/nalutripician/hedgingMMWrites
NaluTripician Nov 11, 2024
cf39c8c
requested changes
NaluTripician Nov 12, 2024
d877e8a
Merge branch 'master' into users/nalutripician/hedgingMMWrites
NaluTripician Nov 14, 2024
21bfafd
Update Microsoft.Azure.Cosmos/src/Routing/AvailabilityStrategy/Availa…
NaluTripician Nov 14, 2024
2637763
Update Microsoft.Azure.Cosmos/src/Routing/AvailabilityStrategy/CrossR…
NaluTripician Nov 14, 2024
56d722c
Merge branch 'master' into users/nalutripician/hedgingMMWrites
NaluTripician Nov 18, 2024
863298d
Merge branch 'master' into users/nalutripician/hedgingMMWrites
NaluTripician Nov 19, 2024
fecb4f9
fixed merge
NaluTripician Nov 19, 2024
0a188b8
Update CosmosAvailabilityStrategyTests.cs
NaluTripician Nov 19, 2024
5951c1b
Merge branch 'master' into users/nalutripician/hedgingMMWrites
NaluTripician Nov 19, 2024
8e346d1
Removed Gone Tests
NaluTripician Nov 19, 2024
68e42b1
Merge branch 'master' into users/nalutripician/hedgingMMWrites
NaluTripician Nov 21, 2024
2846f99
removed unneeded var in documentclient
NaluTripician Nov 22, 2024
2a69844
Merge branch 'master' into users/nalutripician/hedgingMMWrites
NaluTripician Nov 25, 2024
31645a8
Apply suggestions from code review
NaluTripician Nov 25, 2024
833d453
Merge branch 'master' into users/nalutripician/hedgingMMWrites
NaluTripician Nov 25, 2024
d588cca
Merge branch 'master' into users/nalutripician/hedgingMMWrites
NaluTripician Nov 26, 2024
9e6dc08
Merge branch 'master' into users/nalutripician/hedgingMMWrites
NaluTripician Nov 26, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Microsoft.Azure.Cosmos/src/ClientRetryPolicy.cs
Original file line number Diff line number Diff line change
Expand Up @@ -193,7 +193,7 @@ public void OnBeforeSendRequest(DocumentServiceRequest request)
this.canUseMultipleWriteLocations = this.globalEndpointManager.CanUseMultipleWriteLocations(request);
this.documentServiceRequest = request;
this.isMultiMasterWriteRequest = !this.isReadRequest
&& (this.globalEndpointManager?.CanSupportMultipleWriteLocations(request) ?? false);
&& (this.globalEndpointManager?.CanSupportMultipleWriteLocations(request.ResourceType, request.OperationType) ?? false);

// clear previous location-based routing directive
request.RequestContext.ClearRouteToLocation();
Expand Down
4 changes: 0 additions & 4 deletions Microsoft.Azure.Cosmos/src/DocumentClient.cs
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,6 @@ internal partial class DocumentClient : IDisposable, IAuthorizationTokenProvider

private readonly bool IsLocalQuorumConsistency = false;
private readonly bool isReplicaAddressValidationEnabled;
private readonly AvailabilityStrategy availabilityStrategy;

//Fault Injection
private readonly IChaosInterceptorFactory chaosInterceptorFactory;
Expand Down Expand Up @@ -441,7 +440,6 @@ internal DocumentClient(Uri serviceEndpoint,
/// <param name="cosmosClientId"></param>
/// <param name="remoteCertificateValidationCallback">This delegate responsible for validating the third party certificate. </param>
/// <param name="cosmosClientTelemetryOptions">This is distributed tracing flag</param>
/// <param name="availabilityStrategy">This is the availability strategy for the client</param>"
/// <param name="chaosInterceptorFactory">This is the chaos interceptor used for fault injection</param>
/// <remarks>
/// The service endpoint can be obtained from the Azure Management Portal.
Expand Down Expand Up @@ -471,7 +469,6 @@ internal DocumentClient(Uri serviceEndpoint,
string cosmosClientId = null,
RemoteCertificateValidationCallback remoteCertificateValidationCallback = null,
CosmosClientTelemetryOptions cosmosClientTelemetryOptions = null,
AvailabilityStrategy availabilityStrategy = null,
IChaosInterceptorFactory chaosInterceptorFactory = null)
{
if (sendingRequestEventArgs != null)
Expand All @@ -495,7 +492,6 @@ internal DocumentClient(Uri serviceEndpoint,
this.transportClientHandlerFactory = transportClientHandlerFactory;
this.IsLocalQuorumConsistency = isLocalQuorumConsistency;
this.initTaskCache = new AsyncCacheNonBlocking<string, bool>(cancellationToken: this.cancellationTokenSource.Token);
this.availabilityStrategy = availabilityStrategy;
this.chaosInterceptorFactory = chaosInterceptorFactory;
this.chaosInterceptor = chaosInterceptorFactory?.CreateInterceptor(this);

Expand Down
1 change: 0 additions & 1 deletion Microsoft.Azure.Cosmos/src/Resource/ClientContextCore.cs
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,6 @@ internal static CosmosClientContext Create(
cosmosClientId: cosmosClient.Id,
remoteCertificateValidationCallback: ClientContextCore.SslCustomValidationCallBack(clientOptions.GetServerCertificateCustomValidationCallback()),
cosmosClientTelemetryOptions: clientOptions.CosmosClientTelemetryOptions,
availabilityStrategy: clientOptions.AvailabilityStrategy,
chaosInterceptorFactory: clientOptions.ChaosInterceptorFactory);

return ClientContextCore.Create(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,11 +39,17 @@ internal static AvailabilityStrategy DisabledStrategy()
/// </summary>
/// <param name="threshold"> how long before SDK begins hedging</param>
/// <param name="thresholdStep">Period of time between first hedge and next hedging attempts</param>
/// <param name="enableMultiWriteRegionHedge">Whether hedging for write requests on accounts with multi-region writes are enabled
/// Note that this does come with the caveat that there will be more 409 / 412 errors thrown by the SDK.
/// This is expected and applications that adopt this feature should be prepared to handle these exceptions.
/// Application might not be able to be deterministic on Create vs Replace in the case of Upsert Operations</param>
/// <returns>something</returns>
public static AvailabilityStrategy CrossRegionHedgingStrategy(TimeSpan threshold,
TimeSpan? thresholdStep)
public static AvailabilityStrategy CrossRegionHedgingStrategy(
TimeSpan threshold,
TimeSpan? thresholdStep,
bool enableMultiWriteRegionHedge = false)
{
return new CrossRegionHedgingAvailabilityStrategy(threshold, thresholdStep);
return new CrossRegionHedgingAvailabilityStrategy(threshold, thresholdStep, enableMultiWriteRegionHedge);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -36,14 +36,24 @@ internal class CrossRegionHedgingAvailabilityStrategy : AvailabilityStrategyInte
/// </summary>
public TimeSpan ThresholdStep { get; private set; }

/// <summary>
/// Whether hedging for write requests on accounts with multi-region writes is enabled.
/// Note that this does come with the caveat that there will be more 409 / 412 errors thrown by the SDK.
/// This is expected and applications that adopt this feature should be prepared to handle these exceptions.
/// Application might not be able to be deterministic on Create vs Replace in the case of Upsert Operations
/// </summary>
public bool EnableMultiWriteRegionHedge { get; private set; }

/// <summary>
/// Constructor for hedging availability strategy
/// </summary>
/// <param name="threshold"></param>
/// <param name="thresholdStep"></param>
/// <param name="enableMultiWriteRegionHedge"></param>
public CrossRegionHedgingAvailabilityStrategy(
TimeSpan threshold,
TimeSpan? thresholdStep)
TimeSpan? thresholdStep,
bool enableMultiWriteRegionHedge = false)
{
if (threshold <= TimeSpan.Zero)
{
Expand All @@ -57,6 +67,7 @@ public CrossRegionHedgingAvailabilityStrategy(

this.Threshold = threshold;
this.ThresholdStep = thresholdStep ?? TimeSpan.FromMilliseconds(-1);
this.EnableMultiWriteRegionHedge = enableMultiWriteRegionHedge;
}

/// <inheritdoc/>
Expand All @@ -70,18 +81,24 @@ internal override bool Enabled()
/// This availability strategy can only be used if the request is a read-only request on a document request.
/// </summary>
/// <param name="request"></param>
/// <param name="client"></param>
/// <returns>whether the request should be a hedging request.</returns>
internal bool ShouldHedge(RequestMessage request)
internal bool ShouldHedge(RequestMessage request, CosmosClient client)
{
//Only use availability strategy for document point operations
if (request.ResourceType != ResourceType.Document)
{
return false;
}

//check to see if it is a not a read-only request
//check to see if it is a not a read-only request/ if multimaster writes are enabled
if (!OperationTypeExtensions.IsReadOperation(request.OperationType))
{
if (this.EnableMultiWriteRegionHedge
&& client.DocumentClient.GlobalEndpointManager.CanSupportMultipleWriteLocations(request.ResourceType, request.OperationType))
{
return true;
}
return false;
}

Expand All @@ -102,7 +119,7 @@ internal override async Task<ResponseMessage> ExecuteAvailabilityStrategyAsync(
RequestMessage request,
CancellationToken cancellationToken)
{
if (!this.ShouldHedge(request)
if (!this.ShouldHedge(request, client)
|| client.DocumentClient.GlobalEndpointManager.ReadEndpoints.Count == 1)
{
return await sender(request, cancellationToken);
Expand All @@ -113,7 +130,7 @@ internal override async Task<ResponseMessage> ExecuteAvailabilityStrategyAsync(
using (CancellationTokenSource cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken))
{
using (CloneableStream clonedBody = (CloneableStream)(request.Content == null
? null//new CloneableStream(new MemoryStream())
? null
: await StreamExtension.AsClonableStreamAsync(request.Content)))
{
IReadOnlyCollection<string> hedgeRegions = client.DocumentClient.GlobalEndpointManager
Expand Down
13 changes: 8 additions & 5 deletions Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs
Original file line number Diff line number Diff line change
Expand Up @@ -546,14 +546,17 @@ public virtual async Task RefreshLocationAsync(bool forceRefresh = false)
/// Determines whether the current configuration and state of the service allow for supporting multiple write locations.
/// This method returns True is the AvailableWriteLocations in LocationCache is more than 1. Otherwise, it returns False.
/// </summary>
/// <param name="request">The document service request for which the write location support is being evaluated.</param>
/// <returns>A boolean flag indicating if the available write locations are more than one.</returns>
public bool CanSupportMultipleWriteLocations(DocumentServiceRequest request)
/// <param name="resourceType"> resource type of the request</param>
/// <param name="operationType"> operation type of the request</param>
/// <returns>A boolean flag indicating if the available write locations are more than one.</returns>
public bool CanSupportMultipleWriteLocations(
ResourceType resourceType,
OperationType operationType)
{
return this.locationCache.CanUseMultipleWriteLocations()
&& this.locationCache.GetAvailableAccountLevelWriteLocations()?.Count > 1
&& (request.ResourceType == ResourceType.Document ||
(request.ResourceType == ResourceType.StoredProcedure && request.OperationType == OperationType.Execute));
&& (resourceType == ResourceType.Document ||
(resourceType == ResourceType.StoredProcedure && operationType == OperationType.Execute));
}

#pragma warning disable VSTHRD100 // Avoid async void methods
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,6 @@ internal interface IGlobalEndpointManager : IDisposable

ReadOnlyDictionary<string, Uri> GetAvailableReadEndpointsByLocation();

bool CanSupportMultipleWriteLocations(DocumentServiceRequest request);
bool CanSupportMultipleWriteLocations(ResourceType resourceType, OperationType operationType);
}
}
Loading
Loading