Fix CancelOnTimerWithLargeRetryDelay test flakiness due to Stopwatch …

…imprecision (#383)

Port of following PR:
#381

CancelOnTimerWithLargeRetryDelay fails on some environments due
to the fact that Stopwatch class may report incorrect timing results on
different processors, as noted in official documentation:

https://learn.microsoft.com/en-us/dotnet/api/system.diagnostics.stopwatch?view=net-8.0

Internal ADO work item:
https://dev.azure.com/msazure/One/_workitems/edit/29276368

test/unittests/Microsoft.ServiceFabric.Services.Tests/ServicePartitionClientTests.cs

@@ -29,6 +29,11 @@ public class ServicePartitionClientTests
 
         private static readonly TimeSpan DefaultRetryDelay = TimeSpan.FromMilliseconds(50);
 
+        // Due to bugs in BIOS or Hardware Abstraction Layer, Stopwatch sometimes reports slightly smaller value, as
+        // documented here: https://learn.microsoft.com/en-us/dotnet/api/system.diagnostics.stopwatch?view=net-8.0
+        // We need to account for that possibility.
+        private static readonly long StopwatchPrecisionMs = 25;
+
         /// <summary>
         /// Tests handling of cancellation by the passed cancellation token.
         /// </summary>
@@ -67,8 +72,9 @@ public async Task CancelOnTimer()
                 clientRetryTimeout,
                 retryCount,
                 retryDelay);
+            sw.Stop();
 
-            sw.ElapsedMilliseconds.Should().BeGreaterThan((long)clientRetryTimeout.TotalMilliseconds, "Should be longer than the ClientRetryTimeout.");
+            sw.ElapsedMilliseconds.Should().BeGreaterThan((long)clientRetryTimeout.TotalMilliseconds - StopwatchPrecisionMs, "Should be longer than the ClientRetryTimeout.");
             result.ExceptionFromInvoke.Should().BeAssignableTo(typeof(OperationCanceledException), $"Should indicate a canceled operation. {result.ExceptionFromInvoke}");
             result.CallCount.Should().BeLessThan(retryCount, "Should cancel before token is signaled.");
             result.CancellationTokenSource.Token.IsCancellationRequested.Should().Be(false, "Cancellation should have occured due to the timer.");
@@ -92,8 +98,9 @@ public async Task CancelAfterCall()
                 retryCount,
                 retryDelay);
             result.CancellationTokenSource.Cancel();
+            sw.Stop();
 
-            sw.ElapsedMilliseconds.Should().BeGreaterThan((long)clientRetryTimeout.TotalMilliseconds - 10, "Should be longer than the ClientRetryTimeout.");
+            sw.ElapsedMilliseconds.Should().BeGreaterThan((long)clientRetryTimeout.TotalMilliseconds - StopwatchPrecisionMs, "Should be longer than the ClientRetryTimeout.");
             result.ExceptionFromInvoke.Should().BeAssignableTo(typeof(OperationCanceledException), $"Should indicate a canceled operation. {result.ExceptionFromInvoke}");
             result.CallCount.Should().BeLessThan(retryCount, "Should cancel before token is signaled.");
         }
@@ -115,8 +122,9 @@ public async Task CancelOnTimerWithLargeRetryDelay()
                 clientRetryTimeout,
                 retryCount,
                 retryDelay);
+            sw.Stop();
 
-            sw.ElapsedMilliseconds.Should().BeGreaterThan((long)clientRetryTimeout.TotalMilliseconds, "Should be longer than the ClientRetryTimeout.");
+            sw.ElapsedMilliseconds.Should().BeGreaterThan((long)clientRetryTimeout.TotalMilliseconds - StopwatchPrecisionMs, "Should be longer than the ClientRetryTimeout.");
             sw.ElapsedMilliseconds.Should().BeLessThan((long)retryDelay.TotalMilliseconds, "Should return before the retry delay.");
             result.ExceptionFromInvoke.Should().BeAssignableTo(typeof(OperationCanceledException), "Should indicate a canceled operation.");
             result.CallCount.Should().BeLessThan(retryCount, "Should cancel before token is signaled.");