Track current number of txns in retry loop in quorum driver (#16327)

## Description 

Describe the changes or additions included in this PR.

## Test Plan 

How did you test the new or updated feature?

---
If your changes are not user-facing and do not break anything, you can
skip the following section. Otherwise, please briefly describe what has
changed under the Release Notes section.

### Type of Change (Check all that apply)

- [ ] protocol change
- [ ] user-visible impact
- [ ] breaking change for a client SDKs
- [ ] breaking change for FNs (FN binary must upgrade)
- [ ] breaking change for validators or node operators (must upgrade
binaries)
- [ ] breaking change for on-chain data layout
- [ ] necessitate either a data wipe or data migration

### Release notes

crates/sui-core/src/quorum_driver/metrics.rs

@@ -35,6 +35,7 @@ pub struct QuorumDriverMetrics {
     pub(crate) total_times_conflicting_transaction_already_finalized_when_retrying: IntCounter,
     pub(crate) total_retryable_overload_errors: IntCounter,
     pub(crate) transaction_retry_count: Histogram,
+    pub(crate) current_transactions_in_retry: IntGauge,
 
     pub(crate) settlement_finality_latency: HistogramVec,
 }
@@ -113,6 +114,12 @@ impl QuorumDriverMetrics {
                 "Histogram of transaction retry count",
                 registry,
             ),
+            current_transactions_in_retry: register_int_gauge_with_registry!(
+                "current_transactions_in_retry",
+                "Current number of transactions in retry loop in QuorumDriver",
+                registry,
+            )
+            .unwrap(),
             settlement_finality_latency: register_histogram_vec_with_registry!(
                 "quorum_driver_settlement_finality_latency",
                 "Settlement finality latency observed from quorum driver",

crates/sui-core/src/quorum_driver/mod.rs

@@ -116,6 +116,9 @@ impl<A: Clone> QuorumDriver<A> {
                 debug!(?task, "Enqueued task.");
                 self.metrics.current_requests_in_flight.inc();
                 self.metrics.total_enqueued.inc();
+                if task.retry_times == 1 {
+                    self.metrics.current_transactions_in_retry.inc();
+                }
             })
             .map_err(|e| SuiError::QuorumDriverCommunicationError {
                 error: e.to_string(),
@@ -199,6 +202,9 @@ impl<A: Clone> QuorumDriver<A> {
                 Err((*tx_digest, err.clone()))
             }
         };
+        if total_attempts > 1 {
+            self.metrics.current_transactions_in_retry.dec();
+        }
         // On fullnode we expect the send to always succeed because TransactionOrchestrator should be subscribing
         // to this queue all the time. However the if QuorumDriver is used elsewhere log may be noisy.
         if let Err(err) = self.effects_subscribe_sender.send(effects_queue_result) {