From c3cda5656b1d4c2bc41240e0cf1883aff1b8113c Mon Sep 17 00:00:00 2001 From: Zhe Wu Date: Sat, 14 Dec 2024 21:52:07 -0800 Subject: [PATCH] Add a workload that randomly generates different transaction shapes (#20402) ## Description Randomized transaction testing. ## Test plan How did you test the new or updated feature? --- ## Release notes Check each box that your changes affect. If none of the boxes relate to your changes, release notes aren't required. For each box you select, include information after the relevant heading that describes the impact of your changes that a user might notice and any actions they must take to implement updates. - [ ] Protocol: - [ ] Nodes (Validators and Full nodes): - [ ] Indexer: - [ ] JSON-RPC: - [ ] GraphQL: - [ ] CLI: - [ ] Rust SDK: - [ ] REST API: --- .../sui-benchmark/src/drivers/bench_driver.rs | 23 +- crates/sui-benchmark/src/options.rs | 3 + .../src/workloads/adversarial.rs | 2 +- .../src/workloads/batch_payment.rs | 2 +- .../sui-benchmark/src/workloads/delegation.rs | 2 +- .../src/workloads/expected_failure.rs | 3 +- crates/sui-benchmark/src/workloads/mod.rs | 1 + .../src/workloads/randomized_transaction.rs | 531 ++++++++++++++++++ .../sui-benchmark/src/workloads/randomness.rs | 2 +- .../src/workloads/shared_counter.rs | 2 +- .../src/workloads/shared_object_deletion.rs | 2 +- .../src/workloads/transfer_object.rs | 2 +- .../sui-benchmark/src/workloads/workload.rs | 12 +- .../src/workloads/workload_configuration.rs | 25 +- crates/sui-benchmark/tests/simtest.rs | 7 +- 15 files changed, 600 insertions(+), 19 deletions(-) create mode 100644 crates/sui-benchmark/src/workloads/randomized_transaction.rs diff --git a/crates/sui-benchmark/src/drivers/bench_driver.rs b/crates/sui-benchmark/src/drivers/bench_driver.rs index 148d79f51ae6a..1ab4de38c26ed 100644 --- a/crates/sui-benchmark/src/drivers/bench_driver.rs +++ b/crates/sui-benchmark/src/drivers/bench_driver.rs @@ -26,6 +26,7 @@ use crate::drivers::driver::Driver; use crate::drivers::HistogramWrapper; use crate::system_state_observer::SystemStateObserver; use crate::workloads::payload::Payload; +use crate::workloads::workload::ExpectedFailureType; use crate::workloads::{GroupID, WorkloadInfo}; use crate::{ExecutionEffects, ValidatorProxy}; use std::collections::{BTreeMap, VecDeque}; @@ -737,7 +738,10 @@ async fn run_bench_worker( -> NextOp { match result { Ok(effects) => { - assert!(payload.get_failure_type().is_none()); + assert!( + payload.get_failure_type().is_none() + || payload.get_failure_type() == Some(ExpectedFailureType::NoFailure) + ); let latency = start.elapsed(); let time_from_start = total_benchmark_start_time.elapsed(); @@ -796,8 +800,15 @@ async fn run_bench_worker( } } Err(err) => { - error!("{}", err); + tracing::error!( + "Transaction execution got error: {}. Transaction digest: {:?}", + err, + transaction.digest() + ); match payload.get_failure_type() { + Some(ExpectedFailureType::NoFailure) => { + panic!("Transaction failed unexpectedly"); + } Some(_) => { metrics_cloned .num_expected_error @@ -917,10 +928,10 @@ async fn run_bench_worker( if let Some(b) = retry_queue.pop_front() { let tx = b.0; let payload = b.1; - if payload.get_failure_type().is_some() { - num_expected_error_txes += 1; - } else { - num_error_txes += 1; + match payload.get_failure_type() { + Some(ExpectedFailureType::NoFailure) => num_error_txes += 1, + Some(_) => num_expected_error_txes += 1, + None => num_error_txes += 1, } num_submitted += 1; metrics_cloned.num_submitted.with_label_values(&[&payload.to_string()]).inc(); diff --git a/crates/sui-benchmark/src/options.rs b/crates/sui-benchmark/src/options.rs index 4bab34fd41930..1373d506ec351 100644 --- a/crates/sui-benchmark/src/options.rs +++ b/crates/sui-benchmark/src/options.rs @@ -184,6 +184,9 @@ pub enum RunSpec { // relative weight of expected failure transactions in the benchmark workload #[clap(long, num_args(1..), value_delimiter = ',', default_values_t = [0])] expected_failure: Vec, + // relative weight of randomized transaction in the benchmark workload + #[clap(long, num_args(1..), value_delimiter = ',', default_values_t = [0])] + randomized_transaction: Vec, // --- workload-specific options --- (TODO: use subcommands or similar) // 100 for max hotness i.e all requests target diff --git a/crates/sui-benchmark/src/workloads/adversarial.rs b/crates/sui-benchmark/src/workloads/adversarial.rs index 39746b89c31d5..5471ee60f5562 100644 --- a/crates/sui-benchmark/src/workloads/adversarial.rs +++ b/crates/sui-benchmark/src/workloads/adversarial.rs @@ -412,7 +412,7 @@ impl AdversarialWorkloadBuilder { duration: Interval, group: u32, ) -> Option { - let target_qps = (workload_weight * target_qps as f32) as u64; + let target_qps = (workload_weight * target_qps as f32).ceil() as u64; let num_workers = (workload_weight * num_workers as f32).ceil() as u64; let max_ops = target_qps * in_flight_ratio; if max_ops == 0 || num_workers == 0 { diff --git a/crates/sui-benchmark/src/workloads/batch_payment.rs b/crates/sui-benchmark/src/workloads/batch_payment.rs index 94e58da2ae484..c4e068b99f15a 100644 --- a/crates/sui-benchmark/src/workloads/batch_payment.rs +++ b/crates/sui-benchmark/src/workloads/batch_payment.rs @@ -138,7 +138,7 @@ impl BatchPaymentWorkloadBuilder { duration: Interval, group: u32, ) -> Option { - let target_qps = (workload_weight * target_qps as f32) as u64; + let target_qps = (workload_weight * target_qps as f32).ceil() as u64; let num_workers = (workload_weight * num_workers as f32).ceil() as u64; let max_ops = target_qps * in_flight_ratio; if max_ops == 0 || num_workers == 0 { diff --git a/crates/sui-benchmark/src/workloads/delegation.rs b/crates/sui-benchmark/src/workloads/delegation.rs index 24f21a120f12c..3e0fdd651e8b2 100644 --- a/crates/sui-benchmark/src/workloads/delegation.rs +++ b/crates/sui-benchmark/src/workloads/delegation.rs @@ -100,7 +100,7 @@ impl DelegationWorkloadBuilder { duration: Interval, group: u32, ) -> Option { - let target_qps = (workload_weight * target_qps as f32) as u64; + let target_qps = (workload_weight * target_qps as f32).ceil() as u64; let num_workers = (workload_weight * num_workers as f32).ceil() as u64; let max_ops = target_qps * in_flight_ratio; if max_ops == 0 || num_workers == 0 { diff --git a/crates/sui-benchmark/src/workloads/expected_failure.rs b/crates/sui-benchmark/src/workloads/expected_failure.rs index ae200ccbe759b..f5354927a330b 100644 --- a/crates/sui-benchmark/src/workloads/expected_failure.rs +++ b/crates/sui-benchmark/src/workloads/expected_failure.rs @@ -54,6 +54,7 @@ impl ExpectedFailurePayload { tx } ExpectedFailureType::Random => unreachable!(), + ExpectedFailureType::NoFailure => unreachable!(), } } } @@ -112,7 +113,7 @@ impl ExpectedFailureWorkloadBuilder { duration: Interval, group: u32, ) -> Option { - let target_qps = (workload_weight * target_qps as f32) as u64; + let target_qps = (workload_weight * target_qps as f32).ceil() as u64; let num_workers = (workload_weight * num_workers as f32).ceil() as u64; let max_ops = target_qps * in_flight_ratio; if max_ops == 0 || num_workers == 0 { diff --git a/crates/sui-benchmark/src/workloads/mod.rs b/crates/sui-benchmark/src/workloads/mod.rs index 368f2c96fe0c4..466cad016c606 100644 --- a/crates/sui-benchmark/src/workloads/mod.rs +++ b/crates/sui-benchmark/src/workloads/mod.rs @@ -6,6 +6,7 @@ pub mod batch_payment; pub mod delegation; pub mod expected_failure; pub mod payload; +pub mod randomized_transaction; pub mod randomness; pub mod shared_counter; pub mod shared_object_deletion; diff --git a/crates/sui-benchmark/src/workloads/randomized_transaction.rs b/crates/sui-benchmark/src/workloads/randomized_transaction.rs new file mode 100644 index 0000000000000..19f18cccdfd4e --- /dev/null +++ b/crates/sui-benchmark/src/workloads/randomized_transaction.rs @@ -0,0 +1,531 @@ +// Copyright (c) Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +use crate::drivers::Interval; +use crate::system_state_observer::SystemStateObserver; +use crate::util::publish_basics_package; +use crate::workloads::payload::Payload; +use crate::workloads::workload::{ + ExpectedFailureType, Workload, WorkloadBuilder, ESTIMATED_COMPUTATION_COST, MAX_GAS_FOR_TESTING, +}; +use crate::workloads::{Gas, GasCoinConfig, WorkloadBuilderInfo, WorkloadParams}; +use crate::{ExecutionEffects, ValidatorProxy}; +use async_trait::async_trait; +use futures::future::join_all; +use rand::Rng; +use std::sync::Arc; +use sui_test_transaction_builder::TestTransactionBuilder; +use sui_types::base_types::{ObjectID, ObjectRef, SequenceNumber, SuiAddress}; +use sui_types::crypto::{get_key_pair, AccountKeyPair}; +use sui_types::object::Owner; +use sui_types::programmable_transaction_builder::ProgrammableTransactionBuilder; +use sui_types::transaction::{CallArg, ObjectArg, Transaction}; +use sui_types::{Identifier, SUI_RANDOMNESS_STATE_OBJECT_ID}; +use tracing::{error, info}; + +use super::STORAGE_COST_PER_COUNTER; + +pub const MAX_GAS_IN_UNIT: u64 = 1_000_000_000; + +/// A workload that generates random transactions to test the system under different transaction patterns. +/// The workload can: +/// - Create shared counter objects +/// - Make random move calls to increment/read/delete counters +/// - Make calls to the randomness module +/// - Make native transfers +/// - Mix different numbers of pure and shared object inputs +/// - Generate multiple move calls per transaction +/// +/// The exact mix of operations is controlled by random selection within configured bounds. +/// +/// Different from adversarial workload, this workload is not designed to test the system under +/// malicious behaviors, but to test the system under different transaction patterns. +#[derive(Debug)] +pub struct RandomizedTransactionPayload { + package_id: ObjectID, + shared_objects: Vec, + owned_object: ObjectRef, + randomness_initial_shared_version: SequenceNumber, + transfer_to: SuiAddress, + gas: Gas, + system_state_observer: Arc, +} + +impl std::fmt::Display for RandomizedTransactionPayload { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "randomized_transaction") + } +} + +/// Config for a randomized transaction +#[derive(Debug)] +struct RandomizedTransactionConfig { + // Whether the transaction contains the owned object + contain_owned_object: bool, + // Number of pure inputs + num_pure_input: u64, + // Number of shared inputs + num_shared_inputs: u64, + // Number of move calls + num_move_calls: u64, +} + +fn generate_random_transaction_config( + num_shared_objects_exist: u64, +) -> RandomizedTransactionConfig { + let num_shared_inputs = rand::thread_rng().gen_range(0..=num_shared_objects_exist); + RandomizedTransactionConfig { + contain_owned_object: rand::thread_rng().gen_bool(0.5), + num_pure_input: rand::thread_rng().gen_range(0..=5), + num_shared_inputs, + num_move_calls: std::cmp::min(rand::thread_rng().gen_range(0..=3), num_shared_inputs), + } +} + +/// Type of move call to generate. +enum MoveCallType { + ContractCall, + Randomness, + NativeCall, +} + +/// Choose a random move call type. +fn choose_move_call_type(next_shared_input_index: usize, num_shared_inputs: u64) -> MoveCallType { + if next_shared_input_index < num_shared_inputs as usize { + match rand::thread_rng().gen_range(0..=2) { + 0 => MoveCallType::ContractCall, + 1 => MoveCallType::Randomness, + _ => MoveCallType::NativeCall, + } + } else { + match rand::thread_rng().gen_range(0..=1) { + 0 => MoveCallType::Randomness, + _ => MoveCallType::NativeCall, + } + } +} + +impl RandomizedTransactionPayload { + fn make_counter_move_call( + &mut self, + builder: &mut ProgrammableTransactionBuilder, + next_shared_input_index: usize, + ) { + // 33% chance to increment, 33% chance to set value, 33% chance to read value. + match rand::thread_rng().gen_range(0..=2) { + 0 => { + builder + .move_call( + self.package_id, + Identifier::new("counter").unwrap(), + Identifier::new("increment").unwrap(), + vec![], + vec![CallArg::Object(ObjectArg::SharedObject { + id: self.shared_objects[next_shared_input_index].0, + initial_shared_version: self.shared_objects[next_shared_input_index].1, + mutable: true, + })], + ) + .unwrap(); + } + 1 => { + builder + .move_call( + self.package_id, + Identifier::new("counter").unwrap(), + Identifier::new("set_value").unwrap(), + vec![], + vec![ + CallArg::Object(ObjectArg::SharedObject { + id: self.shared_objects[next_shared_input_index].0, + initial_shared_version: self.shared_objects + [next_shared_input_index] + .1, + mutable: true, + }), + CallArg::Pure((10_u64).to_le_bytes().to_vec()), + ], + ) + .unwrap(); + } + _ => { + builder + .move_call( + self.package_id, + Identifier::new("counter").unwrap(), + Identifier::new("value").unwrap(), + vec![], + vec![CallArg::Object(ObjectArg::SharedObject { + id: self.shared_objects[next_shared_input_index].0, + initial_shared_version: self.shared_objects[next_shared_input_index].1, + mutable: false, + })], + ) + .unwrap(); + } + } + } + + fn make_randomness_move_call(&mut self, builder: &mut ProgrammableTransactionBuilder) { + builder + .move_call( + self.package_id, + Identifier::new("random").unwrap(), + Identifier::new("new").unwrap(), + vec![], + vec![CallArg::Object(ObjectArg::SharedObject { + id: SUI_RANDOMNESS_STATE_OBJECT_ID, + initial_shared_version: self.randomness_initial_shared_version, + mutable: false, + })], + ) + .unwrap(); + } + + fn make_native_move_call(&mut self, builder: &mut ProgrammableTransactionBuilder) { + builder + .pay_sui( + vec![self.transfer_to], + vec![rand::thread_rng().gen_range(0..=1)], + ) + .unwrap(); + } +} + +impl Payload for RandomizedTransactionPayload { + fn make_new_payload(&mut self, effects: &ExecutionEffects) { + if !effects.is_ok() { + effects.print_gas_summary(); + error!( + "Randomized transaction failed... Status: {:?}", + effects.status() + ); + } + self.gas.0 = effects.gas_object().0; + + // Update owned object if it's mutated in this transaction + if let Some(owned_in_effects) = effects + .mutated() + .iter() + .find(|(object_ref, _)| object_ref.0 == self.owned_object.0) + .map(|x| x.0) + { + tracing::debug!("Owned object mutated: {:?}", owned_in_effects); + self.owned_object = owned_in_effects; + } + } + + fn make_transaction(&mut self) -> Transaction { + let rgp = self + .system_state_observer + .state + .borrow() + .reference_gas_price; + + let config = generate_random_transaction_config(self.shared_objects.len() as u64); + + let mut builder = ProgrammableTransactionBuilder::new(); + + // Generate inputs in addition to move calls. + if config.contain_owned_object { + builder + .obj(ObjectArg::ImmOrOwnedObject(self.owned_object)) + .unwrap(); + } + for i in 0..config.num_shared_inputs { + builder + .obj(ObjectArg::SharedObject { + id: self.shared_objects[i as usize].0, + initial_shared_version: self.shared_objects[i as usize].1, + mutable: rand::thread_rng().gen_bool(0.5), + }) + .unwrap(); + } + for _i in 0..config.num_pure_input { + let len = rand::thread_rng().gen_range(0..=3); + let mut bytes = vec![0u8; len]; + rand::thread_rng().fill(&mut bytes[..]); + builder.pure_bytes(bytes, false); + } + + // Generate move calls. + let mut next_shared_input_index: usize = 0; + for _i in 0..config.num_move_calls { + match choose_move_call_type(next_shared_input_index, config.num_shared_inputs) { + MoveCallType::ContractCall => { + self.make_counter_move_call(&mut builder, next_shared_input_index); + next_shared_input_index += 1; + } + MoveCallType::Randomness => { + self.make_randomness_move_call(&mut builder); + // TODO: add TransferObject move call after randomness command. + break; + } + MoveCallType::NativeCall => { + self.make_native_move_call(&mut builder); + } + } + } + let tx = builder.finish(); + + tracing::info!("Randomized transaction: {:?}", tx); + + let signed_tx = TestTransactionBuilder::new(self.gas.1, self.gas.0, rgp) + .programmable(tx) + .build_and_sign(self.gas.2.as_ref()); + + tracing::debug!("Signed transaction digest: {:?}", signed_tx.digest()); + signed_tx + } + + fn get_failure_type(&self) -> Option { + // We do not expect randomized transaction to fail + Some(ExpectedFailureType::NoFailure) + } +} + +#[derive(Debug)] +pub struct RandomizedTransactionWorkloadBuilder { + num_payloads: u64, + rgp: u64, +} + +impl RandomizedTransactionWorkloadBuilder { + pub fn from( + workload_weight: f32, + target_qps: u64, + num_workers: u64, + in_flight_ratio: u64, + reference_gas_price: u64, + duration: Interval, + group: u32, + ) -> Option { + let target_qps = (workload_weight * target_qps as f32).ceil() as u64; + let num_workers = (workload_weight * num_workers as f32).ceil() as u64; + let max_ops = target_qps * in_flight_ratio; + + if max_ops == 0 || num_workers == 0 { + None + } else { + let workload_params = WorkloadParams { + group, + target_qps, + num_workers, + max_ops, + duration, + }; + let workload_builder = Box::>::from(Box::new( + RandomizedTransactionWorkloadBuilder { + num_payloads: max_ops, + rgp: reference_gas_price, + }, + )); + Some(WorkloadBuilderInfo { + workload_params, + workload_builder, + }) + } + } +} + +#[async_trait] +impl WorkloadBuilder for RandomizedTransactionWorkloadBuilder { + async fn generate_coin_config_for_init(&self) -> Vec { + let mut configs = vec![]; + + // Gas coin for publishing package + let (address, keypair) = get_key_pair(); + configs.push(GasCoinConfig { + amount: MAX_GAS_FOR_TESTING, + address, + keypair: Arc::new(keypair), + }); + + // Gas coins for creating counters + for _i in 0..self.num_payloads { + let (address, keypair) = get_key_pair(); + configs.push(GasCoinConfig { + amount: MAX_GAS_FOR_TESTING, + address, + keypair: Arc::new(keypair), + }); + } + configs + } + + async fn generate_coin_config_for_payloads(&self) -> Vec { + let mut configs = vec![]; + let amount = MAX_GAS_IN_UNIT * (self.rgp) + + ESTIMATED_COMPUTATION_COST + + STORAGE_COST_PER_COUNTER * self.num_payloads + + MAX_GAS_FOR_TESTING; + // Gas coins for running workload + for _i in 0..self.num_payloads { + let (address, keypair) = get_key_pair(); + configs.push(GasCoinConfig { + amount, + address, + keypair: Arc::new(keypair), + }); + } + configs + } + + async fn build( + &self, + init_gas: Vec, + payload_gas: Vec, + ) -> Box> { + Box::>::from(Box::new(RandomizedTransactionWorkload { + basics_package_id: None, + shared_objects: vec![], + owned_objects: vec![], + transfer_to: None, + init_gas, + payload_gas, + randomness_initial_shared_version: None, + })) + } +} + +#[derive(Debug)] +pub struct RandomizedTransactionWorkload { + pub basics_package_id: Option, + pub shared_objects: Vec, + pub owned_objects: Vec, + pub transfer_to: Option, + pub init_gas: Vec, + pub payload_gas: Vec, + pub randomness_initial_shared_version: Option, +} + +#[async_trait] +impl Workload for RandomizedTransactionWorkload { + async fn init( + &mut self, + proxy: Arc, + system_state_observer: Arc, + ) { + if self.basics_package_id.is_some() { + return; + } + let gas_price = system_state_observer.state.borrow().reference_gas_price; + let (head, tail) = self + .init_gas + .split_first() + .expect("Not enough gas to initialize randomized transaction workload"); + + // Publish basics package + info!("Publishing basics package"); + self.basics_package_id = Some( + publish_basics_package(head.0, proxy.clone(), head.1, &head.2, gas_price) + .await + .0, + ); + + // Create a transfer address + self.transfer_to = Some(get_key_pair::().0); + + // Create shared objects + { + let mut futures = vec![]; + for (gas, sender, keypair) in tail.iter() { + let transaction = TestTransactionBuilder::new(*sender, *gas, gas_price) + .call_counter_create(self.basics_package_id.unwrap()) + .build_and_sign(keypair.as_ref()); + let proxy_ref = proxy.clone(); + futures.push(async move { + proxy_ref + .execute_transaction_block(transaction) + .await + .unwrap() + .created()[0] + .0 + }); + } + self.shared_objects = join_all(futures).await; + } + + // create owned objects + { + let mut futures = vec![]; + for (gas, sender, keypair) in self.payload_gas.iter() { + let transaction = TestTransactionBuilder::new(*sender, *gas, gas_price) + .move_call( + self.basics_package_id.unwrap(), + "object_basics", + "create", + vec![ + CallArg::Pure(bcs::to_bytes(&(16_u64)).unwrap()), + CallArg::Pure(bcs::to_bytes(&sender).unwrap()), + ], + ) + .build_and_sign(keypair.as_ref()); + let proxy_ref = proxy.clone(); + futures.push(async move { + let execution_result = proxy_ref + .execute_transaction_block(transaction) + .await + .unwrap(); + let created_owned = execution_result.created()[0].0; + let updated_gas = execution_result.gas_object().0; + (created_owned, updated_gas) + }); + } + let results = join_all(futures).await; + self.owned_objects = results.iter().map(|x| x.0).collect(); + + // Update gas object in payload gas + for (payload_gas, result) in self.payload_gas.iter_mut().zip(results.iter()) { + payload_gas.0 = result.1; + } + } + + // Get randomness shared object initial version + if self.randomness_initial_shared_version.is_none() { + let obj = proxy + .get_object(SUI_RANDOMNESS_STATE_OBJECT_ID) + .await + .expect("Failed to get randomness object"); + let Owner::Shared { + initial_shared_version, + } = obj.owner() + else { + panic!("randomness object must be shared"); + }; + self.randomness_initial_shared_version = Some(*initial_shared_version); + } + + info!( + "Basics package id {:?}. Total shared objects created {:?}", + self.basics_package_id, + self.shared_objects.len() + ); + } + + async fn make_test_payloads( + &self, + _proxy: Arc, + system_state_observer: Arc, + ) -> Vec> { + info!("Creating randomized transaction payloads..."); + let mut payloads = vec![]; + + for (i, g) in self.payload_gas.iter().enumerate() { + payloads.push(Box::new(RandomizedTransactionPayload { + package_id: self.basics_package_id.unwrap(), + shared_objects: self.shared_objects.clone(), + owned_object: self.owned_objects[i], + randomness_initial_shared_version: self.randomness_initial_shared_version.unwrap(), + transfer_to: self.transfer_to.unwrap(), + gas: g.clone(), + system_state_observer: system_state_observer.clone(), + })); + } + + payloads + .into_iter() + .map(|b| Box::::from(b)) + .collect() + } +} diff --git a/crates/sui-benchmark/src/workloads/randomness.rs b/crates/sui-benchmark/src/workloads/randomness.rs index 6a812b7b4d636..3294fcfb28d2c 100644 --- a/crates/sui-benchmark/src/workloads/randomness.rs +++ b/crates/sui-benchmark/src/workloads/randomness.rs @@ -79,7 +79,7 @@ impl RandomnessWorkloadBuilder { duration: Interval, group: u32, ) -> Option { - let target_qps = (workload_weight * target_qps as f32) as u64; + let target_qps = (workload_weight * target_qps as f32).ceil() as u64; let num_workers = (workload_weight * num_workers as f32).ceil() as u64; let max_ops = target_qps * in_flight_ratio; if max_ops == 0 || num_workers == 0 { diff --git a/crates/sui-benchmark/src/workloads/shared_counter.rs b/crates/sui-benchmark/src/workloads/shared_counter.rs index 5356d53b7184d..38f75fd9707d7 100644 --- a/crates/sui-benchmark/src/workloads/shared_counter.rs +++ b/crates/sui-benchmark/src/workloads/shared_counter.rs @@ -98,7 +98,7 @@ impl SharedCounterWorkloadBuilder { duration: Interval, group: u32, ) -> Option { - let target_qps = (workload_weight * target_qps as f32) as u64; + let target_qps = (workload_weight * target_qps as f32).ceil() as u64; let num_workers = (workload_weight * num_workers as f32).ceil() as u64; let max_ops = target_qps * in_flight_ratio; let shared_counter_ratio = diff --git a/crates/sui-benchmark/src/workloads/shared_object_deletion.rs b/crates/sui-benchmark/src/workloads/shared_object_deletion.rs index 552cd86c85fa8..6db465fa62340 100644 --- a/crates/sui-benchmark/src/workloads/shared_object_deletion.rs +++ b/crates/sui-benchmark/src/workloads/shared_object_deletion.rs @@ -143,7 +143,7 @@ impl SharedCounterDeletionWorkloadBuilder { duration: Interval, group: u32, ) -> Option { - let target_qps = (workload_weight * target_qps as f32) as u64; + let target_qps = (workload_weight * target_qps as f32).ceil() as u64; let num_workers = (workload_weight * num_workers as f32).ceil() as u64; let max_ops = target_qps * in_flight_ratio; let shared_counter_ratio = diff --git a/crates/sui-benchmark/src/workloads/transfer_object.rs b/crates/sui-benchmark/src/workloads/transfer_object.rs index 1835a5d0f17af..d02247154bcdf 100644 --- a/crates/sui-benchmark/src/workloads/transfer_object.rs +++ b/crates/sui-benchmark/src/workloads/transfer_object.rs @@ -108,7 +108,7 @@ impl TransferObjectWorkloadBuilder { duration: Interval, group: u32, ) -> Option { - let target_qps = (workload_weight * target_qps as f32) as u64; + let target_qps = (workload_weight * target_qps as f32).ceil() as u64; let num_workers = (workload_weight * num_workers as f32).ceil() as u64; let max_ops = target_qps * in_flight_ratio; if max_ops == 0 || num_workers == 0 { diff --git a/crates/sui-benchmark/src/workloads/workload.rs b/crates/sui-benchmark/src/workloads/workload.rs index d4c4cb507c270..9d9f766f48157 100644 --- a/crates/sui-benchmark/src/workloads/workload.rs +++ b/crates/sui-benchmark/src/workloads/workload.rs @@ -29,11 +29,15 @@ pub const STORAGE_COST_PER_COUNTER: u64 = 341 * 76 * 100; /// Used to estimate the budget required for each transaction. pub const ESTIMATED_COMPUTATION_COST: u64 = 1_000_000; -#[derive(Debug, EnumCountMacro, EnumIter, Clone, Copy)] +#[derive(Debug, EnumCountMacro, EnumIter, Clone, Copy, PartialEq)] pub enum ExpectedFailureType { Random = 0, InvalidSignature, // TODO: Add other failure types + + // This is not a failure type, but a placeholder for no failure. Marking no failure asserts that + // the transaction must succeed. + NoFailure, } impl TryFrom for ExpectedFailureType { @@ -41,7 +45,11 @@ impl TryFrom for ExpectedFailureType { fn try_from(value: u32) -> Result { match value { - 0 => Ok(rand::random()), + 0 => { + let mut rng = rand::thread_rng(); + let n = rng.gen_range(0..ExpectedFailureType::COUNT - 1); + Ok(ExpectedFailureType::iter().nth(n).unwrap()) + } _ => ExpectedFailureType::iter() .nth(value as usize) .ok_or_else(|| { diff --git a/crates/sui-benchmark/src/workloads/workload_configuration.rs b/crates/sui-benchmark/src/workloads/workload_configuration.rs index 9cbd18800ead3..d6d9b2f753b81 100644 --- a/crates/sui-benchmark/src/workloads/workload_configuration.rs +++ b/crates/sui-benchmark/src/workloads/workload_configuration.rs @@ -18,9 +18,11 @@ use tracing::info; use super::adversarial::{AdversarialPayloadCfg, AdversarialWorkloadBuilder}; use super::expected_failure::{ExpectedFailurePayloadCfg, ExpectedFailureWorkloadBuilder}; +use super::randomized_transaction::RandomizedTransactionWorkloadBuilder; use super::randomness::RandomnessWorkloadBuilder; use super::shared_object_deletion::SharedCounterDeletionWorkloadBuilder; +#[derive(Debug)] pub struct WorkloadWeights { pub shared_counter: u32, pub transfer_object: u32, @@ -30,6 +32,7 @@ pub struct WorkloadWeights { pub adversarial: u32, pub expected_failure: u32, pub randomness: u32, + pub randomized_transaction: u32, } pub struct WorkloadConfig { @@ -69,6 +72,7 @@ impl WorkloadConfiguration { adversarial, expected_failure, randomness, + randomized_transaction, shared_counter_hotness_factor, num_shared_counters, shared_counter_max_tip, @@ -102,6 +106,7 @@ impl WorkloadConfiguration { adversarial: adversarial[i], expected_failure: expected_failure[i], randomness: randomness[i], + randomized_transaction: randomized_transaction[i], }, adversarial_cfg: AdversarialPayloadCfg::from_str(&adversarial_cfg[i]) .unwrap(), @@ -193,6 +198,13 @@ impl WorkloadConfiguration { }: WorkloadConfig, system_state_observer: Arc, ) -> Vec> { + tracing::info!( + "Workload Configuration weights {:?} target_qps: {:?} num_workers: {:?} duration: {:?}", + weights, + target_qps, + num_workers, + duration + ); let total_weight = weights.shared_counter + weights.shared_deletion + weights.transfer_object @@ -200,7 +212,8 @@ impl WorkloadConfiguration { + weights.batch_payment + weights.adversarial + weights.randomness - + weights.expected_failure; + + weights.expected_failure + + weights.randomized_transaction; let reference_gas_price = system_state_observer.state.borrow().reference_gas_price; let mut workload_builders = vec![]; let shared_workload = SharedCounterWorkloadBuilder::from( @@ -288,6 +301,16 @@ impl WorkloadConfiguration { group, ); workload_builders.push(expected_failure_workload); + let randomized_transaction_workload = RandomizedTransactionWorkloadBuilder::from( + weights.randomized_transaction as f32 / total_weight as f32, + target_qps, + num_workers, + in_flight_ratio, + reference_gas_price, + duration, + group, + ); + workload_builders.push(randomized_transaction_workload); workload_builders } diff --git a/crates/sui-benchmark/tests/simtest.rs b/crates/sui-benchmark/tests/simtest.rs index 1335d1ede5a0f..de18808a3af94 100644 --- a/crates/sui-benchmark/tests/simtest.rs +++ b/crates/sui-benchmark/tests/simtest.rs @@ -576,7 +576,7 @@ mod test { info!("Simulated load config: {:?}", simulated_load_config); } - test_simulated_load_with_test_config(test_cluster, 50, simulated_load_config, None, None) + test_simulated_load_with_test_config(test_cluster, 180, simulated_load_config, None, None) .await; } @@ -993,6 +993,7 @@ mod test { shared_deletion_weight: u32, shared_counter_hotness_factor: u32, randomness_weight: u32, + randomized_transaction_weight: u32, num_shared_counters: Option, use_shared_counter_max_tip: bool, shared_counter_max_tip: u64, @@ -1011,6 +1012,7 @@ mod test { shared_deletion_weight: 1, shared_counter_hotness_factor: 50, randomness_weight: 1, + randomized_transaction_weight: 1, num_shared_counters: Some(1), use_shared_counter_max_tip: false, shared_counter_max_tip: 0, @@ -1069,7 +1071,7 @@ mod test { // The default test parameters are somewhat conservative in order to keep the running time // of the test reasonable in CI. - let target_qps = target_qps.unwrap_or(get_var("SIM_STRESS_TEST_QPS", 10)); + let target_qps = target_qps.unwrap_or(get_var("SIM_STRESS_TEST_QPS", 20)); let num_workers = num_workers.unwrap_or(get_var("SIM_STRESS_TEST_WORKERS", 10)); let in_flight_ratio = get_var("SIM_STRESS_TEST_IFR", 2); let batch_payment_size = get_var("SIM_BATCH_PAYMENT_SIZE", 15); @@ -1099,6 +1101,7 @@ mod test { randomness: config.randomness_weight, adversarial: adversarial_weight, expected_failure: config.expected_failure_weight, + randomized_transaction: config.randomized_transaction_weight, }; let workload_config = WorkloadConfig {