From 7f689e469f30cc9896f8d78de86c0350264c309e Mon Sep 17 00:00:00 2001 From: Mike Casale <46603283+mikewcasale@users.noreply.github.com> Date: Thu, 29 Feb 2024 18:57:08 -0800 Subject: [PATCH 01/41] Adds deterministic testing for Ethereum --- fastlane_bot/tests/deterministic/__init__.py | 0 .../_data/static_pool_data_testing.csv | 6 + .../tests/deterministic/dtest_constants.py | 122 ++++ .../tests/deterministic/dtest_manager.py | 653 ++++++++++++++++++ .../tests/deterministic/dtest_pool.py | 118 ++++ .../dtest_pool_params_builder.py | 193 ++++++ .../tests/deterministic/dtest_strategy.py | 129 ++++ .../tests/deterministic/dtest_token.py | 79 +++ .../tests/deterministic/dtest_tx_helper.py | 290 ++++++++ .../tests/deterministic/dtest_wallet.py | 33 + .../tests/deterministic/unit/fastlane_bot | 1 + .../unit/test_dtest_tx_helper.py | 42 ++ .../deterministic/unit/test_dtest_wallet.py | 71 ++ run_deterministic_tests.py | 330 +++++++++ 14 files changed, 2067 insertions(+) create mode 100644 fastlane_bot/tests/deterministic/__init__.py create mode 100644 fastlane_bot/tests/deterministic/_data/static_pool_data_testing.csv create mode 100644 fastlane_bot/tests/deterministic/dtest_constants.py create mode 100644 fastlane_bot/tests/deterministic/dtest_manager.py create mode 100644 fastlane_bot/tests/deterministic/dtest_pool.py create mode 100644 fastlane_bot/tests/deterministic/dtest_pool_params_builder.py create mode 100644 fastlane_bot/tests/deterministic/dtest_strategy.py create mode 100644 fastlane_bot/tests/deterministic/dtest_token.py create mode 100644 fastlane_bot/tests/deterministic/dtest_tx_helper.py create mode 100644 fastlane_bot/tests/deterministic/dtest_wallet.py create mode 120000 fastlane_bot/tests/deterministic/unit/fastlane_bot create mode 100644 fastlane_bot/tests/deterministic/unit/test_dtest_tx_helper.py create mode 100644 fastlane_bot/tests/deterministic/unit/test_dtest_wallet.py create mode 100644 run_deterministic_tests.py diff --git a/fastlane_bot/tests/deterministic/__init__.py b/fastlane_bot/tests/deterministic/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/fastlane_bot/tests/deterministic/_data/static_pool_data_testing.csv b/fastlane_bot/tests/deterministic/_data/static_pool_data_testing.csv new file mode 100644 index 000000000..efa943c2c --- /dev/null +++ b/fastlane_bot/tests/deterministic/_data/static_pool_data_testing.csv @@ -0,0 +1,6 @@ +cid,last_updated,last_updated_block,descr,pair_name,exchange_name,fee,fee_float,address,anchor,tkn0_address,tkn1_address,tkn0_decimals,tkn1_decimals,exchange_id,tkn0_symbol,tkn1_symbol,timestamp,tkn0_balance,tkn1_balance,liquidity,sqrt_price_q96,tick,tick_spacing,exchange,pool_type,tkn0_weight,tkn1_weight,tkn2_address,tkn2_decimals,tkn2_symbol,tkn2_balance,tkn2_weight,tkn3_address,tkn3_decimals,tkn3_symbol,tkn3_balance,tkn3_weight,tkn4_address,tkn4_decimals,tkn4_symbol,tkn4_balance,tkn4_weight,tkn5_address,tkn5_decimals,tkn5_symbol,tkn5_balance,tkn5_weight,tkn6_address,tkn6_decimals,tkn6_symbol,tkn6_balance,tkn6_weight,tkn7_address,tkn7_decimals,tkn7_symbol,tkn7_balance,tkn7_weight,test,exchange_type,pool_address,tkn0_setBalance,tkn1_setBalance,slots,param_lists,param_blockTimestampLast,param_blockTimestampLast_type,param_reserve0,param_reserve0_type,param_reserve1,param_reserve1_type,param_liquidity,param_liquidity_type,param_sqrtPriceX96,param_sqrtPriceX96_type,param_tick,param_tick_type,param_observationIndex,param_observationIndex_type,param_observationCardinality,param_observationCardinality_type,param_observationCardinalityNext,param_observationCardinalityNext_type,param_feeProtocol,param_feeProtocol_type,param_unlocked,param_unlocked_type +,,0,uniswap_v2 0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48/0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2 0.003,0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48/0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2,uniswap_v2,3000,0.003,0xB4e16d0168e52d35CaCD2c6185b44281Ec28C9Dc,,0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48,0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2,6,18,3,USDC,WETH,,,,,,,60,uniswap_v2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,uniswap_v2,0xB4e16d0168e52d35CaCD2c6185b44281Ec28C9Dc,49204613674917,21734090459912674200000,"[8,]","[['blockTimestampLast','reserve1','reserve0'],]",,uint32,49204613674917,uint112,21734090459912674200000,uint112,,,,,,,,,,,,,,,, +,,0,uniswap_v3 0x514910771AF9Ca656af840dff83E8264EcF986CA/0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2 0.003,0x514910771AF9Ca656af840dff83E8264EcF986CA/0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2,uniswap_v3,3000,0.003,0xa6Cc3C2531FdaA6Ae1A3CA84c2855806728693e8,,0x514910771AF9Ca656af840dff83E8264EcF986CA,0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2,18,18,4,LINK,WETH,,,,,,,60,uniswap_v3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,uniswap_v3,0xa6Cc3C2531FdaA6Ae1A3CA84c2855806728693e8,1312731411619255420525571,7101396323681156262130,"[4,0,]","[['liquidity'],['unlocked', 'feeProtocol', 'observationCardinalityNext', 'observationCardinality', 'observationIndex', 'tick', 'sqrtPriceX96']]",,,,,,,1456862313731161106039763,"uint128",6362445213301469813433370622,"uint160",-50441,"int24",85,"uint16",180,"uint16",180,"uint16",0,"uint8",True,bool +,,0,pancakeswap_v3 0x6B175474E89094C44Da98b954EedeAC495271d0F/0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48 0.0001,0x6B175474E89094C44Da98b954EedeAC495271d0F/0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48,pancakeswap_v3,100,0.0001,0xD9e497BD8f491fE163b42A62c296FB54CaEA74B7,,0x6B175474E89094C44Da98b954EedeAC495271d0F,0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48,18,6,10,DAI,USDC,0,0,0,0,0,0,1,pancakeswap_v3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,3,pancakeswap_v3,0xD9e497BD8f491fE163b42A62c296FB54CaEA74B7,13653990890660798693042,135975525713,"[4,0,1,]","[['liquidity'],['observationCardinalityNext', 'observationCardinality', 'observationIndex', 'tick', 'sqrtPriceX96'],['unlocked', 'feeProtocol']]",,,,,,,1275240832730323472063,"uint128",79228147574959555694268,"uint160",-276325,"int24",0,"uint16",1,"uint16",1,"uint16",216272100,"uint32",True,bool +,,0,pancakeswap_v2 0x2260FAC5E5542a773Aa44fBCfeDf7C193bc2C599/0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2 0.0025,0x2260FAC5E5542a773Aa44fBCfeDf7C193bc2C599/0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2,pancakeswap_v2,2500,0.0025,0x4AB6702B3Ed3877e9b1f203f90cbEF13d663B0e8,,0x2260FAC5E5542a773Aa44fBCfeDf7C193bc2C599,0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2,8,18,9,WBTC,WETH,0,0,0,,,,,pancakeswap_v2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,pancakeswap_v2,0x4AB6702B3Ed3877e9b1f203f90cbEF13d663B0e8,421419304,78852853048776778963,"[8,]","[['blockTimestampLast','reserve1','reserve0'],]",,uint32,421419304,uint112,78852853048776778963,uint112,,,,,,,,,,,,,,,, +,,0,uniswap_v3 0x6B175474E89094C44Da98b954EedeAC495271d0F/0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2 0.0005,0x6B175474E89094C44Da98b954EedeAC495271d0F/0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2,uniswap_v3,500,0.0005,0x60594a405d53811d3BC4766596EFD80fd545A270,,0x6B175474E89094C44Da98b954EedeAC495271d0F,0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2,18,18,4,DAI,WETH,,,,,,,10,uniswap_v3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,3,uniswap_v3,0x60594a405d53811d3BC4766596EFD80fd545A270,2850299295281281234085971,1147049345011454055669,"[4,0,]","[['liquidity'],['unlocked', 'feeProtocol', 'observationCardinalityNext', 'observationCardinality', 'observationIndex', 'tick', 'sqrtPriceX96']]",,,,,,,451613902936689743409876,"uint128",1659826732499374354385938036,"uint160",-77317,"int24",110,"uint16",180,"uint16",180,"uint16",0,"uint8",True,bool \ No newline at end of file diff --git a/fastlane_bot/tests/deterministic/dtest_constants.py b/fastlane_bot/tests/deterministic/dtest_constants.py new file mode 100644 index 000000000..0c57d8d7d --- /dev/null +++ b/fastlane_bot/tests/deterministic/dtest_constants.py @@ -0,0 +1,122 @@ +""" +This file contains constants used in the deterministic tests. + +(c) Copyright Bprotocol foundation 2024. +Licensed under MIT License. +""" +from dataclasses import dataclass + +from fastlane_bot.tools.cpc import T + +KNOWN_UNABLE_TO_DELETE = { + 68737038118029569619601670701217178714718: ("pDFS", "ETH"), +} +TEST_MODE_AMT = ( + 115792089237316195423570985008687907853269984665640564039457584007913129639935 +) +ETH_ADDRESS = "0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE" +SUPPORTED_EXCHANGES = ["uniswap_v2", "uniswap_v3", "pancakeswap_v2", "pancakeswap_v3"] +BNT_ADDRESS = "0x1F573D6Fb3F13d689FF844B4cE37794d79a7FF1C" +USDC_ADDRESS = "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48" +USDT_ADDRESS = "0xdAC17F958D2ee523a2206206994597C13D831ec7" +DEFAULT_GAS = 2000000 +DEFAULT_GAS_PRICE = 0 +DEFAULT_FROM_BLOCK = 1000000 +TENDERLY_RPC_KEY = "fb866397-29bd-4886-8406-a2cc7b7c5b1f" # https://virtual.mainnet.rpc.tenderly.co/9ea4ceb3-d0f5-4faf-959e-f51cf1f6b52b, from_block: 19325893, fb866397-29bd-4886-8406-a2cc7b7c5b1f +FILE_DATA_DIR = "fastlane_bot/data/blockchain_data" +TEST_FILE_DATA_DIR = "fastlane_bot/tests/deterministic/_data" +binance14 = "0x28C6c06298d514Db089934071355E5743bf21d60" +TOKENS_MODIFICATIONS = { + "0x0": { + "address": "0x5a3e6A77ba2f983eC0d371ea3B475F8Bc0811AD5", + "modifications": { + "before": { + "0x0000000000000000000000000000000000000000000000000000000000000006": "0x01", + "0x0000000000000000000000000000000000000000000000000000000000000007": "0x01", + "0x000000000000000000000000000000000000000000000000000000000000000d": "0x01", + }, + "after": { + "0x0000000000000000000000000000000000000000000000000000000000000006": "0x0000000000000000000000000000000000000000000000000000000000000019", + "0x0000000000000000000000000000000000000000000000000000000000000007": "0x0000000000000000000000000000000000000000000000000000000000000005", + "0x000000000000000000000000000000000000000000000000000000000000000d": "0x2386f26fc10000", + }, + "balance": 288551667147, + }, + "strategy_id": 9868188640707215440437863615521278132277, + "strategy_beneficiary": "0xe3d51681Dc2ceF9d7373c71D9b02c5308D852dDe", + }, + "PAXG": { + "address": "0x45804880De22913dAFE09f4980848ECE6EcbAf78", + "modifications": { + "before": { + "0x000000000000000000000000000000000000000000000000000000000000000d": "0x00", + }, + "after": { + "0x000000000000000000000000000000000000000000000000000000000000000d": "0x00000000000000000000000000000000000000000000000000000000000000c8", + }, + "balance": 395803389286127, + }, + "strategy_id": 15312706511442230855851857334429569515620, + "strategy_beneficiary": "0xFf365375777069eBd8Fa575635EB31a0787Afa6c", + }, +} + + +@dataclass +class TestCommandLineArgs: + """ + This class is used to mock the command line arguments for the main.py + """ + + cache_latest_only: str = "True" + backdate_pools: str = "True" + static_pool_data_filename: str = "static_pool_data_testing" + arb_mode: str = "multi_pairwise_all" + flashloan_tokens: str = ( + f"{T.LINK},{T.NATIVE_ETH},{T.BNT},{T.WBTC},{T.DAI},{T.USDC},{T.USDT},{T.WETH}" + ) + n_jobs: int = -1 + exchanges: str = "carbon_v1,bancor_v3,bancor_v2,bancor_pol,uniswap_v3,uniswap_v2,sushiswap_v2,balancer,pancakeswap_v2,pancakeswap_v3" + polling_interval: int = 0 + alchemy_max_block_fetch: int = 1 + reorg_delay: int = 0 + logging_path: str = "" + loglevel: str = "INFO" + use_cached_events: str = "False" + run_data_validator: str = "False" + randomizer: int = 1 + limit_bancor3_flashloan_tokens: str = "True" + default_min_profit_gas_token: str = "0.002" # "0.01" + timeout: int = None + target_tokens: str = None + replay_from_block: int = None + tenderly_fork_id: int = None + tenderly_event_exchanges: str = "pancakeswap_v2,pancakeswap_v3" + increment_time: int = 1 + increment_blocks: int = 1 + blockchain: str = "ethereum" + pool_data_update_frequency: int = -1 + use_specific_exchange_for_target_tokens: str = None + prefix_path: str = "" + version_check_frequency: int = 1 + self_fund: str = "False" + read_only: str = "False" + is_args_test: str = "False" + rpc_url: str = None + + @staticmethod + def args_to_command_line(args): + """ + Convert a TestCommandLineArgs instance to a list of command-line arguments. + + Args: + args: An instance of TestCommandLineArgs. + + Returns: + A list of command-line arguments. + """ + cmd_args = [] + for field, value in args.__dict__.items(): + if value is not None: # Only include fields that have a value + cmd_args.extend((f"--{field}", str(value))) + return cmd_args diff --git a/fastlane_bot/tests/deterministic/dtest_manager.py b/fastlane_bot/tests/deterministic/dtest_manager.py new file mode 100644 index 000000000..388b565e6 --- /dev/null +++ b/fastlane_bot/tests/deterministic/dtest_manager.py @@ -0,0 +1,653 @@ +""" +A module to manage Carbon strategies. + +(c) Copyright Bprotocol foundation 2024. +Licensed under MIT License. +""" +import argparse +import glob +import json +import os +import time +from typing import Dict + +import pandas as pd +import requests +from black import datetime +from eth_typing import Address, ChecksumAddress +from web3 import Web3 +from web3.contract import Contract + +from fastlane_bot.data.abi import CARBON_CONTROLLER_ABI +from fastlane_bot.tests.deterministic.dtest_constants import ( + DEFAULT_GAS, + DEFAULT_GAS_PRICE, + ETH_ADDRESS, + TEST_FILE_DATA_DIR, + TOKENS_MODIFICATIONS, + TestCommandLineArgs, +) +from fastlane_bot.tests.deterministic.dtest_strategy import TestStrategy + + +class TestManager: + """ + A class to manage Web3 contracts and Carbon strategies. + """ + + def __init__(self, args: argparse.Namespace): + """ + Initializes the TestManager. + + Args: + args (argparse.Namespace): The command-line arguments. + """ + self.w3 = Web3(Web3.HTTPProvider(args.rpc_url, {"timeout": 60})) + assert self.w3.is_connected(), "Web3 connection failed" + + multichain_addresses_path = os.path.normpath( + "fastlane_bot/data/multichain_addresses.csv" + ) + + # Get the Carbon Controller Address for the network + carbon_controller_address = self.get_carbon_controller_address( + multichain_addresses_path=multichain_addresses_path, network=args.network + ) + + # Initialize the Carbon Controller contract + carbon_controller = self.get_carbon_controller( + address=carbon_controller_address + ) + + self.carbon_controller = carbon_controller + + @property + def logs_path(self) -> str: + return os.path.normpath("./logs/*") + + def get_carbon_controller(self, address: Address or str) -> Contract: + """ + Gets the Carbon Controller contract on the given network. + + Args: + address (Address or str): The address. + + Returns: + Contract: The Carbon Controller contract. + """ + return self.w3.eth.contract(address=address, abi=CARBON_CONTROLLER_ABI) + + @staticmethod + def get_carbon_controller_address( + multichain_addresses_path: str, network: str + ) -> str: + """ + Gets the Carbon Controller contract address on the given network. + + Args: + multichain_addresses_path (str): The path to the multichain addresses file. + network (str): The network. + + Returns: + str: The Carbon Controller contract address. + """ + lookup_table = pd.read_csv(multichain_addresses_path) + return ( + lookup_table.query("exchange_name=='carbon_v1'") + .query(f"chain=='{network}'") + .factory_address.values[0] + ) + + @staticmethod + def create_new_testnet() -> tuple: + """ + Creates a new testnet on Tenderly. + + Returns: + tuple: The URI and the block number. + """ + + # Replace these variables with your actual data + ACCOUNT_SLUG = os.environ["TENDERLY_USER"] + PROJECT_SLUG = os.environ["TENDERLY_PROJECT"] + ACCESS_KEY = os.environ["TENDERLY_ACCESS_KEY"] + + url = f"https://api.tenderly.co/api/v1/account/{ACCOUNT_SLUG}/project/{PROJECT_SLUG}/testnet/container" + + headers = {"Content-Type": "application/json", "X-Access-Key": ACCESS_KEY} + + data = { + "slug": f"testing-api-endpoint-{datetime.now().strftime('%Y%m%d%H%M%S%f')}", + "displayName": "Automated Test Env", + "description": "", + "visibility": "TEAM", + "tags": {"purpose": "development"}, + "networkConfig": { + "networkId": "1", + "blockNumber": "latest", + "baseFeePerGas": "1", + "chainConfig": {"chainId": "1"}, + }, + "private": True, + "syncState": False, + } + + response = requests.post(url, headers=headers, data=json.dumps(data)) + + uri = f"{response.json()['container']['connectivityConfig']['endpoints'][0]['uri']}" + from_block = int(response.json()["container"]["networkConfig"]["blockNumber"]) + + return uri, from_block + + @staticmethod + def process_order_data(log_args: dict, order_key: str) -> dict: + """ + Transforms nested order data by appending a suffix to each key. + + Args: + log_args (dict): The log arguments. + order_key (str): The order key. + + Returns: + dict: The processed order data. + """ + if order_data := log_args.get(order_key): + suffix = order_key[-1] # Assumes order_key is either 'order0' or 'order1' + return {f"{key}{suffix}": value for key, value in order_data.items()} + return {} + + @staticmethod + def print_state_changes( + args: argparse.Namespace, + all_carbon_strategies: list, + deleted_strategies: list, + remaining_carbon_strategies: list, + ) -> None: + """ + Prints the state changes of Carbon strategies. + + Args: + args (argparse.Namespace): The command-line arguments. + all_carbon_strategies (list): The all carbon strategies. + deleted_strategies (list): The deleted strategies. + remaining_carbon_strategies (list): The remaining carbon strategies. + """ + args.logger.debug( + f"{len(all_carbon_strategies)} Carbon strategies have been created" + ) + args.logger.debug( + f"{len(deleted_strategies)} Carbon strategies have been deleted" + ) + args.logger.debug( + f"{len(remaining_carbon_strategies)} Carbon strategies remain" + ) + + def get_generic_events( + self, args: argparse.Namespace, event_name: str, from_block: int + ) -> pd.DataFrame: + """ + Fetches logs for a specified event from a smart contract. + + Args: + args (argparse.Namespace): The command-line arguments. + event_name (str): The event name. + from_block (int): The block number. + + Returns: + pd.DataFrame: The logs for the specified event. + """ + args.logger.debug( + f"Fetching logs for {event_name} event, from_block: {from_block}" + ) + try: + log_list = getattr(self.carbon_controller.events, event_name).get_logs( + fromBlock=from_block + ) + data = [] + for log in log_list: + log_data = { + "block_number": log["blockNumber"], + "transaction_hash": self.w3.to_hex(log["transactionHash"]), + **log["args"], + } + + # Process and update log_data for 'order0' and 'order1', if present + for order_key in ["order0", "order1"]: + if order_data := self.process_order_data(log["args"], order_key): + log_data.update(order_data) + del log_data[order_key] + + data.append(log_data) + + df = pd.DataFrame(data) + return ( + df.sort_values(by="block_number") + if "block_number" in df.columns + else df + ).reset_index(drop=True) + except Exception as e: + args.logger.debug( + f"Error fetching logs for {event_name} event: {e}, returning empty df" + ) + return pd.DataFrame({}) + + def get_state_of_carbon_strategies( + self, args: argparse.Namespace, from_block: int + ) -> tuple: + """ + Fetches the state of Carbon strategies. + + Args: + args (argparse.Namespace): The command-line arguments. + from_block (int): The block number. + + Returns: + tuple: The strategy created dataframe, the strategy deleted dataframe, and the remaining carbon strategies. + """ + strategy_created_df = self.get_generic_events( + args=args, event_name="StrategyCreated", from_block=from_block + ) + all_carbon_strategies = ( + [] + if strategy_created_df.empty + else [ + (strategy_created_df["id"][i], strategy_created_df["owner"][i]) + for i in strategy_created_df.index + ] + ) + strategy_deleted_df = self.get_generic_events( + args=args, event_name="StrategyDeleted", from_block=from_block + ) + deleted_strategies = ( + [] if strategy_deleted_df.empty else strategy_deleted_df["id"].to_list() + ) + remaining_carbon_strategies = [ + x for x in all_carbon_strategies if x[0] not in deleted_strategies + ] + self.print_state_changes( + args, all_carbon_strategies, deleted_strategies, remaining_carbon_strategies + ) + return strategy_created_df, strategy_deleted_df, remaining_carbon_strategies + + def modify_storage(self, w3: Web3, address: str, slot: str, value: str): + """Modify storage directly via Tenderly. + + Args: + w3 (Web3): The Web3 instance. + address (str): The address. + slot (str): The slot. + value (str): The value. + """ + params = [address, slot, value] + w3.provider.make_request(method="tenderly_setStorageAt", params=params) + + @staticmethod + def set_balance_via_faucet( + args: argparse.Namespace, + w3: Web3, + token_address: str, + amount_wei: int, + wallet: Address or ChecksumAddress, + retry_num=0, + ): + """ + Set the balance of a wallet via a faucet. + + Args: + args (argparse.Namespace): The command-line arguments. + w3 (Web3): The Web3 instance. + token_address (str): The token address. + amount_wei (int): The amount in wei. + wallet (Address or ChecksumAddress): The wallet address. + retry_num (int): The number of retries. + """ + token_address = w3.to_checksum_address(token_address) + wallet = w3.to_checksum_address(wallet) + if token_address in {ETH_ADDRESS}: + method = "tenderly_setBalance" + params = [[wallet], w3.to_hex(amount_wei)] + else: + method = "tenderly_setErc20Balance" + params = [token_address, wallet, w3.to_hex(amount_wei)] + + try: + w3.provider.make_request(method=method, params=params) + except requests.exceptions.HTTPError: + time.sleep(1) + if retry_num < 3: + args.logger.debug(f"Retrying faucet request for {token_address}") + TestManager.set_balance_via_faucet( + args, w3, token_address, amount_wei, wallet, retry_num + 1 + ) + + args.logger.debug(f"Reset Balance to {amount_wei}") + + def modify_token( + self, + args: argparse.Namespace, + token_address: str, + modifications: dict, + strategy_id: int, + strategy_beneficiary: Address, + ): + """General function to modify token parameters and handle deletion. + + Args: + args (argparse.Namespace): The command-line arguments. + token_address (str): The token address. + modifications (dict): The modifications to be made. + strategy_id (int): The strategy id. + strategy_beneficiary (Address): The strategy beneficiary. + """ + + # Modify the tax parameters + for slot, value in modifications["before"].items(): + self.modify_storage(self.w3, token_address, slot, value) + + # Ensure there is sufficient funds for withdrawal + self.set_balance_via_faucet( + args, + self.w3, + token_address, + modifications["balance"], + self.carbon_controller.address, + ) + + self.delete_strategy(strategy_id, strategy_beneficiary) + + # Reset the tax parameters to their original state + for slot, value in modifications["after"].items(): + self.modify_storage(self.w3, token_address, slot, value) + + # Empty out this token from CarbonController + self.set_balance_via_faucet( + args, self.w3, token_address, 0, self.carbon_controller.address + ) + + def modify_tokens_for_deletion(self, args: argparse.Namespace) -> None: + """Custom modifications to tokens to allow their deletion from Carbon. + + Args: + args (argparse.Namespace): The command-line arguments. + """ + for token_name, details in TOKENS_MODIFICATIONS.items(): + args.logger.debug(f"Modifying {token_name} token..., details: {details}") + self.modify_token( + args, + details["address"], + details["modifications"], + details["strategy_id"], + details["strategy_beneficiary"], + ) + args.logger.debug(f"Modification for {token_name} token completed.") + + def create_strategy(self, args: argparse.Namespace, strategy: TestStrategy) -> str: + """ + Creates a Carbon strategy. + + Args: + args (argparse.Namespace): The command-line arguments. + strategy (TestStrategy): The test strategy. + + Returns: + str: The transaction hash. + """ + tx_params = { + "from": strategy.wallet.address, + "nonce": strategy.wallet.nonce, + "gasPrice": DEFAULT_GAS_PRICE, + "gas": DEFAULT_GAS, + } + if strategy.value: + tx_params["value"] = strategy.value + + tx_hash = self.carbon_controller.functions.createStrategy( + strategy.token0.address, + strategy.token1.address, + ( + [strategy.y0, strategy.z0, strategy.A0, strategy.B0], + [strategy.y1, strategy.z1, strategy.A1, strategy.B1], + ), + ).transact(tx_params) + + tx_receipt = self.w3.eth.wait_for_transaction_receipt(tx_hash) + if tx_receipt.status != 1: + args.logger.debug("Creation Failed") + return None + else: + args.logger.debug("Successfully Created Strategy") + return self.w3.to_hex(tx_receipt.transactionHash) + + def delete_strategy(self, strategy_id: int, wallet: Address) -> int: + """ + Deletes a Carbon strategy. + + Args: + strategy_id (int): The strategy id. + wallet (Address): The wallet address. + + Returns: + int: The transaction receipt status. + """ + nonce = self.w3.eth.get_transaction_count(wallet) + tx_params = { + "from": wallet, + "nonce": nonce, + "gasPrice": DEFAULT_GAS_PRICE, + "gas": DEFAULT_GAS, + } + tx_hash = self.carbon_controller.functions.deleteStrategy(strategy_id).transact( + tx_params + ) + + tx_receipt = self.w3.eth.wait_for_transaction_receipt(tx_hash) + return tx_receipt.status + + def delete_all_carbon_strategies( + self, args: argparse.Namespace, carbon_strategy_id_owner_list: list + ) -> list: + """ + Deletes all Carbon strategies. + + Args: + args (argparse.Namespace): The command-line arguments. + carbon_strategy_id_owner_list (list): The carbon strategy id owner list. + + Returns: + list: The undeleted strategies. + """ + self.modify_tokens_for_deletion(args) + undeleted_strategies = [] + for strategy_id, owner in carbon_strategy_id_owner_list: + args.logger.debug("Attempt 1") + status = self.delete_strategy(strategy_id, owner) + if status == 0: + try: + strategy_info = self.carbon_controller.functions.strategy( + strategy_id + ).call() + current_owner = strategy_info[1] + try: + args.logger.debug("Attempt 2") + status = self.delete_strategy(strategy_id, current_owner) + if status == 0: + args.logger.debug( + f"Unable to delete strategy {strategy_id}" + ) + undeleted_strategies += [strategy_id] + except Exception as e: + args.logger.debug( + f"Strategy {strategy_id} not found - already deleted {e}" + ) + except Exception as e: + args.logger.debug( + f"Strategy {strategy_id} not found - already deleted {e}" + ) + elif status == 1: + args.logger.debug(f"Strategy {strategy_id} successfully deleted") + else: + args.logger.debug("Possible error") + return undeleted_strategies + + @staticmethod + def get_test_strategies(args: argparse.Namespace) -> dict: + """ + Gets test strategies from a JSON file. + """ + test_strategies_path = os.path.normpath( + f"{TEST_FILE_DATA_DIR}/test_strategies.json" + ) + with open(test_strategies_path) as file: + test_strategies = json.load(file)["test_strategies"] + args.logger.debug(f"{len(test_strategies.keys())} test strategies imported") + return test_strategies + + def append_strategy_ids( + self, args: argparse.Namespace, test_strategy_txhashs: dict, from_block: int + ) -> dict: + """ + Appends the strategy ids to the test strategies. + + Args: + args (argparse.Namespace): The command-line arguments. + test_strategy_txhashs (dict): The test strategy txhashs. + from_block (int): The block number. + + Returns: + dict: The test strategy txhashs with the strategy ids. + """ + args.logger.debug("\nAdd the strategy ids...") + + # Get the new state of the carbon strategies + ( + strategy_created_df, + strategy_deleted_df, + remaining_carbon_strategies, + ) = self.get_state_of_carbon_strategies(args, from_block) + + for i in test_strategy_txhashs: + try: + test_strategy_txhashs[i]["strategyid"] = strategy_created_df[ + strategy_created_df["transaction_hash"] + == test_strategy_txhashs[i]["txhash"] + ].id.values[0] + args.logger.debug(f"Added the strategy ids: {i}") + except Exception as e: + args.logger.debug(f"Add the strategy ids Error: {i}, {e}") + return test_strategy_txhashs + + @staticmethod + def write_strategy_txhashs_to_json(test_strategy_txhashs: dict): + """ + Writes the test strategy txhashs to a file. + + Args: + test_strategy_txhashs (dict): The test strategy txhashs. + """ + test_strategy_txhashs_path = os.path.normpath( + f"{TEST_FILE_DATA_DIR}/test_strategy_txhashs.json" + ) + with open(test_strategy_txhashs_path, "w") as f: + json.dump(test_strategy_txhashs, f) + f.close() + + @staticmethod + def get_strats_created_from_block(args: argparse.Namespace, w3: Web3) -> int: + """ + Gets the block number from which new strategies were created. + + Args: + args (argparse.Namespace): The command-line arguments. + + Returns: + int: The block number. + """ + strats_created_from_block = w3.eth.get_block_number() + args.logger.debug(f"strats_created_from_block: {strats_created_from_block}") + return strats_created_from_block + + def approve_and_create_strategies( + self, args: argparse.Namespace, test_strategies: dict, from_block: int + ) -> dict: + """ + Approves and creates test strategies. + + Args: + args (argparse.Namespace): The command-line arguments. + test_strategies (dict): The test strategies. + from_block (int): The block number. + + Returns: + dict: All the relevant test strategies + """ + test_strategy_txhashs: Dict[TestStrategy] or Dict = {} + for i, (key, arg) in enumerate(test_strategies.items()): + arg["w3"] = self.w3 + test_strategy = TestStrategy(**arg) + test_strategy.get_token_approval( + token_id=0, approval_address=self.carbon_controller.address + ) + test_strategy.get_token_approval( + token_id=1, approval_address=self.carbon_controller.address + ) + tx_hash = self.create_strategy(args, test_strategy) + test_strategy_txhashs[str(i + 1)] = {"txhash": tx_hash} + + test_strategy_txhashs = self.append_strategy_ids( + args, test_strategy_txhashs, from_block + ) + return test_strategy_txhashs + + @staticmethod + def overwrite_command_line_args(args: argparse.Namespace) -> TestCommandLineArgs: + """ + Overwrites the command-line arguments with the default main args. + + Args: + args (argparse.Namespace): The command-line arguments. + + Returns: + TestCommandLineArgs: The default main args. + """ + # Get the default main args + default_main_args = TestCommandLineArgs() + default_main_args.blockchain = args.network + default_main_args.arb_mode = args.arb_mode + default_main_args.timeout = args.timeout + default_main_args.rpc_url = args.rpc_url + return default_main_args + + def get_most_recent_pool_data_path(self, args: argparse.Namespace) -> str: + """ + Gets the path to the most recent pool data file. + + Args: + args (argparse.Namespace): The command-line arguments. + + Returns: + str: The path to the most recent pool data file. + """ + + # Use glob to list all directories + most_recent_log_folder = [ + f for f in glob.glob(self.logs_path) if os.path.isdir(f) + ][-1] + args.logger.debug(f"Accessing log folder {most_recent_log_folder}") + return os.path.join(most_recent_log_folder, "latest_pool_data.json") + + def delete_old_logs(self, args: argparse.Namespace): + """ + Deletes all files and directories in the logs directory. + + Args: + args (argparse.Namespace): The command-line arguments. + """ + logs = [f for f in glob.glob(self.logs_path) if os.path.isdir(f)] + for log in logs: + for root, dirs, files in os.walk(log, topdown=False): + for name in files: + os.remove(os.path.join(root, name)) + for name in dirs: + os.rmdir(os.path.join(root, name)) + os.rmdir(log) + args.logger.debug("Deleted old logs") diff --git a/fastlane_bot/tests/deterministic/dtest_pool.py b/fastlane_bot/tests/deterministic/dtest_pool.py new file mode 100644 index 000000000..74ccbc92f --- /dev/null +++ b/fastlane_bot/tests/deterministic/dtest_pool.py @@ -0,0 +1,118 @@ +""" +This module contains the Wallet class, which is a dataclass that represents a wallet on the given network. + +(c) Copyright Bprotocol foundation 2024. +Licensed under MIT License. +""" +import argparse +import ast +import os +from dataclasses import dataclass + +import pandas as pd +from web3 import Web3 +from web3.types import RPCEndpoint + +from fastlane_bot.tests.deterministic.dtest_constants import ( + SUPPORTED_EXCHANGES, + TEST_FILE_DATA_DIR, +) +from fastlane_bot.tests.deterministic.dtest_token import TestTokenBalance + + +@dataclass +class TestPool: + """ + This class is used to represent a pool on the given network. + """ + + exchange_type: str + pool_address: str + tkn0_address: str + tkn1_address: str + slots: str or list # List after __post_init__, str before + param_lists: str or list # List after __post_init__, str before + tkn0_setBalance: TestTokenBalance or int # TokenBalance after __post_init__, int before + tkn1_setBalance: TestTokenBalance or int # TokenBalance after __post_init__, int before + param_blockTimestampLast: int = None + param_blockTimestampLast_type: str = None + param_reserve0: int = None + param_reserve0_type: str = None + param_reserve1: int = None + param_reserve1_type: str = None + param_liquidity: int = None + param_liquidity_type: str = None + param_sqrtPriceX96: int = None + param_sqrtPriceX96_type: str = None + param_tick: int = None + param_tick_type: str = None + param_observationIndex: int = None + param_observationIndex_type: str = None + param_observationCardinality: int = None + param_observationCardinality_type: str = None + param_observationCardinalityNext: int = None + param_observationCardinalityNext_type: str = None + param_feeProtocol: int = None + param_feeProtocol_type: str = None + param_unlocked: int = None + param_unlocked_type: str = None + + def __post_init__(self): + self.slots = ast.literal_eval(self.slots) + self.param_lists = ast.literal_eval(self.param_lists) + + @staticmethod + def attributes(): + """ + Returns the attributes of the TestPool class. + """ + return list(TestPool.__dataclass_fields__.keys()) + + @property + def param_dict(self): + """ + Returns a dictionary mapping the slots to the param_lists. + """ + return dict(zip(self.slots, self.param_lists)) + + @property + def is_supported(self): + """ + Returns True if the pool is supported, otherwise False. + """ + return self.exchange_type in SUPPORTED_EXCHANGES + + def set_balance_via_faucet(self, args: argparse.Namespace, + w3: Web3, token_id: int): + """ + This method sets the balance of the given token to the given amount using the faucet. + + Args: + args: The command-line arguments. + w3: The Web3 instance. + token_id: The token id. + """ + token_address = self.tkn0_address if token_id == 0 else self.tkn1_address + amount_wei = self.tkn0_setBalance if token_id == 0 else self.tkn1_setBalance + token_balance = TestTokenBalance(token=token_address, balance=amount_wei) + params = token_balance.faucet_params(wallet_address=self.pool_address) + method_name = RPCEndpoint( + "tenderly_setBalance" + if token_balance.token.is_eth + else "tenderly_setErc20Balance" + ) + w3.provider.make_request(method=method_name, params=params) + token_balance.balance = amount_wei + if token_id == 0: + self.tkn0_setBalance = token_balance + else: + self.tkn1_setBalance = token_balance + args.logger.debug(f"Reset Balance to {amount_wei}") + + @staticmethod + def load_test_pools(): + # Import pool data + static_pool_data_testing_path = os.path.normpath( + f"{TEST_FILE_DATA_DIR}/static_pool_data_testing.csv" + ) + return pd.read_csv(static_pool_data_testing_path, dtype=str) diff --git a/fastlane_bot/tests/deterministic/dtest_pool_params_builder.py b/fastlane_bot/tests/deterministic/dtest_pool_params_builder.py new file mode 100644 index 000000000..18d698610 --- /dev/null +++ b/fastlane_bot/tests/deterministic/dtest_pool_params_builder.py @@ -0,0 +1,193 @@ +""" +This module is used to build the parameters for the test pool. It is used to build the parameters for the test pool and +encode them into a string that can be used to update the storage of the pool contract. + +(c) Copyright Bprotocol foundation 2024. +Licensed under MIT License. +""" +import re +from dataclasses import dataclass + +import eth_abi +from web3 import Web3 +from web3.types import RPCEndpoint + +from fastlane_bot.tests.deterministic.dtest_pool import TestPool + + +@dataclass +class TestPoolParam: + """ + This class is used to represent a parameter of the test pool. + """ + + type: str + value: any + + +class TestPoolParamsBuilder: + """ + This class is used to build the parameters for the test pool. + """ + + def __init__(self, w3: Web3): + self.w3 = w3 + + @staticmethod + def convert_to_bool(value: str or int) -> bool: + """ + This method is used to convert a string value to a boolean. + """ + if isinstance(value, str): + return value.lower() in ["true", "1"] + return bool(value) + + @staticmethod + def safe_int_conversion(value: any) -> int or None: + """ + This method is used to convert a value to an integer. + """ + try: + return int(value) + except (ValueError, TypeError): + print(f"Error converting {value} to int") + return None + + @staticmethod + def append_zeros(value: any, type_str: str) -> str: + """ + This method is used to append zeros to a value based on the type. + """ + result = None + if type_str == "bool": + result = "0001" if str(value).lower() in {"true", "1"} else "0000" + elif type_str == "int24": + long_hex = eth_abi.encode(["int24"], [value]).hex() + result = long_hex[-6:] + elif "int" in type_str: + try: + hex_value = hex(value)[2:] + length = int(re.search(r"\d+", type_str).group()) // 4 + result = "0" * (length - len(hex_value)) + hex_value + except Exception as e: + print(f"Error building append_zeros {str(e)}") + return result + + def build_type_val_dict( + self, pool: TestPool, param_list_single: list[str] + ) -> tuple: + """ + This method is used to build the type_val_dict and the encoded_params for the given pool. + """ + type_val_dict = {} + for param in param_list_single: + param_value = self.get_param_value(pool, param) + if param_value is not None: + type_val_dict[param] = TestPoolParam( + type=pool.__getattribute__(f"param_{param}_type") or "uint256", + value=param_value, + ) + + encoded_params = self.encode_params(type_val_dict, param_list_single) + return type_val_dict, encoded_params + + def get_param_value(self, pool: TestPool, param: str) -> int or bool: + """ + This method is used to get the value of the given parameter. + """ + if param == "blockTimestampLast": + return self.get_latest_block_timestamp() + elif param == "unlocked": + return self.convert_to_bool(pool.param_unlocked) + else: + return self.safe_int_conversion( + pool.__getattribute__(f"param_{param}") or 0 + ) + + def get_latest_block_timestamp(self): + """ + This method is used to get the latest block timestamp. + """ + try: + return int(self.w3.eth.get_block("latest")["timestamp"]) + except Exception as e: + print(f"Error fetching latest block timestamp: {e}") + return None + + def encode_params(self, type_val_dict: dict, param_list_single: list[str]) -> str: + """ + This method is used to encode the parameters into a string that can be used to update the storage of the pool + contract. + + Args: + type_val_dict (dict): The type value dictionary. + param_list_single (list): The list of parameters. + + Returns: + str: The encoded parameters. + """ + try: + result = "".join( + self.append_zeros(type_val_dict[param].value, type_val_dict[param].type) + for param in param_list_single + ) + return "0x" + "0" * (64 - len(result)) + result + except Exception as e: + print(f"Error encoding params: {e}, {type_val_dict}") + return None + + def get_update_params_dict(self, pool: TestPool) -> dict: + """ + This method is used to get the update parameters dictionary for the given pool. + + Args: + pool (TestPool): The test pool. + + Returns: + dict: The update parameters dictionary. + """ + params_dict = {} + for i in range(len(pool.slots)): + params_dict[pool.slots[i]] = { + "slot": "0x" + self.append_zeros(int(pool.slots[i]), "uint256") + } + type_val_dict, encoded_params = self.build_type_val_dict( + pool, param_list_single=pool.param_lists[i] + ) + + params_dict[pool.slots[i]]["type_val_dict"] = type_val_dict + params_dict[pool.slots[i]]["encoded_params"] = encoded_params + return params_dict + + def set_storage_at(self, pool_address: str, update_params_dict_single: dict): + method = RPCEndpoint("tenderly_setStorageAt") + self.w3.provider.make_request( + method=method, + params=[ + pool_address, + update_params_dict_single["slot"], + update_params_dict_single["encoded_params"], + ], + ) + print(f"[set_storage_at] {pool_address}, {update_params_dict_single['slot']}") + print( + f"[set_storage_at] Updated storage parameters for {pool_address} at slot {update_params_dict_single['slot']}" + ) + + @staticmethod + def update_pools_by_exchange(args, builder, pools, w3): + # Handle each exchange_type differently for the required updates + for pool in pools: + # Set balances on pool + pool.set_balance_via_faucet(args, w3, 0) + pool.set_balance_via_faucet(args, w3, 1) + + # Set storage parameters + update_params_dict = builder.get_update_params_dict(pool) + + # Update storage parameters + for slot, params in update_params_dict.items(): + builder.set_storage_at(pool.pool_address, params) + args.logger.debug( + f"Updated storage parameters for {pool.pool_address} at slot {slot}" + ) diff --git a/fastlane_bot/tests/deterministic/dtest_strategy.py b/fastlane_bot/tests/deterministic/dtest_strategy.py new file mode 100644 index 000000000..d67b2db00 --- /dev/null +++ b/fastlane_bot/tests/deterministic/dtest_strategy.py @@ -0,0 +1,129 @@ +""" +This file contains the Strategy class, which is used to represent a strategy in the deterministic tests. + +(c) Copyright Bprotocol foundation 2024. +Licensed under MIT License. +""" +import argparse +from dataclasses import dataclass + +from eth_typing import ChecksumAddress +from web3 import Web3 + +from fastlane_bot.data.abi import ERC20_ABI +from fastlane_bot.tests.deterministic.dtest_constants import ( + BNT_ADDRESS, + DEFAULT_GAS, + DEFAULT_GAS_PRICE, + TEST_MODE_AMT, + USDC_ADDRESS, + USDT_ADDRESS, +) +from fastlane_bot.tests.deterministic.dtest_token import TestToken +from fastlane_bot.tests.deterministic.dtest_wallet import TestWallet + + +@dataclass +class TestStrategy: + """ + A class to represent a strategy in the deterministic tests. + """ + + w3: Web3 + token0: TestToken + token1: TestToken + y0: int + z0: int + A0: int + B0: int + y1: int + z1: int + A1: int + B1: int + wallet: TestWallet + + @property + def id(self): + return self._id or None + + @id.setter + def id(self, id: int): + self._id = id + + def __post_init__(self): + self.token0 = TestToken(self.token0) + self.token0.contract = self.w3.eth.contract( + address=self.token0.address, abi=ERC20_ABI + ) + self.token1 = TestToken(self.token1) + self.token1.contract = self.w3.eth.contract( + address=self.token1.address, abi=ERC20_ABI + ) + self.wallet = TestWallet(self.w3, self.wallet) + + def get_token_approval( + self, args: argparse.Namespace, token_id: int, approval_address: ChecksumAddress + ) -> str: + """ + This method is used to get the token approval for the given token and approval address. + + Args: + args (argparse.Namespace): The command line arguments. + token_id (int): The token ID. Should be 0 or 1. + approval_address (ChecksumAddress): The approval address. + + Returns: + str: The transaction hash. + """ + token = self.token0 if token_id == 0 else self.token1 + if token.address in [ + BNT_ADDRESS, + USDC_ADDRESS, + USDT_ADDRESS, + ]: + function_call = token.contract.functions.approve( + approval_address, 0 + ).transact( + { + "gasPrice": DEFAULT_GAS_PRICE, + "gas": DEFAULT_GAS, + "from": self.wallet.address, + "nonce": self.wallet.nonce, + } + ) + tx_reciept = self.w3.eth.wait_for_transaction_receipt(function_call) + tx_hash = self.w3.to_hex(dict(tx_reciept)["transactionHash"]) + + if dict(tx_reciept)["status"] != 1: + args.logger.debug("Approval Failed") + else: + args.logger.debug("Successfully Approved for 0") + + args.logger.debug(f"tx_hash = {tx_hash}") + + function_call = token.contract.functions.approve( + approval_address, TEST_MODE_AMT + ).transact( + { + "gasPrice": DEFAULT_GAS_PRICE, + "gas": DEFAULT_GAS, + "from": self.wallet.address, + "nonce": self.wallet.nonce, + } + ) + tx_reciept = self.w3.eth.wait_for_transaction_receipt(function_call) + tx_hash = self.w3.to_hex(dict(tx_reciept)["transactionHash"]) + + if dict(tx_reciept)["status"] != 1: + args.logger.debug("Approval Failed") + else: + args.logger.debug("Successfully Approved Token for Unlimited") + + args.logger.debug(f"tx_hash = {tx_hash}") + return tx_hash + + @property + def value(self): + return ( + self.y0 if self.token0.is_eth else self.y1 if self.token1.is_eth else None + ) diff --git a/fastlane_bot/tests/deterministic/dtest_token.py b/fastlane_bot/tests/deterministic/dtest_token.py new file mode 100644 index 000000000..6a1dd5251 --- /dev/null +++ b/fastlane_bot/tests/deterministic/dtest_token.py @@ -0,0 +1,79 @@ +""" +Token class to store token address and contract object for interacting with the token on the blockchain. + +(c) Copyright Bprotocol foundation 2024. +Licensed under MIT License. +""" +from dataclasses import dataclass + +from eth_typing import Address +from web3 import Web3 +from web3.contract import Contract + +from fastlane_bot.tests.deterministic.dtest_constants import ETH_ADDRESS + + +@dataclass +class TestToken: + """ + A class to represent a token on the blockchain. + + Attributes: + address: str or Address + """ + + address: str or Address # Address after __post_init__, str before + + def __post_init__(self): + self.address = Web3.to_checksum_address(self.address) + self._contract = None + + @property + def contract(self): + return self._contract + + @contract.setter + def contract(self, contract: Contract): + self._contract = contract + + @property + def is_eth(self): + return self.address == ETH_ADDRESS + + +@dataclass +class TestTokenBalance: + """ + A class to represent the balance of a token in a wallet in the deterministic tests. + + Attributes: + token: TestToken or str + balance: int + """ + + token: TestToken or str # Token after __post_init__, str before + balance: int + + def __post_init__(self): + self.token = TestToken(self.token) + self.balance = int(self.balance) + + @property + def hex_balance(self): + return Web3.to_hex(self.balance) + + def faucet_params(self, wallet_address: str = None) -> list: + """ + This method is used to return the faucet parameters for the token balance. + + Args: + wallet_address: str + + Returns: + list: The faucet parameters. + """ + return ( + [[self.token.address], self.hex_balance] + if self.token.is_eth + else [self.token.address, wallet_address, self.hex_balance] + ) diff --git a/fastlane_bot/tests/deterministic/dtest_tx_helper.py b/fastlane_bot/tests/deterministic/dtest_tx_helper.py new file mode 100644 index 000000000..cea03f8c1 --- /dev/null +++ b/fastlane_bot/tests/deterministic/dtest_tx_helper.py @@ -0,0 +1,290 @@ +""" +This module contains the TxHelper class which is a utility class to scan the logs directory for successful transactions +and clean and extract the transaction data. + +(c) Copyright Bprotocol foundation 2024. +Licensed under MIT License. +""" +import argparse +import glob +import json +import logging +import os +import time + +from fastlane_bot.tests.deterministic.dtest_constants import ( + TEST_FILE_DATA_DIR, +) + + +class TestTxHelper: + """ + This is a utility class to scan the logs directory for successful transactions and clean and extract the + transaction data. + """ + + @staticmethod + def find_most_recent_log_folder(logs_path="./logs/*") -> str: + """Find the most recent log folder. + + Args: + logs_path (str): The path to the logs directory. Defaults to "./logs/*". + + Returns: + str: The most recent log folder. + + """ + log_folders = [f for f in glob.glob(logs_path) if os.path.isdir(f)] + return max(log_folders, key=os.path.getmtime) + + @staticmethod + def wait_for_file(file_path: str, logger: logging.Logger, timeout: int = 120, check_interval: int = 10) -> bool: + """Wait for a specific file to exist, with a timeout. + + Args: + file_path (str): The path to the file. + logger (logging.Logger): The logger. + timeout (int): The timeout in seconds. Defaults to 120. + check_interval (int): The check interval in seconds. Defaults to 10. + + """ + start_time = time.time() + while not os.path.exists(file_path): + if time.time() - start_time > timeout: + logger.debug("Timeout waiting for file.") + return False + logger.debug("File not found, waiting.") + time.sleep(check_interval) + logger.debug("File found.") + return True + + @staticmethod + def load_json_data(file_path: str) -> dict: + """Safely load JSON data from a file. + + Args: + file_path (str): The path to the file. + + Returns: + dict: The JSON data. + """ + with open(file_path, "r") as file: + return json.load(file) + + @staticmethod + def read_transaction_files(log_folder: str) -> list: + """Read all transaction files in a folder and return their content. + + Args: + log_folder (str): The path to the log folder. + + Returns: + list: A list of transaction data. + """ + tx_files = glob.glob(os.path.join(log_folder, "*.txt")) + transactions = [] + for tx_file in tx_files: + with open(tx_file, "r") as file: + transactions.append(file.read()) + return transactions + + def tx_scanner(self, args: argparse.Namespace) -> list: + """ + Scan for successful transactions in the most recent log folder. + + Args: + args (argparse.Namespace): The command-line arguments. + + Returns: + list: A list of successful transactions. + """ + most_recent_log_folder = self.find_most_recent_log_folder() + args.logger.debug(f"Accessing log folder {most_recent_log_folder}") + + pool_data_file = os.path.join(most_recent_log_folder, "latest_pool_data.json") + if self.wait_for_file(pool_data_file, args.logger): + pool_data = self.load_json_data(pool_data_file) + args.logger.debug(f"len(pool_data): {len(pool_data)}") + + transactions = self.read_transaction_files(most_recent_log_folder) + successful_txs = [tx for tx in transactions if "'status': 1" in tx] + args.logger.debug(f"Found {len(successful_txs)} successful transactions.") + + return successful_txs + + @staticmethod + def clean_tx_data(tx_data: dict) -> dict: + """ + This method takes a transaction data dictionary and removes the cid0 key from the trades. + + Args: + tx_data (dict): The transaction data. + + Returns: + dict: The cleaned transaction data. + """ + if not tx_data: + return tx_data + + for trade in tx_data["trades"]: + if trade["exchange"] == "carbon_v1" and "cid0" in trade: + del trade["cid0"] + return tx_data + + @staticmethod + def get_tx_data(strategy_id: int, txt_all_successful_txs: list) -> dict: + """ + This method takes a list of successful transactions and a strategy_id and returns the transaction data for the + given strategy_id. + + Args: + strategy_id (int): The strategy id. + txt_all_successful_txs (list): A list of successful transactions. + + Returns: + dict: The transaction data for the given strategy_id. + """ + for tx in txt_all_successful_txs: + if str(strategy_id) in tx: + return json.loads( + tx.split( + """ + +""" + )[-1] + ) + + @staticmethod + def load_json_file(file_name: str, args: argparse.Namespace) -> dict: + """ + This method loads a json file and returns the data as a dictionary. + + Args: + file_name (str): The name of the file. + args (argparse.Namespace): The command-line arguments. + + Returns: + dict: The data from the json file. + """ + file_path = ( + os.path.normpath(f"{TEST_FILE_DATA_DIR}/{file_name}") + if "/" not in file_name + else os.path.normpath(file_name) + ) + + try: + with open(file_path) as f: + data = json.load(f) + args.logger.debug(f"len({file_name})={len(data)}") + except FileNotFoundError: + data = {} + return data + + @staticmethod + def log_txs(tx_list: list, args: argparse.Namespace): + """ + This method logs the transactions in a list. + + Args: + tx_list (list): A list of transactions. + args (argparse.Namespace): The command-line arguments. + """ + for i, tx in enumerate(tx_list): + args.logger.debug(f"\nsuccessful_txs[{i}]: {tx}") + + def log_results(self, args: argparse.Namespace, actual_txs: list, expected_txs: dict, test_strategy_txhashs: dict) -> dict: + """ + Logs the results of the tests and returns a dictionary with the results. + + Args: + args (argparse.Namespace): The command-line arguments. + actual_txs (list): A list of actual transactions. + expected_txs (dict): A dictionary of expected transactions. + test_strategy_txhashs (dict): A dictionary of test strategy txhashs. + + Returns: + dict: A dictionary with the results of the tests. + """ + results_description = {} + all_tests_passed = True + + for test_id, strategy in test_strategy_txhashs.items(): + strategy_id = strategy.get("strategyid") + + # Failure case 1: strategyid missing in test_strategy_txhashs + if not strategy_id: + self.log_test_failure(test_id, "strategyid missing in test_strategy_txhashs", results_description) + continue + + tx_data = self.clean_tx_data(self.get_tx_data(strategy_id, actual_txs)) + + # Failure case 2: The test_id is not found in expected_txs + if test_id not in expected_txs["test_data"]: + self.log_test_failure(test_id, f"Test ID {test_id} not found in expected_txs", results_description, tx_data) + all_tests_passed = False + continue + + expected_test_data = expected_txs["test_data"][test_id] + + # Failure case 3: The tx_data does not match the expected_test_data + if tx_data != expected_test_data: + self.log_test_failure(test_id, "Data mismatch", results_description, tx_data, expected_test_data) + all_tests_passed = False + continue + + # Success case + results_description[test_id] = {"msg": f"Test {test_id} PASSED"} + + self.log_final_result(args, all_tests_passed) + return results_description + + @staticmethod + def log_test_failure(test_id: int, + reason: str, results_description: dict, + tx_data: dict = None, expected_data: dict = None): + """ + Logs a test failure. + + Args: + test_id (int): The test id. + reason (str): The reason for the failure. + results_description (dict): The dictionary with the results. + tx_data (dict): The transaction data. + expected_data (dict): The expected data. + """ + result = {"msg": f"Test {test_id} FAILED", "reason": reason} + if tx_data: + result["tx_data"] = tx_data + if expected_data: + result["expected_data"] = expected_data + results_description[test_id] = result + + @staticmethod + def log_final_result(args: argparse.Namespace, all_tests_passed: bool): + """ + Logs the final result of all tests. + + Args: + args (argparse.Namespace): The command-line arguments. + all_tests_passed (bool): Whether all tests passed. + """ + if all_tests_passed: + args.logger.info("ALL TESTS PASSED") + else: + args.logger.warning("SOME TESTS FAILED") + + def wait_for_txs(self, args: argparse.Namespace) -> dict: + """ + This method waits for the transactions to be executed and returns the test strategy txhashs. + + Args: + args (argparse.Namespace): The command-line arguments. + + Returns: + dict: The test strategy txhashs. + """ + test_strategy_txhashs = self.load_json_file("test_strategy_txhashs.json", args) + sleep_seconds = int(35 * len(test_strategy_txhashs.keys()) + 15) + args.logger.debug(f"sleep_seconds: {sleep_seconds}") + time.sleep(sleep_seconds) + return test_strategy_txhashs diff --git a/fastlane_bot/tests/deterministic/dtest_wallet.py b/fastlane_bot/tests/deterministic/dtest_wallet.py new file mode 100644 index 000000000..1f1942d27 --- /dev/null +++ b/fastlane_bot/tests/deterministic/dtest_wallet.py @@ -0,0 +1,33 @@ +""" +This module contains the Wallet class, which is a dataclass that represents a wallet on the Ethereum network. + +(c) Copyright Bprotocol foundation 2024. +Licensed under MIT License. +""" +from dataclasses import dataclass + +from eth_typing import Address +from web3 import Web3 + +from fastlane_bot.tests.deterministic.dtest_token import TestTokenBalance + + +@dataclass +class TestWallet: + """ + A class to represent a wallet on the Ethereum network. + """ + + w3: Web3 + address: str or Address # Address after __post_init__, str before + balances: list[ + TestTokenBalance or dict + ] = None # List of TokenBalances after __post_init__, list of dicts before + + def __post_init__(self): + self.address = Web3.to_checksum_address(self.address) + self.balances = [TestTokenBalance(**args) for args in self.balances or []] + + @property + def nonce(self): + return self.w3.eth.get_transaction_count(self.address) diff --git a/fastlane_bot/tests/deterministic/unit/fastlane_bot b/fastlane_bot/tests/deterministic/unit/fastlane_bot new file mode 120000 index 000000000..57929bed7 --- /dev/null +++ b/fastlane_bot/tests/deterministic/unit/fastlane_bot @@ -0,0 +1 @@ +../../../../fastlane_bot \ No newline at end of file diff --git a/fastlane_bot/tests/deterministic/unit/test_dtest_tx_helper.py b/fastlane_bot/tests/deterministic/unit/test_dtest_tx_helper.py new file mode 100644 index 000000000..fbd1349b5 --- /dev/null +++ b/fastlane_bot/tests/deterministic/unit/test_dtest_tx_helper.py @@ -0,0 +1,42 @@ +import json + +import pytest +from unittest.mock import patch, MagicMock + +from fastlane_bot.tests.deterministic.dtest_tx_helper import TestTxHelper + + +@pytest.fixture +def mock_logger(): + return MagicMock() + + +def test_find_most_recent_log_folder(mocker): + mocker.patch('glob.glob', return_value=['./logs/folder1', './logs/folder2']) + mocker.patch('os.path.isdir', side_effect=lambda x: True) + mocker.patch('os.path.getmtime', side_effect=lambda x: {'./logs/folder1': 1, './logs/folder2': 2}[x]) + + assert TestTxHelper.find_most_recent_log_folder() == './logs/folder2' + + +def test_wait_for_file_exists_before_timeout(mocker, mock_logger): + mocker.patch('os.path.exists', return_value=True) + assert TestTxHelper.wait_for_file("dummy_path", mock_logger) is True + mock_logger.debug.assert_called_with("File found.") + + +def test_wait_for_file_timeout(mocker, mock_logger): + mocker.patch('os.path.exists', return_value=False) + mocker.patch('time.time', side_effect=[0, 0, 121]) # Simulating timeout + assert TestTxHelper.wait_for_file("dummy_path", mock_logger, timeout=120) is False + mock_logger.debug.assert_called_with("Timeout waiting for file.") + +def test_load_json_data(tmpdir): + # Create a temporary JSON file + file = tmpdir.join("test.json") + data = {"key": "value"} + file.write(json.dumps(data)) + + # Test loading the JSON data + loaded_data = TestTxHelper.load_json_data(str(file)) + assert loaded_data == data diff --git a/fastlane_bot/tests/deterministic/unit/test_dtest_wallet.py b/fastlane_bot/tests/deterministic/unit/test_dtest_wallet.py new file mode 100644 index 000000000..c123509b9 --- /dev/null +++ b/fastlane_bot/tests/deterministic/unit/test_dtest_wallet.py @@ -0,0 +1,71 @@ +from dataclasses import dataclass +from unittest.mock import MagicMock, patch +import pytest +from web3 import Web3 + +from fastlane_bot.tests.deterministic.dtest_constants import ETH_ADDRESS +from fastlane_bot.tests.deterministic.dtest_token import TestToken, TestTokenBalance +from fastlane_bot.tests.deterministic.dtest_wallet import TestWallet + + +@pytest.fixture +def mock_web3(): + mock = MagicMock(spec=Web3) + eth_mock = MagicMock() + eth_mock.get_transaction_count = MagicMock(return_value=123) # Example nonce value + mock.eth = eth_mock + return mock + +def test_wallet_initialization_with_token_balances(mock_web3): + # Use a valid Ethereum address for the test token + valid_eth_address = '0x' + '1' * 40 # Example of a valid Ethereum address + valid_token_address = '0x' + '2' * 40 # Another example of a valid Ethereum address + + balances = [ + {'token': valid_eth_address, 'balance': 100}, # ETH + {'token': valid_token_address, 'balance': 200} # Another token + ] + wallet = TestWallet(w3=mock_web3, address=valid_eth_address, balances=balances) + + # Verify that the wallet's balances have been initialized correctly + assert all(isinstance(balance, TestTokenBalance) for balance in wallet.balances), "Balances should be instances of TestTokenBalance" + assert all(isinstance(balance.token, TestToken) for balance in wallet.balances), "Tokens should be instances of TestToken" + assert wallet.balances[0].token.address == Web3.to_checksum_address(valid_eth_address), "Token addresses should be checksummed" + assert wallet.balances[0].balance == 100, "Balance should be correctly set" + + +def test_testtoken_initialization_and_contract_assignment(mock_web3): + token_address = '0x' + '1' * 40 # Example token address + token = TestToken(token_address) + + assert token.address == Web3.to_checksum_address(token_address), "Token address should be checksummed" + + # Simulate assigning a contract + mock_contract = MagicMock() + token.contract = mock_contract + assert token.contract == mock_contract, "Contract assignment should work correctly" + +def test_testtokenbalance_initialization_and_properties(mock_web3): + token_address = '0x' + '1' * 40 + balance = 100 + token_balance = TestTokenBalance(token=token_address, balance=balance) + + assert isinstance(token_balance.token, TestToken), "Token should be an instance of TestToken" + assert token_balance.balance == balance, "Balance should be set correctly" + assert token_balance.hex_balance == Web3.to_hex(balance), "Hex balance should match" + +def test_faucet_params_for_eth_and_non_eth(mock_web3): + eth_token_balance = TestTokenBalance(token=ETH_ADDRESS, balance=100) + non_eth_token_balance = TestTokenBalance(token='0x' + '2' * 40, balance=200) + wallet_address = '0x' + '3' * 40 + + # ETH token + eth_params = eth_token_balance.faucet_params(wallet_address) + assert eth_params[0] == [ETH_ADDRESS], "ETH faucet params should include token address" + assert eth_params[1] == Web3.to_hex(100), "ETH faucet params should include hex balance" + + # Non-ETH token + non_eth_params = non_eth_token_balance.faucet_params(wallet_address) + assert non_eth_params[0] == non_eth_token_balance.token.address, "Non-ETH faucet params should include token address" + assert non_eth_params[1] == wallet_address, "Non-ETH faucet params should include wallet address" + assert non_eth_params[2] == Web3.to_hex(200), "Non-ETH faucet params should include hex balance" diff --git a/run_deterministic_tests.py b/run_deterministic_tests.py new file mode 100644 index 000000000..3d3dcfe8d --- /dev/null +++ b/run_deterministic_tests.py @@ -0,0 +1,330 @@ +""" +This script is used to run deterministic tests on the Fastlane Bot. + +The script is run from the command line with the following command: `python run_deterministic_tests.py --task +--rpc_url --network --arb_mode ` --timeout_minutes --from_block + --create_new_testnet + +The `--task` argument specifies the task to run. The options are: +- `set_test_state`: Set the test state based on the static_pool_data_testing.csv file. +- `get_carbon_strategies_and_delete`: Get the carbon strategies and delete them. +- `run_tests_on_mode`: Run tests on the specified arbitrage mode. +- `end_to_end`: Run all of the above tasks. + +The `--rpc_url` argument specifies the URL for the RPC endpoint. + +The `--network` argument specifies the network to test. The options are: +- `ethereum`: Ethereum network. + +The `--arb_mode` argument specifies the arbitrage mode to test. The options are: +- `single`: Single arbitrage mode. +- `multi`: Multi arbitrage mode. +- `triangle`: Triangle arbitrage mode. +- `multi_triangle`: Multi triangle arbitrage mode. + +The `--timeout_minutes` argument specifies the timeout for the tests (in minutes). + +The `--from_block` argument specifies the block number to start from. + +The `--create_new_testnet` argument specifies whether to create a new testnet. The options are: +- `True`: Create a new testnet. +- `False`: Do not create a new testnet. + +The script uses the `fastlane_bot/tests/deterministic/dtest_constants.py` file to get the constants used in the tests. + +The script uses the `fastlane_bot/tests/deterministic/utils.py` file to get the utility functions used in the tests. + +All data used in the tests is stored in the `fastlane_bot/tests/deterministic/_data` directory. + +Note: This script uses the function `get_default_main_args` which returns the default command line arguments for the +`main` function in the `main.py` file. If these arguments change in main.py then they should be updated in the +`get_default_main_args` function as well. + +(c) Copyright Bprotocol foundation 2024. +Licensed under MIT License. +""" +import argparse +import logging +import os +import subprocess +import time +from typing import Dict + +from web3 import Web3 + +from fastlane_bot.tests.deterministic.dtest_constants import ( + DEFAULT_FROM_BLOCK, + KNOWN_UNABLE_TO_DELETE, + TENDERLY_RPC_KEY, + TestCommandLineArgs, +) +from fastlane_bot.tests.deterministic.dtest_manager import TestManager +from fastlane_bot.tests.deterministic.dtest_pool import TestPool +from fastlane_bot.tests.deterministic.dtest_pool_params_builder import ( + TestPoolParamsBuilder, +) +from fastlane_bot.tests.deterministic.dtest_tx_helper import TestTxHelper + + +def get_logger(args: argparse.Namespace) -> logging.Logger: + """ + Get the logger for the script. + """ + logger = logging.getLogger(__name__) + logger.setLevel(args.loglevel) + logger.handlers.clear() # Clear existing handlers to avoid duplicate logging + + # Create console handler + ch = logging.StreamHandler() + ch.setLevel(args.loglevel) + + # Create formatter and add it to the handlers + formatter = logging.Formatter( + "%(asctime)s - %(name)s - %(levelname)s - %(message)s" + ) + ch.setFormatter(formatter) + + # Add the console handler to the logger + logger.addHandler(ch) + + return logger + + +def set_test_state_task(w3: Web3): + """ + Sets the test state based on the static_pool_data_testing.csv file. + + Args: + w3: Web3 instance + """ + args.logger.info("\nRunning set_test_state_task...") + + test_pools = TestPool.load_test_pools() + + pools = [ + TestPool(**test_pools_row[TestPool.attributes()].to_dict()) + for index, test_pools_row in test_pools.iterrows() + ] + pools = [pool for pool in pools if pool.is_supported] + builder = TestPoolParamsBuilder(w3) + builder.update_pools_by_exchange(args, builder, pools, w3) + + +def get_carbon_strategies_and_delete_task( + test_manager: TestManager, args: argparse.Namespace +): + """ + Get the carbon strategies and delete them. + + Args: + test_manager: TestManager, the test manager + args: argparse.Namespace, the command line arguments + """ + args.logger.info("\nRunning get_carbon_strategies_and_delete_task...") + + # Get the state of the carbon strategies + ( + strategy_created_df, + strategy_deleted_df, + remaining_carbon_strategies, + ) = test_manager.get_state_of_carbon_strategies(args, args.from_block) + + # takes about 4 minutes per 100 strategies, so 450 ~ 18 minutes + undeleted_strategies = test_manager.delete_all_carbon_strategies( + args, remaining_carbon_strategies + ) + + # These strategies cannot be deleted on Ethereum + assert all( + x in KNOWN_UNABLE_TO_DELETE for x in undeleted_strategies + ), f"Strategies not deleted that are unknown: {undeleted_strategies}" + + +def run_tests_on_mode_task( + args: argparse.Namespace, + test_manager: TestManager, + test_strategies: Dict, +): + """ + Run tests on the specified arbitrage mode. + + Args: + args: argparse.Namespace, the command line arguments + test_manager: TestManager, the test manager + test_strategies: Dict, the test strategies + """ + args.logger.info("\nRunning run_tests_on_mode_task...") + + # Get the default main.py CL args, then overwrite based on the current command line args + default_main_args = test_manager.overwrite_command_line_args(args) + + # Print the default main args + args.logger.debug(f"command-line args: {default_main_args}") + + # Run the main.py script with the default main args + cmd_args = ["python", "main.py"] + TestCommandLineArgs.args_to_command_line( + default_main_args + ) + proc = subprocess.Popen(cmd_args) + time.sleep(3) + most_recent_pool_data_path = test_manager.get_most_recent_pool_data_path(args) + + # Wait for the main.py script to create the latest_pool_data.json file + while not os.path.exists(most_recent_pool_data_path): + time.sleep(3) + args.logger.debug("Waiting for pool data...") + + strats_created_from_block = test_manager.get_strats_created_from_block( + args, test_manager.w3 + ) + + # Approve and create the strategies + test_strategy_txhashs = test_manager.approve_and_create_strategies( + args, test_strategies, strats_created_from_block + ) + + # Write the strategy txhashs to a json file + test_manager.write_strategy_txhashs_to_json(test_strategy_txhashs) + + # Run the results crosscheck task + run_results_crosscheck_task(args, proc) + + +def run_results_crosscheck_task(args, proc: subprocess.Popen): + """ + Run the results crosscheck task. + + Args: + args: argparse.Namespace, the command line arguments + proc: subprocess.Popen, the process + """ + args.logger.info("\nRunning run_results_crosscheck_task...") + + # Initialize the tx helper + tx_helper = TestTxHelper() + + # Wait for the transactions to be completed + test_strategy_txhashs = tx_helper.wait_for_txs(args) + + # Scan for successful transactions on Tenderly which are marked by status=1 + actual_txs = tx_helper.tx_scanner(args) + # tx_helper.log_txs(actual_txs, args) + expected_txs = tx_helper.load_json_file("test_results.json", args) + + results_description = tx_helper.log_results( + args, actual_txs, expected_txs, test_strategy_txhashs + ) + proc.terminate() + for k, v in results_description.items(): + args.logger.info(f"{k}: {v}") + + +def main(args: argparse.Namespace): + """ + Main function for the script. Runs the specified task based on the command line arguments. + + Args: + args: argparse.Namespace, the command line arguments + """ + + # Set up the logger + args.logger = get_logger(args) + args.logger.info(f"Running task: {args.task}") + + # Set the timeout in seconds + args.timeout = args.timeout_minutes * 60 + + if str(args.create_new_testnet).lower() == "true": + uri, from_block = TestManager.create_new_testnet() + args.rpc_url = uri + + # Initialize the Web3 Manager + test_manager = TestManager(args=args) + test_manager.delete_old_logs(args) + + if args.task == "set_test_state": + set_test_state_task(test_manager.w3) + elif args.task == "get_carbon_strategies_and_delete": + get_carbon_strategies_and_delete_task(test_manager, args) + elif args.task == "run_tests_on_mode": + _extracted_task_handling(test_manager, args) + elif args.task == "end_to_end": + get_carbon_strategies_and_delete_task(test_manager, args) + set_test_state_task(test_manager.w3) + _extracted_task_handling(test_manager, args) + else: + raise ValueError(f"Task {args.task} not recognized") + + +def _extracted_task_handling(test_manager: TestManager, args: argparse.Namespace): + """ + Extracted task handling. + """ + test_strategies = test_manager.get_test_strategies(args) + run_tests_on_mode_task(args, test_manager, test_strategies) + + +if __name__ == "__main__": + # Parse the command line arguments + parser = argparse.ArgumentParser(description="Fastlane Bot") + parser.add_argument( + "--task", + default="update_pool_params", + type=str, + choices=[ + "set_test_state", + "get_carbon_strategies_and_delete", + "run_tests_on_mode", + "run_results_crosscheck", + "end_to_end", + ], + help="Task to run", + ) + parser.add_argument( + "--rpc_url", + default=f"https://virtual.mainnet.rpc.tenderly.co/{TENDERLY_RPC_KEY}", + type=str, + help="URL for the RPC endpoint", + ) + parser.add_argument( + "--network", + default="ethereum", + type=str, + help="Network to test", + choices=["ethereum"], # TODO: add support for other networks + ) + parser.add_argument( + "--arb_mode", + default="multi", + type=str, + choices=["single", "multi", "triangle", "multi_triangle"], + help="Arbitrage mode to test", + ) + parser.add_argument( + "--timeout_minutes", + default=10, + type=int, + help="Timeout for the tests (in minutes)", + ) + parser.add_argument( + "--from_block", + default=DEFAULT_FROM_BLOCK, + type=int, + help="Replay from block", + ) + parser.add_argument( + "--create_new_testnet", + default="False", + type=str, + help="Create a new testnet", + ) + parser.add_argument( + "--loglevel", + default="DEBUG", + type=str, + help="Logging level", + choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], + ) + + args = parser.parse_args() + main(args) From 608e03b773bf8f96169c8ce007a317e17c640ea2 Mon Sep 17 00:00:00 2001 From: Mike Casale <46603283+mikewcasale@users.noreply.github.com> Date: Thu, 29 Feb 2024 19:04:18 -0800 Subject: [PATCH 02/41] adding test data --- .../deterministic/_data/test_results.json | 160 ++++++++++++++++++ .../deterministic/_data/test_strategies.json | 56 ++++++ 2 files changed, 216 insertions(+) create mode 100644 fastlane_bot/tests/deterministic/_data/test_results.json create mode 100644 fastlane_bot/tests/deterministic/_data/test_strategies.json diff --git a/fastlane_bot/tests/deterministic/_data/test_results.json b/fastlane_bot/tests/deterministic/_data/test_results.json new file mode 100644 index 000000000..5d01b25a3 --- /dev/null +++ b/fastlane_bot/tests/deterministic/_data/test_results.json @@ -0,0 +1,160 @@ +{ + "test_data": { + "1": { + "type": "multi", + "profit_gas_token": 0.113, + "profit_usd": 255.8626, + "flashloan": [ + { + "token": "WETH", + "amount": 1.1466, + "profit": 0.113 + } + ], + "trades": [ + { + "trade_index": 0, + "exchange": "uniswap_v2", + "tkn_in": { + "WETH": "0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2" + }, + "amount_in": 1.1466, + "tkn_out": { + "USDC": "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48" + }, + "amt_out": 2587.6497, + "cid0": "fe98c78ce1" + }, + { + "trade_index": 1, + "exchange": "carbon_v1", + "tkn_in": { + "USDC": "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48" + }, + "amount_in": 2587.6497, + "tkn_out": { + "WETH": "0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2" + }, + "amt_out": 1.2596 + } + ] + }, + "2": { + "type": "multi", + "profit_gas_token": 0.1817, + "profit_usd": 411.3437, + "flashloan": [ + { + "token": "LINK", + "amount": 51.3308, + "profit": 28.1742 + } + ], + "trades": [ + { + "trade_index": 0, + "exchange": "uniswap_v3", + "tkn_in": { + "LINK": "0x514910771AF9Ca656af840dff83E8264EcF986CA" + }, + "amount_in": 51.3308, + "tkn_out": { + "WETH": "0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2" + }, + "amt_out": 0.33, + "cid0": "b800399517" + }, + { + "trade_index": 1, + "exchange": "carbon_v1", + "tkn_in": { + "WETH": "0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2" + }, + "amount_in": 0.33, + "tkn_out": { + "LINK": "0x514910771AF9Ca656af840dff83E8264EcF986CA" + }, + "amt_out": 79.505 + } + ] + }, + "3": { + "type": "multi", + "profit_gas_token": 0.1293, + "profit_usd": 292.8006, + "flashloan": [ + { + "token": "USDC", + "amount": 2697.3, + "profit": 292.8006 + } + ], + "trades": [ + { + "trade_index": 0, + "exchange": "carbon_v1", + "tkn_in": { + "USDC": "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48" + }, + "amount_in": 2697.3, + "tkn_out": { + "DAI": "0x6B175474E89094C44Da98b954EedeAC495271d0F" + }, + "amt_out": 2990.7069 + }, + { + "trade_index": 1, + "exchange": "pancakeswap_v3", + "tkn_in": { + "DAI": "0x6B175474E89094C44Da98b954EedeAC495271d0F" + }, + "amount_in": 2990.7069, + "tkn_out": { + "USDC": "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48" + }, + "amt_out": 2990.1006, + "cid0": "d198cf907e" + } + ] + }, + "4": { + "type": "multi", + "profit_gas_token": 0.0198, + "profit_usd": 44.7633, + "flashloan": [ + { + "token": "WETH", + "amount": 1.3329, + "profit": 0.0198 + } + ], + "trades": [ + { + "trade_index": 0, + "exchange": "carbon_v1", + "tkn_in": { + "WETH": "0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2" + }, + "amount_in": 1.3329, + "tkn_out": { + "WBTC": "0x2260FAC5E5542a773Aa44fBCfeDf7C193bc2C599" + }, + "amt_out": 0.0737 + }, + { + "trade_index": 1, + "exchange": "pancakeswap_v2", + "tkn_in": { + "WBTC": "0x2260FAC5E5542a773Aa44fBCfeDf7C193bc2C599" + }, + "amount_in": 0.0737, + "tkn_out": { + "WETH": "0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2" + }, + "amt_out": 1.3526, + "cid0": "66c0a836b8" + } + ] + } + } +} \ No newline at end of file diff --git a/fastlane_bot/tests/deterministic/_data/test_strategies.json b/fastlane_bot/tests/deterministic/_data/test_strategies.json new file mode 100644 index 000000000..b60e11464 --- /dev/null +++ b/fastlane_bot/tests/deterministic/_data/test_strategies.json @@ -0,0 +1,56 @@ +{ + "test_strategies": { + "1": { + "token0": "0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE", + "token1": "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48", + "y0": 1267460000029073000, + "z0": 1267460000029073000, + "A0": 0, + "B0": 4411844567835374, + "y1": 7634017977, + "z1": 7649999999, + "A1": 0, + "B1": 11433567603, + "wallet": "0x28C6c06298d514Db089934071355E5743bf21d60" + }, + "2": { + "token0": "0x514910771AF9Ca656af840dff83E8264EcF986CA", + "token1": "0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE", + "y0": 80000000000000000000, + "z0": 80000000000000000000, + "A0": 0, + "B0": 1399247190132272, + "y1": 0, + "z1": 0, + "A1": 0, + "B1": 17102934719008, + "wallet": "0x28C6c06298d514Db089934071355E5743bf21d60" + }, + "3": { + "token0": "0x6B175474E89094C44Da98b954EedeAC495271d0F", + "token1": "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48", + "y0": 3000000000000000000000, + "z0": 3000000000000000000000, + "A0": 0, + "B0": 6052452418541427, + "y1": 0, + "z1": 0, + "A1": 0, + "B1": 0, + "wallet": "0x28C6c06298d514Db089934071355E5743bf21d60" + }, + "4": { + "token0": "0x2260FAC5E5542a773Aa44fBCfeDf7C193bc2C599", + "token1": "0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE", + "y0": 100000000, + "z0": 100000000, + "A0": 17694999, + "B0": 645747883, + "y1": 0, + "z1": 0, + "A1": 0, + "B1": 0, + "wallet": "0x28C6c06298d514Db089934071355E5743bf21d60" + } + } +} \ No newline at end of file From d250e3b586ee899c8cd75ae34bf80dbcedd8d7f0 Mon Sep 17 00:00:00 2001 From: Mike Casale <46603283+mikewcasale@users.noreply.github.com> Date: Thu, 29 Feb 2024 19:06:21 -0800 Subject: [PATCH 03/41] Create static_pool_data_testing.csv --- .../blockchain_data/ethereum/static_pool_data_testing.csv | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 fastlane_bot/data/blockchain_data/ethereum/static_pool_data_testing.csv diff --git a/fastlane_bot/data/blockchain_data/ethereum/static_pool_data_testing.csv b/fastlane_bot/data/blockchain_data/ethereum/static_pool_data_testing.csv new file mode 100644 index 000000000..efa943c2c --- /dev/null +++ b/fastlane_bot/data/blockchain_data/ethereum/static_pool_data_testing.csv @@ -0,0 +1,6 @@ +cid,last_updated,last_updated_block,descr,pair_name,exchange_name,fee,fee_float,address,anchor,tkn0_address,tkn1_address,tkn0_decimals,tkn1_decimals,exchange_id,tkn0_symbol,tkn1_symbol,timestamp,tkn0_balance,tkn1_balance,liquidity,sqrt_price_q96,tick,tick_spacing,exchange,pool_type,tkn0_weight,tkn1_weight,tkn2_address,tkn2_decimals,tkn2_symbol,tkn2_balance,tkn2_weight,tkn3_address,tkn3_decimals,tkn3_symbol,tkn3_balance,tkn3_weight,tkn4_address,tkn4_decimals,tkn4_symbol,tkn4_balance,tkn4_weight,tkn5_address,tkn5_decimals,tkn5_symbol,tkn5_balance,tkn5_weight,tkn6_address,tkn6_decimals,tkn6_symbol,tkn6_balance,tkn6_weight,tkn7_address,tkn7_decimals,tkn7_symbol,tkn7_balance,tkn7_weight,test,exchange_type,pool_address,tkn0_setBalance,tkn1_setBalance,slots,param_lists,param_blockTimestampLast,param_blockTimestampLast_type,param_reserve0,param_reserve0_type,param_reserve1,param_reserve1_type,param_liquidity,param_liquidity_type,param_sqrtPriceX96,param_sqrtPriceX96_type,param_tick,param_tick_type,param_observationIndex,param_observationIndex_type,param_observationCardinality,param_observationCardinality_type,param_observationCardinalityNext,param_observationCardinalityNext_type,param_feeProtocol,param_feeProtocol_type,param_unlocked,param_unlocked_type +,,0,uniswap_v2 0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48/0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2 0.003,0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48/0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2,uniswap_v2,3000,0.003,0xB4e16d0168e52d35CaCD2c6185b44281Ec28C9Dc,,0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48,0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2,6,18,3,USDC,WETH,,,,,,,60,uniswap_v2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,uniswap_v2,0xB4e16d0168e52d35CaCD2c6185b44281Ec28C9Dc,49204613674917,21734090459912674200000,"[8,]","[['blockTimestampLast','reserve1','reserve0'],]",,uint32,49204613674917,uint112,21734090459912674200000,uint112,,,,,,,,,,,,,,,, +,,0,uniswap_v3 0x514910771AF9Ca656af840dff83E8264EcF986CA/0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2 0.003,0x514910771AF9Ca656af840dff83E8264EcF986CA/0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2,uniswap_v3,3000,0.003,0xa6Cc3C2531FdaA6Ae1A3CA84c2855806728693e8,,0x514910771AF9Ca656af840dff83E8264EcF986CA,0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2,18,18,4,LINK,WETH,,,,,,,60,uniswap_v3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,uniswap_v3,0xa6Cc3C2531FdaA6Ae1A3CA84c2855806728693e8,1312731411619255420525571,7101396323681156262130,"[4,0,]","[['liquidity'],['unlocked', 'feeProtocol', 'observationCardinalityNext', 'observationCardinality', 'observationIndex', 'tick', 'sqrtPriceX96']]",,,,,,,1456862313731161106039763,"uint128",6362445213301469813433370622,"uint160",-50441,"int24",85,"uint16",180,"uint16",180,"uint16",0,"uint8",True,bool +,,0,pancakeswap_v3 0x6B175474E89094C44Da98b954EedeAC495271d0F/0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48 0.0001,0x6B175474E89094C44Da98b954EedeAC495271d0F/0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48,pancakeswap_v3,100,0.0001,0xD9e497BD8f491fE163b42A62c296FB54CaEA74B7,,0x6B175474E89094C44Da98b954EedeAC495271d0F,0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48,18,6,10,DAI,USDC,0,0,0,0,0,0,1,pancakeswap_v3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,3,pancakeswap_v3,0xD9e497BD8f491fE163b42A62c296FB54CaEA74B7,13653990890660798693042,135975525713,"[4,0,1,]","[['liquidity'],['observationCardinalityNext', 'observationCardinality', 'observationIndex', 'tick', 'sqrtPriceX96'],['unlocked', 'feeProtocol']]",,,,,,,1275240832730323472063,"uint128",79228147574959555694268,"uint160",-276325,"int24",0,"uint16",1,"uint16",1,"uint16",216272100,"uint32",True,bool +,,0,pancakeswap_v2 0x2260FAC5E5542a773Aa44fBCfeDf7C193bc2C599/0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2 0.0025,0x2260FAC5E5542a773Aa44fBCfeDf7C193bc2C599/0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2,pancakeswap_v2,2500,0.0025,0x4AB6702B3Ed3877e9b1f203f90cbEF13d663B0e8,,0x2260FAC5E5542a773Aa44fBCfeDf7C193bc2C599,0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2,8,18,9,WBTC,WETH,0,0,0,,,,,pancakeswap_v2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,pancakeswap_v2,0x4AB6702B3Ed3877e9b1f203f90cbEF13d663B0e8,421419304,78852853048776778963,"[8,]","[['blockTimestampLast','reserve1','reserve0'],]",,uint32,421419304,uint112,78852853048776778963,uint112,,,,,,,,,,,,,,,, +,,0,uniswap_v3 0x6B175474E89094C44Da98b954EedeAC495271d0F/0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2 0.0005,0x6B175474E89094C44Da98b954EedeAC495271d0F/0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2,uniswap_v3,500,0.0005,0x60594a405d53811d3BC4766596EFD80fd545A270,,0x6B175474E89094C44Da98b954EedeAC495271d0F,0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2,18,18,4,DAI,WETH,,,,,,,10,uniswap_v3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,3,uniswap_v3,0x60594a405d53811d3BC4766596EFD80fd545A270,2850299295281281234085971,1147049345011454055669,"[4,0,]","[['liquidity'],['unlocked', 'feeProtocol', 'observationCardinalityNext', 'observationCardinality', 'observationIndex', 'tick', 'sqrtPriceX96']]",,,,,,,451613902936689743409876,"uint128",1659826732499374354385938036,"uint160",-77317,"int24",110,"uint16",180,"uint16",180,"uint16",0,"uint8",True,bool \ No newline at end of file From 0134d335a2d292542657dc7fa5fd25754d37279a Mon Sep 17 00:00:00 2001 From: NIXBNT <88088888+NIXBNT@users.noreply.github.com> Date: Sun, 3 Mar 2024 14:12:28 +1100 Subject: [PATCH 04/41] minor bug fixes --- fastlane_bot/config/multicaller.py | 111 ++++++++++++++---- .../tests/deterministic/dtest_manager.py | 2 +- 2 files changed, 90 insertions(+), 23 deletions(-) diff --git a/fastlane_bot/config/multicaller.py b/fastlane_bot/config/multicaller.py index 76b031bf6..270d65cc1 100644 --- a/fastlane_bot/config/multicaller.py +++ b/fastlane_bot/config/multicaller.py @@ -44,25 +44,92 @@ class MultiCaller: __DATE__ = "2022-09-26" __VERSION__ = "0.0.2" - def __init__(self, web3: Any, multicall_contract_address: str): - self.multicall_contract = web3.eth.contract(abi=MULTICALL_ABI, address=multicall_contract_address) - self.contract_calls: List[ContractFunction] = [] - self.output_types_list: List[List[str]] = [] - - def add_call(self, call: ContractFunction): - self.contract_calls.append({'target': call.address, 'callData': call._encode_transaction_data()}) - self.output_types_list.append([collapse_if_tuple(item) for item in call.abi['outputs']]) - - def run_calls(self, block_identifier: Any = 'latest') -> List[Any]: - encoded_data = self.multicall_contract.functions.tryAggregate( - False, - self.contract_calls - ).call(block_identifier=block_identifier) - - result_list = [ - decode(output_types, encoded_output[1]) if encoded_output[0] else (None,) - for output_types, encoded_output in zip(self.output_types_list, encoded_data) - ] - - # Convert every single-value tuple into a single value - return [result if len(result) > 1 else result[0] for result in result_list] + + def __init__(self, contract: MultiProviderContractWrapper or web3.contract.Contract, + web3: Web3, + block_identifier: Any = 'latest', multicall_address = "0x5BA1e12693Dc8F9c48aAD8770482f4739bEeD696"): + self._contract_calls: List[Callable] = [] + self.contract = contract + self.block_identifier = block_identifier + self.web3 = web3 + self.MULTICALL_CONTRACT_ADDRESS = self.web3.to_checksum_address(multicall_address) + + def __enter__(self) -> 'MultiCaller': + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + pass + + def add_call(self, fn: Callable, *args, **kwargs) -> None: + self._contract_calls.append(partial(fn, *args, **kwargs)) + + def multicall(self) -> List[Any]: + calls_for_aggregate = [] + output_types_list = [] + _calls_for_aggregate = {} + _output_types_list = {} + for fn in self._contract_calls: + fn_name = str(fn).split('functools.partial(')[0] + output_types = get_output_types_from_abi(self.contract.abi, fn_name) + if fn_name in _calls_for_aggregate: + _calls_for_aggregate[fn_name].append({ + 'target': self.contract.address, + 'callData': fn()._encode_transaction_data() + }) + _output_types_list[fn_name].append(output_types) + else: + _calls_for_aggregate[fn_name] = [{ + 'target': self.contract.address, + 'callData': fn()._encode_transaction_data() + }] + _output_types_list[fn_name] = [output_types] + + for fn_list in _calls_for_aggregate.keys(): + calls_for_aggregate += (_calls_for_aggregate[fn_list]) + output_types_list += (_output_types_list[fn_list]) + + _encoded_data = [] + + function_keys = _calls_for_aggregate.keys() + for fn_list in function_keys: + _encoded_data.append(self.web3.eth.contract( + abi=MULTICALL_ABI, + address=self.MULTICALL_CONTRACT_ADDRESS + ).functions.aggregate(_calls_for_aggregate[fn_list]).call(block_identifier=self.block_identifier)) + + if _encoded_data == []: + pass + else: + if not isinstance(_encoded_data[0], list): + raise TypeError(f"Expected encoded_data to be a list, got {type(_encoded_data[0])} instead.") + + encoded_data = self.web3.eth.contract( + abi=MULTICALL_ABI, + address=self.MULTICALL_CONTRACT_ADDRESS + ).functions.aggregate(calls_for_aggregate).call(block_identifier=self.block_identifier) + + if not isinstance(encoded_data, list): + raise TypeError(f"Expected encoded_data to be a list, got {type(encoded_data)} instead.") + + encoded_data = encoded_data[1] + decoded_data_list = [] + for output_types, encoded_output in zip(output_types_list, encoded_data): + decoded_data = decode(output_types, encoded_output) + decoded_data_list.append(decoded_data) + + return_data = [i[0] for i in decoded_data_list if len(i) == 1] + return_data += [i[1] for i in decoded_data_list if len(i) > 1] + + # Handling for Bancor POL - combine results into a Tuple + if "tokenPrice" in function_keys and "amountAvailableForTrading" in function_keys: + new_return = [] + returned_items = int(len(return_data)) + total_pools = int(returned_items / 2) + assert returned_items % 2 == 0, f"[multicaller.py multicall] non-even number of returned calls for Bancor POL {returned_items}" + total_pools = int(total_pools) + + for idx in range(total_pools): + new_return.append((return_data[idx][0], return_data[idx][1], return_data[idx + total_pools])) + return_data = new_return + + return return_data diff --git a/fastlane_bot/tests/deterministic/dtest_manager.py b/fastlane_bot/tests/deterministic/dtest_manager.py index 388b565e6..2c9483f8c 100644 --- a/fastlane_bot/tests/deterministic/dtest_manager.py +++ b/fastlane_bot/tests/deterministic/dtest_manager.py @@ -13,7 +13,7 @@ import pandas as pd import requests -from black import datetime +from datetime import datetime from eth_typing import Address, ChecksumAddress from web3 import Web3 from web3.contract import Contract From ada8b74ddde6cf96079bfec27e6ec32207eb2b11 Mon Sep 17 00:00:00 2001 From: Mike Casale <46603283+mikewcasale@users.noreply.github.com> Date: Sun, 3 Mar 2024 09:08:29 -0800 Subject: [PATCH 05/41] adding missing args parameter to function --- fastlane_bot/tests/deterministic/dtest_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fastlane_bot/tests/deterministic/dtest_manager.py b/fastlane_bot/tests/deterministic/dtest_manager.py index 2c9483f8c..2483851b5 100644 --- a/fastlane_bot/tests/deterministic/dtest_manager.py +++ b/fastlane_bot/tests/deterministic/dtest_manager.py @@ -585,10 +585,10 @@ def approve_and_create_strategies( arg["w3"] = self.w3 test_strategy = TestStrategy(**arg) test_strategy.get_token_approval( - token_id=0, approval_address=self.carbon_controller.address + args=arg, token_id=0, approval_address=self.carbon_controller.address ) test_strategy.get_token_approval( - token_id=1, approval_address=self.carbon_controller.address + args=arg, token_id=1, approval_address=self.carbon_controller.address ) tx_hash = self.create_strategy(args, test_strategy) test_strategy_txhashs[str(i + 1)] = {"txhash": tx_hash} From 9f5df662181ff3ff0f3e683d64bc7a4aa6de6cf5 Mon Sep 17 00:00:00 2001 From: Mike Casale <46603283+mikewcasale@users.noreply.github.com> Date: Sun, 3 Mar 2024 09:13:30 -0800 Subject: [PATCH 06/41] added option to not delete old logs --- run_deterministic_tests.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/run_deterministic_tests.py b/run_deterministic_tests.py index 3d3dcfe8d..b8091bfa7 100644 --- a/run_deterministic_tests.py +++ b/run_deterministic_tests.py @@ -208,7 +208,6 @@ def run_results_crosscheck_task(args, proc: subprocess.Popen): # Scan for successful transactions on Tenderly which are marked by status=1 actual_txs = tx_helper.tx_scanner(args) - # tx_helper.log_txs(actual_txs, args) expected_txs = tx_helper.load_json_file("test_results.json", args) results_description = tx_helper.log_results( @@ -240,7 +239,8 @@ def main(args: argparse.Namespace): # Initialize the Web3 Manager test_manager = TestManager(args=args) - test_manager.delete_old_logs(args) + if args.delete_old_logs: + test_manager.delete_old_logs(args) if args.task == "set_test_state": set_test_state_task(test_manager.w3) @@ -325,6 +325,13 @@ def _extracted_task_handling(test_manager: TestManager, args: argparse.Namespace help="Logging level", choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], ) + parser.add_argument( + "--delete_old_logs", + default="True", + type=str, + choices=["True", "False"], + ) args = parser.parse_args() + args.delete_old_logs = args.delete_old_logs.lower() == "true" main(args) From c7816362054dbd09c34ea99460fbb58442fa9baf Mon Sep 17 00:00:00 2001 From: Mike Casale <46603283+mikewcasale@users.noreply.github.com> Date: Sun, 3 Mar 2024 11:35:13 -0800 Subject: [PATCH 07/41] minor fixes --- fastlane_bot/tests/deterministic/dtest_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fastlane_bot/tests/deterministic/dtest_manager.py b/fastlane_bot/tests/deterministic/dtest_manager.py index 2483851b5..c2ed4c1a5 100644 --- a/fastlane_bot/tests/deterministic/dtest_manager.py +++ b/fastlane_bot/tests/deterministic/dtest_manager.py @@ -585,10 +585,10 @@ def approve_and_create_strategies( arg["w3"] = self.w3 test_strategy = TestStrategy(**arg) test_strategy.get_token_approval( - args=arg, token_id=0, approval_address=self.carbon_controller.address + args=args, token_id=0, approval_address=self.carbon_controller.address ) test_strategy.get_token_approval( - args=arg, token_id=1, approval_address=self.carbon_controller.address + args=args, token_id=1, approval_address=self.carbon_controller.address ) tx_hash = self.create_strategy(args, test_strategy) test_strategy_txhashs[str(i + 1)] = {"txhash": tx_hash} From 98424b31bf7719dc629746b792ac6d218c0ad45a Mon Sep 17 00:00:00 2001 From: Mike Casale <46603283+mikewcasale@users.noreply.github.com> Date: Mon, 11 Mar 2024 06:32:10 -0700 Subject: [PATCH 08/41] Update multicaller.py --- fastlane_bot/config/multicaller.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/fastlane_bot/config/multicaller.py b/fastlane_bot/config/multicaller.py index 270d65cc1..e9b6ea019 100644 --- a/fastlane_bot/config/multicaller.py +++ b/fastlane_bot/config/multicaller.py @@ -97,12 +97,9 @@ def multicall(self) -> List[Any]: address=self.MULTICALL_CONTRACT_ADDRESS ).functions.aggregate(_calls_for_aggregate[fn_list]).call(block_identifier=self.block_identifier)) - if _encoded_data == []: - pass - else: - if not isinstance(_encoded_data[0], list): - raise TypeError(f"Expected encoded_data to be a list, got {type(_encoded_data[0])} instead.") - + if len(_encoded_data) > 0 and not isinstance(_encoded_data[0], list): + raise TypeError(f"Expected encoded_data to be a list, got {type(_encoded_data[0])} instead.") + encoded_data = self.web3.eth.contract( abi=MULTICALL_ABI, address=self.MULTICALL_CONTRACT_ADDRESS From c58f5d93fc26f52bc0d45fa75d3b2d8db11f8e37 Mon Sep 17 00:00:00 2001 From: Mike Casale <46603283+mikewcasale@users.noreply.github.com> Date: Mon, 11 Mar 2024 06:35:17 -0700 Subject: [PATCH 09/41] change constant names per @barakman --- fastlane_bot/config/multicaller.py | 2 +- fastlane_bot/tests/deterministic/dtest_constants.py | 4 ++-- fastlane_bot/tests/deterministic/dtest_manager.py | 6 +++--- fastlane_bot/tests/deterministic/dtest_pool.py | 4 ++-- fastlane_bot/tests/deterministic/dtest_tx_helper.py | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/fastlane_bot/config/multicaller.py b/fastlane_bot/config/multicaller.py index e9b6ea019..9a73b4678 100644 --- a/fastlane_bot/config/multicaller.py +++ b/fastlane_bot/config/multicaller.py @@ -99,7 +99,7 @@ def multicall(self) -> List[Any]: if len(_encoded_data) > 0 and not isinstance(_encoded_data[0], list): raise TypeError(f"Expected encoded_data to be a list, got {type(_encoded_data[0])} instead.") - + encoded_data = self.web3.eth.contract( abi=MULTICALL_ABI, address=self.MULTICALL_CONTRACT_ADDRESS diff --git a/fastlane_bot/tests/deterministic/dtest_constants.py b/fastlane_bot/tests/deterministic/dtest_constants.py index 0c57d8d7d..fa7074052 100644 --- a/fastlane_bot/tests/deterministic/dtest_constants.py +++ b/fastlane_bot/tests/deterministic/dtest_constants.py @@ -23,8 +23,8 @@ DEFAULT_GAS_PRICE = 0 DEFAULT_FROM_BLOCK = 1000000 TENDERLY_RPC_KEY = "fb866397-29bd-4886-8406-a2cc7b7c5b1f" # https://virtual.mainnet.rpc.tenderly.co/9ea4ceb3-d0f5-4faf-959e-f51cf1f6b52b, from_block: 19325893, fb866397-29bd-4886-8406-a2cc7b7c5b1f -FILE_DATA_DIR = "fastlane_bot/data/blockchain_data" -TEST_FILE_DATA_DIR = "fastlane_bot/tests/deterministic/_data" +REAL_DATA_DIR = "fastlane_bot/data/blockchain_data" +TEST_DATA_DIR = "fastlane_bot/tests/deterministic/_data" binance14 = "0x28C6c06298d514Db089934071355E5743bf21d60" TOKENS_MODIFICATIONS = { "0x0": { diff --git a/fastlane_bot/tests/deterministic/dtest_manager.py b/fastlane_bot/tests/deterministic/dtest_manager.py index c2ed4c1a5..fb0840a2c 100644 --- a/fastlane_bot/tests/deterministic/dtest_manager.py +++ b/fastlane_bot/tests/deterministic/dtest_manager.py @@ -23,7 +23,7 @@ DEFAULT_GAS, DEFAULT_GAS_PRICE, ETH_ADDRESS, - TEST_FILE_DATA_DIR, + TEST_DATA_DIR, TOKENS_MODIFICATIONS, TestCommandLineArgs, ) @@ -495,7 +495,7 @@ def get_test_strategies(args: argparse.Namespace) -> dict: Gets test strategies from a JSON file. """ test_strategies_path = os.path.normpath( - f"{TEST_FILE_DATA_DIR}/test_strategies.json" + f"{TEST_DATA_DIR}/test_strategies.json" ) with open(test_strategies_path) as file: test_strategies = json.load(file)["test_strategies"] @@ -545,7 +545,7 @@ def write_strategy_txhashs_to_json(test_strategy_txhashs: dict): test_strategy_txhashs (dict): The test strategy txhashs. """ test_strategy_txhashs_path = os.path.normpath( - f"{TEST_FILE_DATA_DIR}/test_strategy_txhashs.json" + f"{TEST_DATA_DIR}/test_strategy_txhashs.json" ) with open(test_strategy_txhashs_path, "w") as f: json.dump(test_strategy_txhashs, f) diff --git a/fastlane_bot/tests/deterministic/dtest_pool.py b/fastlane_bot/tests/deterministic/dtest_pool.py index 74ccbc92f..7f8049ec5 100644 --- a/fastlane_bot/tests/deterministic/dtest_pool.py +++ b/fastlane_bot/tests/deterministic/dtest_pool.py @@ -15,7 +15,7 @@ from fastlane_bot.tests.deterministic.dtest_constants import ( SUPPORTED_EXCHANGES, - TEST_FILE_DATA_DIR, + TEST_DATA_DIR, ) from fastlane_bot.tests.deterministic.dtest_token import TestTokenBalance @@ -113,6 +113,6 @@ def set_balance_via_faucet(self, args: argparse.Namespace, def load_test_pools(): # Import pool data static_pool_data_testing_path = os.path.normpath( - f"{TEST_FILE_DATA_DIR}/static_pool_data_testing.csv" + f"{TEST_DATA_DIR}/static_pool_data_testing.csv" ) return pd.read_csv(static_pool_data_testing_path, dtype=str) diff --git a/fastlane_bot/tests/deterministic/dtest_tx_helper.py b/fastlane_bot/tests/deterministic/dtest_tx_helper.py index cea03f8c1..62c7723a7 100644 --- a/fastlane_bot/tests/deterministic/dtest_tx_helper.py +++ b/fastlane_bot/tests/deterministic/dtest_tx_helper.py @@ -13,7 +13,7 @@ import time from fastlane_bot.tests.deterministic.dtest_constants import ( - TEST_FILE_DATA_DIR, + TEST_DATA_DIR, ) @@ -167,7 +167,7 @@ def load_json_file(file_name: str, args: argparse.Namespace) -> dict: dict: The data from the json file. """ file_path = ( - os.path.normpath(f"{TEST_FILE_DATA_DIR}/{file_name}") + os.path.normpath(f"{TEST_DATA_DIR}/{file_name}") if "/" not in file_name else os.path.normpath(file_name) ) From 69f55357eaab6c9e3578532d50cae67e925cbdac Mon Sep 17 00:00:00 2001 From: Mike Casale <46603283+mikewcasale@users.noreply.github.com> Date: Mon, 11 Mar 2024 06:39:23 -0700 Subject: [PATCH 10/41] Update dtest_token.py --- fastlane_bot/tests/deterministic/dtest_token.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/fastlane_bot/tests/deterministic/dtest_token.py b/fastlane_bot/tests/deterministic/dtest_token.py index 6a1dd5251..b6b875223 100644 --- a/fastlane_bot/tests/deterministic/dtest_token.py +++ b/fastlane_bot/tests/deterministic/dtest_token.py @@ -23,18 +23,10 @@ class TestToken: """ address: str or Address # Address after __post_init__, str before + contract: Contract = None def __post_init__(self): self.address = Web3.to_checksum_address(self.address) - self._contract = None - - @property - def contract(self): - return self._contract - - @contract.setter - def contract(self, contract: Contract): - self._contract = contract @property def is_eth(self): From 578a5f70df31f299b0e9de064afa10ed84ac1969 Mon Sep 17 00:00:00 2001 From: Mike Casale <46603283+mikewcasale@users.noreply.github.com> Date: Mon, 11 Mar 2024 06:58:41 -0700 Subject: [PATCH 11/41] Update dtest_constants.py --- fastlane_bot/tests/deterministic/dtest_constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastlane_bot/tests/deterministic/dtest_constants.py b/fastlane_bot/tests/deterministic/dtest_constants.py index fa7074052..a4dd154e5 100644 --- a/fastlane_bot/tests/deterministic/dtest_constants.py +++ b/fastlane_bot/tests/deterministic/dtest_constants.py @@ -80,7 +80,7 @@ class TestCommandLineArgs: polling_interval: int = 0 alchemy_max_block_fetch: int = 1 reorg_delay: int = 0 - logging_path: str = "" + logging_path: str = "logs_dtest" loglevel: str = "INFO" use_cached_events: str = "False" run_data_validator: str = "False" From e7d869d3f6b345eb43bdf1279cd58470c3b83061 Mon Sep 17 00:00:00 2001 From: Mike Casale <46603283+mikewcasale@users.noreply.github.com> Date: Mon, 11 Mar 2024 06:58:44 -0700 Subject: [PATCH 12/41] Update run_deterministic_tests.py --- run_deterministic_tests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/run_deterministic_tests.py b/run_deterministic_tests.py index b8091bfa7..fa230a6c0 100644 --- a/run_deterministic_tests.py +++ b/run_deterministic_tests.py @@ -269,13 +269,14 @@ def _extracted_task_handling(test_manager: TestManager, args: argparse.Namespace parser = argparse.ArgumentParser(description="Fastlane Bot") parser.add_argument( "--task", - default="update_pool_params", + default="end_to_end", type=str, choices=[ "set_test_state", "get_carbon_strategies_and_delete", "run_tests_on_mode", "run_results_crosscheck", + "update_pool_params", "end_to_end", ], help="Task to run", From c61bef52850000bc62af01f21e97a7f01a6247fa Mon Sep 17 00:00:00 2001 From: Mike Casale <46603283+mikewcasale@users.noreply.github.com> Date: Mon, 11 Mar 2024 07:01:39 -0700 Subject: [PATCH 13/41] Update dtest_pool.py --- fastlane_bot/tests/deterministic/dtest_pool.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fastlane_bot/tests/deterministic/dtest_pool.py b/fastlane_bot/tests/deterministic/dtest_pool.py index 7f8049ec5..d6bfdfe4b 100644 --- a/fastlane_bot/tests/deterministic/dtest_pool.py +++ b/fastlane_bot/tests/deterministic/dtest_pool.py @@ -92,8 +92,8 @@ def set_balance_via_faucet(self, args: argparse.Namespace, w3: The Web3 instance. token_id: The token id. """ - token_address = self.tkn0_address if token_id == 0 else self.tkn1_address - amount_wei = self.tkn0_setBalance if token_id == 0 else self.tkn1_setBalance + token_address = [self.tkn0_address, self.tkn1_address][token_id] + amount_wei = [self.tkn0_setBalance, self.tkn1_setBalance][token_id] token_balance = TestTokenBalance(token=token_address, balance=amount_wei) params = token_balance.faucet_params(wallet_address=self.pool_address) method_name = RPCEndpoint( From 46001f331b2e5aef81c7c2214b26f35532c8cc14 Mon Sep 17 00:00:00 2001 From: Mike Casale <46603283+mikewcasale@users.noreply.github.com> Date: Mon, 11 Mar 2024 11:11:41 -0700 Subject: [PATCH 14/41] Update requirements.txt --- requirements.txt | 90 +++++++++++------------------------------------- 1 file changed, 21 insertions(+), 69 deletions(-) diff --git a/requirements.txt b/requirements.txt index 7cec2637d..8c94bafdc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,69 +1,21 @@ -aiohttp==3.9.3 ; python_version >= "3.8" and python_version < "4.0" -aiosignal==1.3.1 ; python_version >= "3.8" and python_version < "4.0" -alchemy-sdk==0.1.1 ; python_version >= "3.8" and python_version < "4.0" -async-timeout==4.0.3 ; python_version >= "3.8" and python_version < "3.11" -attrs==23.2.0 ; python_version >= "3.8" and python_version < "4.0" -backoff==2.2.1 ; python_version >= "3.8" and python_version < "4.0" -bitarray==2.9.2 ; python_version >= "3.8" and python_version < "4" -certifi==2024.2.2 ; python_version >= "3.8" and python_version < "4.0" -charset-normalizer==3.3.2 ; python_version >= "3.8" and python_version < "4.0" -ckzg==1.0.0 ; python_version >= "3.8" and python_version < "4" -colorama==0.4.6 ; python_version >= "3.8" and python_version < "4.0" and platform_system == "Windows" -contourpy==1.1.1 ; python_version >= "3.8" and python_version < "4.0" -cycler==0.12.1 ; python_version >= "3.8" and python_version < "4.0" -cytoolz==0.12.3 ; python_version >= "3.8" and python_version < "4" and implementation_name == "cpython" -dataclass-wizard==0.22.3 ; python_version >= "3.8" and python_version < "4.0" -eth-abi==5.1.0 ; python_version >= "3.8" and python_version < "4" -eth-account==0.11.2 ; python_version >= "3.8" and python_version < "4" -eth-hash==0.7.0 ; python_version >= "3.8" and python_version < "4" -eth-hash[pycryptodome]==0.7.0 ; python_version >= "3.8" and python_version < "4" -eth-keyfile==0.8.0 ; python_version >= "3.8" and python_version < "4" -eth-keys==0.5.0 ; python_version >= "3.8" and python_version < "4" -eth-rlp==1.0.1 ; python_version >= "3.8" and python_version < "4" -eth-typing==4.1.0 ; python_version >= "3.8" and python_version < "4" -eth-utils==4.1.0 ; python_version >= "3.8" and python_version < "4" -fonttools==4.51.0 ; python_version >= "3.8" and python_version < "4.0" -frozenlist==1.4.1 ; python_version >= "3.8" and python_version < "4.0" -hexbytes==0.3.1 ; python_version >= "3.8" and python_version < "4" -idna==3.7 ; python_version >= "3.8" and python_version < "4.0" -importlib-resources==6.4.0 ; python_version >= "3.8" and python_version < "3.10" -joblib==1.4.0 ; python_version >= "3.8" and python_version < "4.0" -jsonschema-specifications==2023.12.1 ; python_version >= "3.8" and python_version < "4.0" -jsonschema==4.21.1 ; python_version >= "3.8" and python_version < "4.0" -kiwisolver==1.4.5 ; python_version >= "3.8" and python_version < "4.0" -lru-dict==1.2.0 ; python_version >= "3.8" and python_version < "4.0" -matplotlib==3.7.5 ; python_version >= "3.8" and python_version < "4.0" -multidict==6.0.5 ; python_version >= "3.8" and python_version < "4.0" -nest-asyncio==1.6.0 ; python_version >= "3.8" and python_version < "4.0" -networkx==3.1 ; python_version >= "3.8" and python_version < "4.0" -numpy==1.24.4 ; python_version >= "3.8" and python_version < "4.0" -packaging==21.3 ; python_version >= "3.8" and python_version < "4.0" -pandas==1.5.3 ; python_version >= "3.8" and python_version < "4.0" -parsimonious==0.10.0 ; python_version >= "3.8" and python_version < "4" -pillow==10.3.0 ; python_version >= "3.8" and python_version < "4.0" -pkgutil-resolve-name==1.3.10 ; python_version >= "3.8" and python_version < "3.9" -protobuf==4.25.3 ; python_version >= "3.8" and python_version < "4.0" -psutil==5.9.8 ; python_version >= "3.8" and python_version < "4.0" -pyarrow==11.0.0 ; python_version >= "3.8" and python_version < "4.0" -pycryptodome==3.20.0 ; python_version >= "3.8" and python_version < "4" -pyparsing==3.1.2 ; python_version >= "3.8" and python_version < "4.0" -python-dateutil==2.9.0.post0 ; python_version >= "3.8" and python_version < "4.0" -python-dotenv==0.16.0 ; python_version >= "3.8" and python_version < "4.0" -pytz==2024.1 ; python_version >= "3.8" and python_version < "4.0" -pyunormalize==15.1.0 ; python_version >= "3.8" and python_version < "4.0" -pywin32==306 ; python_version >= "3.8" and python_version < "4.0" and platform_system == "Windows" -referencing==0.34.0 ; python_version >= "3.8" and python_version < "4.0" -regex==2023.12.25 ; python_version >= "3.8" and python_version < "4" -requests==2.31.0 ; python_version >= "3.8" and python_version < "4.0" -rlp==4.0.0 ; python_version >= "3.8" and python_version < "4" -rpds-py==0.18.0 ; python_version >= "3.8" and python_version < "4.0" -setuptools==67.8.0 ; python_version >= "3.8" and python_version < "4.0" -six==1.16.0 ; python_version >= "3.8" and python_version < "4.0" -toolz==0.12.1 ; python_version >= "3.8" and python_version < "4" and (implementation_name == "pypy" or implementation_name == "cpython") -tqdm==4.66.2 ; python_version >= "3.8" and python_version < "4.0" -typing-extensions==4.11.0 ; python_version >= "3.8" and python_version < "4.0" -urllib3==2.2.1 ; python_version >= "3.8" and python_version < "4.0" -web3==6.16.0 ; python_version >= "3.8" and python_version < "4.0" -websockets==12.0 ; python_version >= "3.8" and python_version < "4.0" -yarl==1.9.4 ; python_version >= "3.8" and python_version < "4.0" -zipp==3.18.1 ; python_version >= "3.8" and python_version < "3.10" +psutil~=5.9.6 +packaging==21.3 +requests~=2.31.0 +python-dateutil~=2.8.2 +typing-extensions~=4.7.1 +python-dotenv~=0.16.0 +joblib~=1.2.0 +pandas~=1.5.2 +alchemy-sdk~=0.1.1 +pyarrow~=11.0.0 +networkx~=3.0 +cvxpy~=1.3.1 +matplotlib~=3.7.1 +dataclass_wizard~=0.22.2 +hexbytes~=0.3.1 +setuptools~=67.6.1 +protobuf~=4.24.4 +tqdm~=4.64.1 +web3~=6.11.2 +nest-asyncio~=1.5.8 +eth-abi~=5.0.0 From e19eb488c6ac2a917415490ec978bb14d31ebc22 Mon Sep 17 00:00:00 2001 From: Mike Casale <46603283+mikewcasale@users.noreply.github.com> Date: Tue, 12 Mar 2024 12:03:52 -0700 Subject: [PATCH 15/41] Update run_deterministic_tests.py --- run_deterministic_tests.py | 1 - 1 file changed, 1 deletion(-) diff --git a/run_deterministic_tests.py b/run_deterministic_tests.py index fa230a6c0..76257bdd2 100644 --- a/run_deterministic_tests.py +++ b/run_deterministic_tests.py @@ -276,7 +276,6 @@ def _extracted_task_handling(test_manager: TestManager, args: argparse.Namespace "get_carbon_strategies_and_delete", "run_tests_on_mode", "run_results_crosscheck", - "update_pool_params", "end_to_end", ], help="Task to run", From 775e251cfd33fbc3fcffb389a1f457f6e19bbc8f Mon Sep 17 00:00:00 2001 From: Mike Casale <46603283+mikewcasale@users.noreply.github.com> Date: Tue, 12 Mar 2024 12:08:50 -0700 Subject: [PATCH 16/41] Update run_deterministic_tests.py --- run_deterministic_tests.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/run_deterministic_tests.py b/run_deterministic_tests.py index 76257bdd2..adb7f401c 100644 --- a/run_deterministic_tests.py +++ b/run_deterministic_tests.py @@ -247,16 +247,16 @@ def main(args: argparse.Namespace): elif args.task == "get_carbon_strategies_and_delete": get_carbon_strategies_and_delete_task(test_manager, args) elif args.task == "run_tests_on_mode": - _extracted_task_handling(test_manager, args) + _extracted_run_tests_on_mode(test_manager, args) elif args.task == "end_to_end": get_carbon_strategies_and_delete_task(test_manager, args) set_test_state_task(test_manager.w3) - _extracted_task_handling(test_manager, args) + _extracted_run_tests_on_mode(test_manager, args) else: raise ValueError(f"Task {args.task} not recognized") -def _extracted_task_handling(test_manager: TestManager, args: argparse.Namespace): +def _extracted_run_tests_on_mode(test_manager: TestManager, args: argparse.Namespace): """ Extracted task handling. """ From 290e69366b066e2a47ab3be71c0a8267ac69e5fe Mon Sep 17 00:00:00 2001 From: Mike Casale <46603283+mikewcasale@users.noreply.github.com> Date: Tue, 26 Mar 2024 07:27:31 -0700 Subject: [PATCH 17/41] bugfixes and update gitignore --- .gitignore | 1 + fastlane_bot/tests/deterministic/dtest_manager.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index a35a7e42a..54b2ff1f2 100644 --- a/.gitignore +++ b/.gitignore @@ -68,6 +68,7 @@ latest_pool_data.json missing_events.json *.log logs/* +logs_dtest/* /token_details.csv /fastlane_bot/data/blockchain_data/*/token_detail/ missing_tokens_df.csv diff --git a/fastlane_bot/tests/deterministic/dtest_manager.py b/fastlane_bot/tests/deterministic/dtest_manager.py index fb0840a2c..69949eadb 100644 --- a/fastlane_bot/tests/deterministic/dtest_manager.py +++ b/fastlane_bot/tests/deterministic/dtest_manager.py @@ -63,7 +63,8 @@ def __init__(self, args: argparse.Namespace): @property def logs_path(self) -> str: - return os.path.normpath("./logs/*") + """TODO: This should be read from dtest_constants""" + return os.path.normpath("./logs_dtest/logs/*") def get_carbon_controller(self, address: Address or str) -> Contract: """ From 85195f6d35fef1aa71c0ab8ee4d7893a30317bfa Mon Sep 17 00:00:00 2001 From: NIXBNT <88088888+NIXBNT@users.noreply.github.com> Date: Wed, 27 Mar 2024 08:17:54 +1100 Subject: [PATCH 18/41] path handling bugfix --- fastlane_bot/tests/deterministic/dtest_manager.py | 4 ++-- fastlane_bot/tests/deterministic/dtest_tx_helper.py | 10 ++++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/fastlane_bot/tests/deterministic/dtest_manager.py b/fastlane_bot/tests/deterministic/dtest_manager.py index 69949eadb..684f4c30d 100644 --- a/fastlane_bot/tests/deterministic/dtest_manager.py +++ b/fastlane_bot/tests/deterministic/dtest_manager.py @@ -63,7 +63,7 @@ def __init__(self, args: argparse.Namespace): @property def logs_path(self) -> str: - """TODO: This should be read from dtest_constants""" + """TODO TESTS: This should be read from dtest_constants""" return os.path.normpath("./logs_dtest/logs/*") def get_carbon_controller(self, address: Address or str) -> Contract: @@ -633,7 +633,7 @@ def get_most_recent_pool_data_path(self, args: argparse.Namespace) -> str: most_recent_log_folder = [ f for f in glob.glob(self.logs_path) if os.path.isdir(f) ][-1] - args.logger.debug(f"Accessing log folder {most_recent_log_folder}") + args.logger.debug(f"[dtest_manager.get_most_recent_pool_data_path] Accessing log folder {most_recent_log_folder}") return os.path.join(most_recent_log_folder, "latest_pool_data.json") def delete_old_logs(self, args: argparse.Namespace): diff --git a/fastlane_bot/tests/deterministic/dtest_tx_helper.py b/fastlane_bot/tests/deterministic/dtest_tx_helper.py index 62c7723a7..8ff8363ee 100644 --- a/fastlane_bot/tests/deterministic/dtest_tx_helper.py +++ b/fastlane_bot/tests/deterministic/dtest_tx_helper.py @@ -22,6 +22,10 @@ class TestTxHelper: This is a utility class to scan the logs directory for successful transactions and clean and extract the transaction data. """ + @property + def logs_path(self) -> str: + """TODO TESTS: This should be read from dtest_constants""" + return os.path.normpath("./logs_dtest/logs/*") @staticmethod def find_most_recent_log_folder(logs_path="./logs/*") -> str: @@ -98,8 +102,10 @@ def tx_scanner(self, args: argparse.Namespace) -> list: Returns: list: A list of successful transactions. """ - most_recent_log_folder = self.find_most_recent_log_folder() - args.logger.debug(f"Accessing log folder {most_recent_log_folder}") + most_recent_log_folder = [ + f for f in glob.glob(self.logs_path) if os.path.isdir(f) + ][-1] + args.logger.debug(f"[dtest_tx_helper.tx_scanner] Accessing log folder {most_recent_log_folder}") pool_data_file = os.path.join(most_recent_log_folder, "latest_pool_data.json") if self.wait_for_file(pool_data_file, args.logger): From cf2bb0d1b250160ecb9c4916cc53a204f61eaeb3 Mon Sep 17 00:00:00 2001 From: NIXBNT <88088888+NIXBNT@users.noreply.github.com> Date: Wed, 27 Mar 2024 09:31:02 +1100 Subject: [PATCH 19/41] add strategy_ids to the logging --- fastlane_bot/bot.py | 84 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 83 insertions(+), 1 deletion(-) diff --git a/fastlane_bot/bot.py b/fastlane_bot/bot.py index 5fe5800d5..f40e33cca 100644 --- a/fastlane_bot/bot.py +++ b/fastlane_bot/bot.py @@ -855,7 +855,89 @@ def _handle_trade_instructions( maximize_last_trade_per_tkn(route_struct=route_struct_processed) - # Log the flashloan details + # Get the cids + cids = [(ti.cid,ti.strategy_id) for ti in ordered_trade_instructions_objects] + + # Check if the network is tenderly and submit the transaction accordingly + if self.ConfigObj.NETWORK == self.ConfigObj.NETWORK_TENDERLY: + return ( + self._validate_and_submit_transaction_tenderly( + ConfigObj=self.ConfigObj, + flashloan_struct=flashloan_struct, + route_struct=route_struct_maximized, + src_amount=flashloan_amount_wei, + src_address=flashloan_token_address, + ), + cids, + route_struct_maximized, + log_dict, + ) + + # Log the route_struct + self.handle_logging_for_trade_instructions( + 4, # The log id + flashloan_amount=flashloan_amount_wei, + flashloan_token_symbol=fl_token_symbol, + flashloan_token_address=flashloan_token_address, + route_struct=route_struct_maximized, + best_trade_instructions_dic=best_trade_instructions_dic, + ) + + # Get the tx helpers class + tx_helpers = TxHelpers(ConfigObj=self.ConfigObj) + + # Return the validate and submit transaction + return ( + tx_helpers.validate_and_submit_transaction( + route_struct=route_struct_maximized, + src_amt=flashloan_amount_wei, + src_address=flashloan_token_address, + expected_profit_gastkn=best_profit_gastkn, + expected_profit_usd=best_profit_usd, + safety_override=False, + verbose=True, + log_object=log_dict, + flashloan_struct=flashloan_struct, + ), + cids, + route_struct, + log_dict, + ) + + def handle_logging_for_trade_instructions(self, log_id: int, **kwargs): + """ + Handles logging for trade instructions based on log_id. + + Parameters + ---------- + log_id : int + The ID for log type. + **kwargs : dict + Additional parameters required for logging. + + Returns + ------- + None + """ + log_actions = { + 1: self.log_best_profit, + 2: self.log_calculated_arb, + 3: self.log_flashloan_amount, + 4: self.log_flashloan_details, + } + log_action = log_actions.get(log_id) + if log_action: + log_action(**kwargs) + + def log_best_profit(self, best_profit: Optional[float] = None): + """ + Logs the best profit. + + Parameters + ---------- + best_profit : Optional[float], optional + The best profit, by default None + """ self.ConfigObj.logger.debug( f"[bot._handle_trade_instructions] Flashloan of {fl_token_symbol}, amount: {flashloan_amount_wei}" ) From 5380e84ba840268727cc97f23f986df617ee3126 Mon Sep 17 00:00:00 2001 From: Mike Casale <46603283+mikewcasale@users.noreply.github.com> Date: Thu, 28 Mar 2024 04:45:16 -0700 Subject: [PATCH 20/41] Fix python 3.8 type-hint --- .../tests/deterministic/dtest_manager.py | 32 +++++++++---------- .../dtest_pool_params_builder.py | 11 ++++--- .../tests/deterministic/dtest_token.py | 3 +- .../tests/deterministic/dtest_tx_helper.py | 23 ++++++------- .../tests/deterministic/dtest_wallet.py | 3 +- 5 files changed, 38 insertions(+), 34 deletions(-) diff --git a/fastlane_bot/tests/deterministic/dtest_manager.py b/fastlane_bot/tests/deterministic/dtest_manager.py index 684f4c30d..8bf1682e4 100644 --- a/fastlane_bot/tests/deterministic/dtest_manager.py +++ b/fastlane_bot/tests/deterministic/dtest_manager.py @@ -9,7 +9,7 @@ import json import os import time -from typing import Dict +from typing import Dict, List, Tuple import pandas as pd import requests @@ -100,7 +100,7 @@ def get_carbon_controller_address( ) @staticmethod - def create_new_testnet() -> tuple: + def create_new_testnet() -> Tuple[str, int]: """ Creates a new testnet on Tenderly. @@ -141,7 +141,7 @@ def create_new_testnet() -> tuple: return uri, from_block @staticmethod - def process_order_data(log_args: dict, order_key: str) -> dict: + def process_order_data(log_args: Dict, order_key: str) -> Dict: """ Transforms nested order data by appending a suffix to each key. @@ -160,9 +160,9 @@ def process_order_data(log_args: dict, order_key: str) -> dict: @staticmethod def print_state_changes( args: argparse.Namespace, - all_carbon_strategies: list, - deleted_strategies: list, - remaining_carbon_strategies: list, + all_carbon_strategies: List, + deleted_strategies: List, + remaining_carbon_strategies: List, ) -> None: """ Prints the state changes of Carbon strategies. @@ -234,7 +234,7 @@ def get_generic_events( def get_state_of_carbon_strategies( self, args: argparse.Namespace, from_block: int - ) -> tuple: + ) -> Tuple: """ Fetches the state of Carbon strategies. @@ -327,7 +327,7 @@ def modify_token( self, args: argparse.Namespace, token_address: str, - modifications: dict, + modifications: Dict, strategy_id: int, strategy_beneficiary: Address, ): @@ -445,8 +445,8 @@ def delete_strategy(self, strategy_id: int, wallet: Address) -> int: return tx_receipt.status def delete_all_carbon_strategies( - self, args: argparse.Namespace, carbon_strategy_id_owner_list: list - ) -> list: + self, args: argparse.Namespace, carbon_strategy_id_owner_list: List + ) -> List: """ Deletes all Carbon strategies. @@ -491,7 +491,7 @@ def delete_all_carbon_strategies( return undeleted_strategies @staticmethod - def get_test_strategies(args: argparse.Namespace) -> dict: + def get_test_strategies(args: argparse.Namespace) -> Dict: """ Gets test strategies from a JSON file. """ @@ -504,8 +504,8 @@ def get_test_strategies(args: argparse.Namespace) -> dict: return test_strategies def append_strategy_ids( - self, args: argparse.Namespace, test_strategy_txhashs: dict, from_block: int - ) -> dict: + self, args: argparse.Namespace, test_strategy_txhashs: Dict, from_block: int + ) -> Dict: """ Appends the strategy ids to the test strategies. @@ -538,7 +538,7 @@ def append_strategy_ids( return test_strategy_txhashs @staticmethod - def write_strategy_txhashs_to_json(test_strategy_txhashs: dict): + def write_strategy_txhashs_to_json(test_strategy_txhashs: Dict): """ Writes the test strategy txhashs to a file. @@ -568,8 +568,8 @@ def get_strats_created_from_block(args: argparse.Namespace, w3: Web3) -> int: return strats_created_from_block def approve_and_create_strategies( - self, args: argparse.Namespace, test_strategies: dict, from_block: int - ) -> dict: + self, args: argparse.Namespace, test_strategies: Dict, from_block: int + ) -> Dict: """ Approves and creates test strategies. diff --git a/fastlane_bot/tests/deterministic/dtest_pool_params_builder.py b/fastlane_bot/tests/deterministic/dtest_pool_params_builder.py index 18d698610..ad4420d23 100644 --- a/fastlane_bot/tests/deterministic/dtest_pool_params_builder.py +++ b/fastlane_bot/tests/deterministic/dtest_pool_params_builder.py @@ -7,6 +7,7 @@ """ import re from dataclasses import dataclass +from typing import Dict, List, Tuple import eth_abi from web3 import Web3 @@ -74,8 +75,8 @@ def append_zeros(value: any, type_str: str) -> str: return result def build_type_val_dict( - self, pool: TestPool, param_list_single: list[str] - ) -> tuple: + self, pool: TestPool, param_list_single: List[str] + ) -> Tuple: """ This method is used to build the type_val_dict and the encoded_params for the given pool. """ @@ -114,7 +115,7 @@ def get_latest_block_timestamp(self): print(f"Error fetching latest block timestamp: {e}") return None - def encode_params(self, type_val_dict: dict, param_list_single: list[str]) -> str: + def encode_params(self, type_val_dict: Dict, param_list_single: List[str]) -> str: """ This method is used to encode the parameters into a string that can be used to update the storage of the pool contract. @@ -136,7 +137,7 @@ def encode_params(self, type_val_dict: dict, param_list_single: list[str]) -> st print(f"Error encoding params: {e}, {type_val_dict}") return None - def get_update_params_dict(self, pool: TestPool) -> dict: + def get_update_params_dict(self, pool: TestPool) -> Dict: """ This method is used to get the update parameters dictionary for the given pool. @@ -159,7 +160,7 @@ def get_update_params_dict(self, pool: TestPool) -> dict: params_dict[pool.slots[i]]["encoded_params"] = encoded_params return params_dict - def set_storage_at(self, pool_address: str, update_params_dict_single: dict): + def set_storage_at(self, pool_address: str, update_params_dict_single: Dict): method = RPCEndpoint("tenderly_setStorageAt") self.w3.provider.make_request( method=method, diff --git a/fastlane_bot/tests/deterministic/dtest_token.py b/fastlane_bot/tests/deterministic/dtest_token.py index b6b875223..8aed1aeb6 100644 --- a/fastlane_bot/tests/deterministic/dtest_token.py +++ b/fastlane_bot/tests/deterministic/dtest_token.py @@ -5,6 +5,7 @@ Licensed under MIT License. """ from dataclasses import dataclass +from typing import Any, List from eth_typing import Address from web3 import Web3 @@ -54,7 +55,7 @@ def __post_init__(self): def hex_balance(self): return Web3.to_hex(self.balance) - def faucet_params(self, wallet_address: str = None) -> list: + def faucet_params(self, wallet_address: str = None) -> List: """ This method is used to return the faucet parameters for the token balance. diff --git a/fastlane_bot/tests/deterministic/dtest_tx_helper.py b/fastlane_bot/tests/deterministic/dtest_tx_helper.py index 8ff8363ee..0d18784ec 100644 --- a/fastlane_bot/tests/deterministic/dtest_tx_helper.py +++ b/fastlane_bot/tests/deterministic/dtest_tx_helper.py @@ -11,6 +11,7 @@ import logging import os import time +from typing import Dict, List from fastlane_bot.tests.deterministic.dtest_constants import ( TEST_DATA_DIR, @@ -63,7 +64,7 @@ def wait_for_file(file_path: str, logger: logging.Logger, timeout: int = 120, ch return True @staticmethod - def load_json_data(file_path: str) -> dict: + def load_json_data(file_path: str) -> Dict: """Safely load JSON data from a file. Args: @@ -76,7 +77,7 @@ def load_json_data(file_path: str) -> dict: return json.load(file) @staticmethod - def read_transaction_files(log_folder: str) -> list: + def read_transaction_files(log_folder: str) -> List: """Read all transaction files in a folder and return their content. Args: @@ -92,7 +93,7 @@ def read_transaction_files(log_folder: str) -> list: transactions.append(file.read()) return transactions - def tx_scanner(self, args: argparse.Namespace) -> list: + def tx_scanner(self, args: argparse.Namespace) -> List: """ Scan for successful transactions in the most recent log folder. @@ -119,7 +120,7 @@ def tx_scanner(self, args: argparse.Namespace) -> list: return successful_txs @staticmethod - def clean_tx_data(tx_data: dict) -> dict: + def clean_tx_data(tx_data: Dict) -> Dict: """ This method takes a transaction data dictionary and removes the cid0 key from the trades. @@ -138,7 +139,7 @@ def clean_tx_data(tx_data: dict) -> dict: return tx_data @staticmethod - def get_tx_data(strategy_id: int, txt_all_successful_txs: list) -> dict: + def get_tx_data(strategy_id: int, txt_all_successful_txs: List) -> Dict: """ This method takes a list of successful transactions and a strategy_id and returns the transaction data for the given strategy_id. @@ -161,7 +162,7 @@ def get_tx_data(strategy_id: int, txt_all_successful_txs: list) -> dict: ) @staticmethod - def load_json_file(file_name: str, args: argparse.Namespace) -> dict: + def load_json_file(file_name: str, args: argparse.Namespace) -> Dict: """ This method loads a json file and returns the data as a dictionary. @@ -187,7 +188,7 @@ def load_json_file(file_name: str, args: argparse.Namespace) -> dict: return data @staticmethod - def log_txs(tx_list: list, args: argparse.Namespace): + def log_txs(tx_list: List, args: argparse.Namespace): """ This method logs the transactions in a list. @@ -198,7 +199,7 @@ def log_txs(tx_list: list, args: argparse.Namespace): for i, tx in enumerate(tx_list): args.logger.debug(f"\nsuccessful_txs[{i}]: {tx}") - def log_results(self, args: argparse.Namespace, actual_txs: list, expected_txs: dict, test_strategy_txhashs: dict) -> dict: + def log_results(self, args: argparse.Namespace, actual_txs: List, expected_txs: Dict, test_strategy_txhashs: Dict) -> Dict: """ Logs the results of the tests and returns a dictionary with the results. @@ -246,8 +247,8 @@ def log_results(self, args: argparse.Namespace, actual_txs: list, expected_txs: @staticmethod def log_test_failure(test_id: int, - reason: str, results_description: dict, - tx_data: dict = None, expected_data: dict = None): + reason: str, results_description: Dict, + tx_data: Dict = None, expected_data: Dict = None): """ Logs a test failure. @@ -279,7 +280,7 @@ def log_final_result(args: argparse.Namespace, all_tests_passed: bool): else: args.logger.warning("SOME TESTS FAILED") - def wait_for_txs(self, args: argparse.Namespace) -> dict: + def wait_for_txs(self, args: argparse.Namespace) -> Dict: """ This method waits for the transactions to be executed and returns the test strategy txhashs. diff --git a/fastlane_bot/tests/deterministic/dtest_wallet.py b/fastlane_bot/tests/deterministic/dtest_wallet.py index 1f1942d27..ad9b026c2 100644 --- a/fastlane_bot/tests/deterministic/dtest_wallet.py +++ b/fastlane_bot/tests/deterministic/dtest_wallet.py @@ -5,6 +5,7 @@ Licensed under MIT License. """ from dataclasses import dataclass +from typing import List from eth_typing import Address from web3 import Web3 @@ -20,7 +21,7 @@ class TestWallet: w3: Web3 address: str or Address # Address after __post_init__, str before - balances: list[ + balances: List[ TestTokenBalance or dict ] = None # List of TokenBalances after __post_init__, list of dicts before From 3c95f60f8cbd08749f89e4ad8addb532bb9b541e Mon Sep 17 00:00:00 2001 From: Mike Casale <46603283+mikewcasale@users.noreply.github.com> Date: Thu, 28 Mar 2024 04:45:38 -0700 Subject: [PATCH 21/41] Cleanup run script for readability --- .../tests/deterministic/dtest_main.py | 212 +++++++++++++++++ run_deterministic_tests.py | 219 +----------------- 2 files changed, 213 insertions(+), 218 deletions(-) create mode 100644 fastlane_bot/tests/deterministic/dtest_main.py diff --git a/fastlane_bot/tests/deterministic/dtest_main.py b/fastlane_bot/tests/deterministic/dtest_main.py new file mode 100644 index 000000000..b02b5441b --- /dev/null +++ b/fastlane_bot/tests/deterministic/dtest_main.py @@ -0,0 +1,212 @@ +import argparse +import logging +import os +import subprocess +import time +from typing import Dict + +from web3 import Web3 + +from fastlane_bot.tests.deterministic.dtest_constants import KNOWN_UNABLE_TO_DELETE, TestCommandLineArgs +from fastlane_bot.tests.deterministic.dtest_manager import TestManager +from fastlane_bot.tests.deterministic.dtest_pool import TestPool +from fastlane_bot.tests.deterministic.dtest_pool_params_builder import TestPoolParamsBuilder +from fastlane_bot.tests.deterministic.dtest_tx_helper import TestTxHelper + + +def get_logger(args: argparse.Namespace) -> logging.Logger: + """ + Get the logger for the script. + """ + logger = logging.getLogger(__name__) + logger.setLevel(args.loglevel) + logger.handlers.clear() # Clear existing handlers to avoid duplicate logging + + # Create console handler + ch = logging.StreamHandler() + ch.setLevel(args.loglevel) + + # Create formatter and add it to the handlers + formatter = logging.Formatter( + "%(asctime)s - %(name)s - %(levelname)s - %(message)s" + ) + ch.setFormatter(formatter) + + # Add the console handler to the logger + logger.addHandler(ch) + + return logger + + +def set_test_state_task(args: argparse.Namespace): + """ + Sets the test state based on the static_pool_data_testing.csv file. + + Args: + args: argparse.Namespace, the command line arguments + """ + args.logger.info("\nRunning set_test_state_task...") + + test_pools = TestPool.load_test_pools() + + pools = [ + TestPool(**test_pools_row[TestPool.attributes()].to_dict()) + for index, test_pools_row in test_pools.iterrows() + ] + pools = [pool for pool in pools if pool.is_supported] + builder = TestPoolParamsBuilder(args.w3) + builder.update_pools_by_exchange(args, builder, pools, args.w3) + + +def get_carbon_strategies_and_delete_task( + test_manager: TestManager, args: argparse.Namespace +): + """ + Get the carbon strategies and delete them. + + Args: + test_manager: TestManager, the test manager + args: argparse.Namespace, the command line arguments + """ + args.logger.info("\nRunning get_carbon_strategies_and_delete_task...") + + # Get the state of the carbon strategies + ( + strategy_created_df, + strategy_deleted_df, + remaining_carbon_strategies, + ) = test_manager.get_state_of_carbon_strategies(args, args.from_block) + + # takes about 4 minutes per 100 strategies, so 450 ~ 18 minutes + undeleted_strategies = test_manager.delete_all_carbon_strategies( + args, remaining_carbon_strategies + ) + + # These strategies cannot be deleted on Ethereum + assert all( + x in KNOWN_UNABLE_TO_DELETE for x in undeleted_strategies + ), f"Strategies not deleted that are unknown: {undeleted_strategies}" + + +def run_tests_on_mode_task( + args: argparse.Namespace, + test_manager: TestManager, + test_strategies: Dict, +): + """ + Run tests on the specified arbitrage mode. + + Args: + args: argparse.Namespace, the command line arguments + test_manager: TestManager, the test manager + test_strategies: Dict, the test strategies + """ + args.logger.info("\nRunning run_tests_on_mode_task...") + + # Get the default main.py CL args, then overwrite based on the current command line args + default_main_args = test_manager.overwrite_command_line_args(args) + + # Print the default main args + args.logger.debug(f"command-line args: {default_main_args}") + + # Run the main.py script with the default main args + cmd_args = ["python", "main.py"] + TestCommandLineArgs.args_to_command_line( + default_main_args + ) + proc = subprocess.Popen(cmd_args) + time.sleep(3) + most_recent_pool_data_path = test_manager.get_most_recent_pool_data_path(args) + + # Wait for the main.py script to create the latest_pool_data.json file + while not os.path.exists(most_recent_pool_data_path): + time.sleep(3) + args.logger.debug("Waiting for pool data...") + + strats_created_from_block = test_manager.get_strats_created_from_block( + args, test_manager.w3 + ) + + # Approve and create the strategies + test_strategy_txhashs = test_manager.approve_and_create_strategies( + args, test_strategies, strats_created_from_block + ) + + # Write the strategy txhashs to a json file + test_manager.write_strategy_txhashs_to_json(test_strategy_txhashs) + + # Run the results crosscheck task + run_results_crosscheck_task(args, proc) + + +def run_results_crosscheck_task(args, proc: subprocess.Popen): + """ + Run the results crosscheck task. + + Args: + args: argparse.Namespace, the command line arguments + proc: subprocess.Popen, the process + """ + args.logger.info("\nRunning run_results_crosscheck_task...") + + # Initialize the tx helper + tx_helper = TestTxHelper() + + # Wait for the transactions to be completed + test_strategy_txhashs = tx_helper.wait_for_txs(args) + + # Scan for successful transactions on Tenderly which are marked by status=1 + actual_txs = tx_helper.tx_scanner(args) + expected_txs = tx_helper.load_json_file("test_results.json", args) + + results_description = tx_helper.log_results( + args, actual_txs, expected_txs, test_strategy_txhashs + ) + proc.terminate() + for k, v in results_description.items(): + args.logger.info(f"{k}: {v}") + + +def main(args: argparse.Namespace): + """ + Main function for the script. Runs the specified task based on the command line arguments. + + Args: + args: argparse.Namespace, the command line arguments + """ + + # Set up the logger + args.logger = get_logger(args) + args.logger.info(f"Running task: {args.task}") + + # Set the timeout in seconds + args.timeout = args.timeout_minutes * 60 + + if str(args.create_new_testnet).lower() == "true": + uri, from_block = TestManager.create_new_testnet() + args.rpc_url = uri + + # Initialize the Web3 Manager + test_manager = TestManager(args=args) + if args.delete_old_logs: + test_manager.delete_old_logs(args) + + if args.task == "set_test_state": + set_test_state_task(test_manager.w3) + elif args.task == "get_carbon_strategies_and_delete": + get_carbon_strategies_and_delete_task(test_manager, args) + elif args.task == "run_tests_on_mode": + _extracted_run_tests_on_mode(test_manager, args) + elif args.task == "end_to_end": + get_carbon_strategies_and_delete_task(test_manager, args) + set_test_state_task(test_manager.w3) + _extracted_run_tests_on_mode(test_manager, args) + else: + raise ValueError(f"Task {args.task} not recognized") + + +def _extracted_run_tests_on_mode(test_manager: TestManager, args: argparse.Namespace): + """ + Extracted task handling. + """ + test_strategies = test_manager.get_test_strategies(args) + run_tests_on_mode_task(args, test_manager, test_strategies) \ No newline at end of file diff --git a/run_deterministic_tests.py b/run_deterministic_tests.py index adb7f401c..19c087d82 100644 --- a/run_deterministic_tests.py +++ b/run_deterministic_tests.py @@ -30,10 +30,6 @@ - `True`: Create a new testnet. - `False`: Do not create a new testnet. -The script uses the `fastlane_bot/tests/deterministic/dtest_constants.py` file to get the constants used in the tests. - -The script uses the `fastlane_bot/tests/deterministic/utils.py` file to get the utility functions used in the tests. - All data used in the tests is stored in the `fastlane_bot/tests/deterministic/_data` directory. Note: This script uses the function `get_default_main_args` which returns the default command line arguments for the @@ -44,225 +40,12 @@ Licensed under MIT License. """ import argparse -import logging -import os -import subprocess -import time -from typing import Dict - -from web3 import Web3 from fastlane_bot.tests.deterministic.dtest_constants import ( DEFAULT_FROM_BLOCK, - KNOWN_UNABLE_TO_DELETE, TENDERLY_RPC_KEY, - TestCommandLineArgs, -) -from fastlane_bot.tests.deterministic.dtest_manager import TestManager -from fastlane_bot.tests.deterministic.dtest_pool import TestPool -from fastlane_bot.tests.deterministic.dtest_pool_params_builder import ( - TestPoolParamsBuilder, ) -from fastlane_bot.tests.deterministic.dtest_tx_helper import TestTxHelper - - -def get_logger(args: argparse.Namespace) -> logging.Logger: - """ - Get the logger for the script. - """ - logger = logging.getLogger(__name__) - logger.setLevel(args.loglevel) - logger.handlers.clear() # Clear existing handlers to avoid duplicate logging - - # Create console handler - ch = logging.StreamHandler() - ch.setLevel(args.loglevel) - - # Create formatter and add it to the handlers - formatter = logging.Formatter( - "%(asctime)s - %(name)s - %(levelname)s - %(message)s" - ) - ch.setFormatter(formatter) - - # Add the console handler to the logger - logger.addHandler(ch) - - return logger - - -def set_test_state_task(w3: Web3): - """ - Sets the test state based on the static_pool_data_testing.csv file. - - Args: - w3: Web3 instance - """ - args.logger.info("\nRunning set_test_state_task...") - - test_pools = TestPool.load_test_pools() - - pools = [ - TestPool(**test_pools_row[TestPool.attributes()].to_dict()) - for index, test_pools_row in test_pools.iterrows() - ] - pools = [pool for pool in pools if pool.is_supported] - builder = TestPoolParamsBuilder(w3) - builder.update_pools_by_exchange(args, builder, pools, w3) - - -def get_carbon_strategies_and_delete_task( - test_manager: TestManager, args: argparse.Namespace -): - """ - Get the carbon strategies and delete them. - - Args: - test_manager: TestManager, the test manager - args: argparse.Namespace, the command line arguments - """ - args.logger.info("\nRunning get_carbon_strategies_and_delete_task...") - - # Get the state of the carbon strategies - ( - strategy_created_df, - strategy_deleted_df, - remaining_carbon_strategies, - ) = test_manager.get_state_of_carbon_strategies(args, args.from_block) - - # takes about 4 minutes per 100 strategies, so 450 ~ 18 minutes - undeleted_strategies = test_manager.delete_all_carbon_strategies( - args, remaining_carbon_strategies - ) - - # These strategies cannot be deleted on Ethereum - assert all( - x in KNOWN_UNABLE_TO_DELETE for x in undeleted_strategies - ), f"Strategies not deleted that are unknown: {undeleted_strategies}" - - -def run_tests_on_mode_task( - args: argparse.Namespace, - test_manager: TestManager, - test_strategies: Dict, -): - """ - Run tests on the specified arbitrage mode. - - Args: - args: argparse.Namespace, the command line arguments - test_manager: TestManager, the test manager - test_strategies: Dict, the test strategies - """ - args.logger.info("\nRunning run_tests_on_mode_task...") - - # Get the default main.py CL args, then overwrite based on the current command line args - default_main_args = test_manager.overwrite_command_line_args(args) - - # Print the default main args - args.logger.debug(f"command-line args: {default_main_args}") - - # Run the main.py script with the default main args - cmd_args = ["python", "main.py"] + TestCommandLineArgs.args_to_command_line( - default_main_args - ) - proc = subprocess.Popen(cmd_args) - time.sleep(3) - most_recent_pool_data_path = test_manager.get_most_recent_pool_data_path(args) - - # Wait for the main.py script to create the latest_pool_data.json file - while not os.path.exists(most_recent_pool_data_path): - time.sleep(3) - args.logger.debug("Waiting for pool data...") - - strats_created_from_block = test_manager.get_strats_created_from_block( - args, test_manager.w3 - ) - - # Approve and create the strategies - test_strategy_txhashs = test_manager.approve_and_create_strategies( - args, test_strategies, strats_created_from_block - ) - - # Write the strategy txhashs to a json file - test_manager.write_strategy_txhashs_to_json(test_strategy_txhashs) - - # Run the results crosscheck task - run_results_crosscheck_task(args, proc) - - -def run_results_crosscheck_task(args, proc: subprocess.Popen): - """ - Run the results crosscheck task. - - Args: - args: argparse.Namespace, the command line arguments - proc: subprocess.Popen, the process - """ - args.logger.info("\nRunning run_results_crosscheck_task...") - - # Initialize the tx helper - tx_helper = TestTxHelper() - - # Wait for the transactions to be completed - test_strategy_txhashs = tx_helper.wait_for_txs(args) - - # Scan for successful transactions on Tenderly which are marked by status=1 - actual_txs = tx_helper.tx_scanner(args) - expected_txs = tx_helper.load_json_file("test_results.json", args) - - results_description = tx_helper.log_results( - args, actual_txs, expected_txs, test_strategy_txhashs - ) - proc.terminate() - for k, v in results_description.items(): - args.logger.info(f"{k}: {v}") - - -def main(args: argparse.Namespace): - """ - Main function for the script. Runs the specified task based on the command line arguments. - - Args: - args: argparse.Namespace, the command line arguments - """ - - # Set up the logger - args.logger = get_logger(args) - args.logger.info(f"Running task: {args.task}") - - # Set the timeout in seconds - args.timeout = args.timeout_minutes * 60 - - if str(args.create_new_testnet).lower() == "true": - uri, from_block = TestManager.create_new_testnet() - args.rpc_url = uri - - # Initialize the Web3 Manager - test_manager = TestManager(args=args) - if args.delete_old_logs: - test_manager.delete_old_logs(args) - - if args.task == "set_test_state": - set_test_state_task(test_manager.w3) - elif args.task == "get_carbon_strategies_and_delete": - get_carbon_strategies_and_delete_task(test_manager, args) - elif args.task == "run_tests_on_mode": - _extracted_run_tests_on_mode(test_manager, args) - elif args.task == "end_to_end": - get_carbon_strategies_and_delete_task(test_manager, args) - set_test_state_task(test_manager.w3) - _extracted_run_tests_on_mode(test_manager, args) - else: - raise ValueError(f"Task {args.task} not recognized") - - -def _extracted_run_tests_on_mode(test_manager: TestManager, args: argparse.Namespace): - """ - Extracted task handling. - """ - test_strategies = test_manager.get_test_strategies(args) - run_tests_on_mode_task(args, test_manager, test_strategies) - +from fastlane_bot.tests.deterministic.dtest_main import main if __name__ == "__main__": # Parse the command line arguments From 5a6518c35965d484c8dedd29d0ef806ec60ab5ca Mon Sep 17 00:00:00 2001 From: Mike Casale <46603283+mikewcasale@users.noreply.github.com> Date: Thu, 28 Mar 2024 04:51:43 -0700 Subject: [PATCH 22/41] Addressing comment about this class not being a constant. see https://github.com/bancorprotocol/fastlane-bot/pull/399#discussion_r1535340526 --- .../deterministic/dtest_cmd_line_args.py | 63 +++++++++++++++++++ .../tests/deterministic/dtest_constants.py | 61 ------------------ .../tests/deterministic/dtest_main.py | 3 +- .../tests/deterministic/dtest_manager.py | 2 +- 4 files changed, 66 insertions(+), 63 deletions(-) create mode 100644 fastlane_bot/tests/deterministic/dtest_cmd_line_args.py diff --git a/fastlane_bot/tests/deterministic/dtest_cmd_line_args.py b/fastlane_bot/tests/deterministic/dtest_cmd_line_args.py new file mode 100644 index 000000000..e3cfa5fdc --- /dev/null +++ b/fastlane_bot/tests/deterministic/dtest_cmd_line_args.py @@ -0,0 +1,63 @@ +from dataclasses import dataclass + +from fastlane_bot.tools.cpc import T + + +@dataclass +class TestCommandLineArgs: + """ + This class is used to mock the command line arguments for the main.py + """ + + cache_latest_only: str = "True" + backdate_pools: str = "True" + static_pool_data_filename: str = "static_pool_data_testing" + arb_mode: str = "multi_pairwise_all" + flashloan_tokens: str = ( + f"{T.LINK},{T.NATIVE_ETH},{T.BNT},{T.WBTC},{T.DAI},{T.USDC},{T.USDT},{T.WETH}" + ) + n_jobs: int = -1 + exchanges: str = "carbon_v1,bancor_v3,bancor_v2,bancor_pol,uniswap_v3,uniswap_v2,sushiswap_v2,balancer,pancakeswap_v2,pancakeswap_v3" + polling_interval: int = 0 + alchemy_max_block_fetch: int = 1 + reorg_delay: int = 0 + logging_path: str = "logs_dtest" + loglevel: str = "INFO" + use_cached_events: str = "False" + run_data_validator: str = "False" + randomizer: int = 1 + limit_bancor3_flashloan_tokens: str = "True" + default_min_profit_gas_token: str = "0.002" # "0.01" + timeout: int = None + target_tokens: str = None + replay_from_block: int = None + tenderly_fork_id: int = None + tenderly_event_exchanges: str = "pancakeswap_v2,pancakeswap_v3" + increment_time: int = 1 + increment_blocks: int = 1 + blockchain: str = "ethereum" + pool_data_update_frequency: int = -1 + use_specific_exchange_for_target_tokens: str = None + prefix_path: str = "" + version_check_frequency: int = 1 + self_fund: str = "False" + read_only: str = "False" + is_args_test: str = "False" + rpc_url: str = None + + @staticmethod + def args_to_command_line(args): + """ + Convert a TestCommandLineArgs instance to a list of command-line arguments. + + Args: + args: An instance of TestCommandLineArgs. + + Returns: + A list of command-line arguments. + """ + cmd_args = [] + for field, value in args.__dict__.items(): + if value is not None: # Only include fields that have a value + cmd_args.extend((f"--{field}", str(value))) + return cmd_args diff --git a/fastlane_bot/tests/deterministic/dtest_constants.py b/fastlane_bot/tests/deterministic/dtest_constants.py index a4dd154e5..4e6f55b54 100644 --- a/fastlane_bot/tests/deterministic/dtest_constants.py +++ b/fastlane_bot/tests/deterministic/dtest_constants.py @@ -4,9 +4,6 @@ (c) Copyright Bprotocol foundation 2024. Licensed under MIT License. """ -from dataclasses import dataclass - -from fastlane_bot.tools.cpc import T KNOWN_UNABLE_TO_DELETE = { 68737038118029569619601670701217178714718: ("pDFS", "ETH"), @@ -62,61 +59,3 @@ } -@dataclass -class TestCommandLineArgs: - """ - This class is used to mock the command line arguments for the main.py - """ - - cache_latest_only: str = "True" - backdate_pools: str = "True" - static_pool_data_filename: str = "static_pool_data_testing" - arb_mode: str = "multi_pairwise_all" - flashloan_tokens: str = ( - f"{T.LINK},{T.NATIVE_ETH},{T.BNT},{T.WBTC},{T.DAI},{T.USDC},{T.USDT},{T.WETH}" - ) - n_jobs: int = -1 - exchanges: str = "carbon_v1,bancor_v3,bancor_v2,bancor_pol,uniswap_v3,uniswap_v2,sushiswap_v2,balancer,pancakeswap_v2,pancakeswap_v3" - polling_interval: int = 0 - alchemy_max_block_fetch: int = 1 - reorg_delay: int = 0 - logging_path: str = "logs_dtest" - loglevel: str = "INFO" - use_cached_events: str = "False" - run_data_validator: str = "False" - randomizer: int = 1 - limit_bancor3_flashloan_tokens: str = "True" - default_min_profit_gas_token: str = "0.002" # "0.01" - timeout: int = None - target_tokens: str = None - replay_from_block: int = None - tenderly_fork_id: int = None - tenderly_event_exchanges: str = "pancakeswap_v2,pancakeswap_v3" - increment_time: int = 1 - increment_blocks: int = 1 - blockchain: str = "ethereum" - pool_data_update_frequency: int = -1 - use_specific_exchange_for_target_tokens: str = None - prefix_path: str = "" - version_check_frequency: int = 1 - self_fund: str = "False" - read_only: str = "False" - is_args_test: str = "False" - rpc_url: str = None - - @staticmethod - def args_to_command_line(args): - """ - Convert a TestCommandLineArgs instance to a list of command-line arguments. - - Args: - args: An instance of TestCommandLineArgs. - - Returns: - A list of command-line arguments. - """ - cmd_args = [] - for field, value in args.__dict__.items(): - if value is not None: # Only include fields that have a value - cmd_args.extend((f"--{field}", str(value))) - return cmd_args diff --git a/fastlane_bot/tests/deterministic/dtest_main.py b/fastlane_bot/tests/deterministic/dtest_main.py index b02b5441b..939a4de8b 100644 --- a/fastlane_bot/tests/deterministic/dtest_main.py +++ b/fastlane_bot/tests/deterministic/dtest_main.py @@ -7,7 +7,8 @@ from web3 import Web3 -from fastlane_bot.tests.deterministic.dtest_constants import KNOWN_UNABLE_TO_DELETE, TestCommandLineArgs +from fastlane_bot.tests.deterministic.dtest_constants import KNOWN_UNABLE_TO_DELETE +from fastlane_bot.tests.deterministic.dtest_cmd_line_args import TestCommandLineArgs from fastlane_bot.tests.deterministic.dtest_manager import TestManager from fastlane_bot.tests.deterministic.dtest_pool import TestPool from fastlane_bot.tests.deterministic.dtest_pool_params_builder import TestPoolParamsBuilder diff --git a/fastlane_bot/tests/deterministic/dtest_manager.py b/fastlane_bot/tests/deterministic/dtest_manager.py index 8bf1682e4..2697f46f3 100644 --- a/fastlane_bot/tests/deterministic/dtest_manager.py +++ b/fastlane_bot/tests/deterministic/dtest_manager.py @@ -25,8 +25,8 @@ ETH_ADDRESS, TEST_DATA_DIR, TOKENS_MODIFICATIONS, - TestCommandLineArgs, ) +from fastlane_bot.tests.deterministic.dtest_cmd_line_args import TestCommandLineArgs from fastlane_bot.tests.deterministic.dtest_strategy import TestStrategy From 77a93089a782846e18a0e7cb545112d992974bf1 Mon Sep 17 00:00:00 2001 From: Mike Casale <46603283+mikewcasale@users.noreply.github.com> Date: Thu, 28 Mar 2024 05:20:18 -0700 Subject: [PATCH 23/41] fixes recursive symlink error when invoking ./run_tests script --- .../_data/static_pool_data_testing.csv | 0 .../deterministic/_data/test_results.json | 160 ------------------ .../deterministic/_data/test_strategies.json | 56 ------ .../tests/deterministic/dtest_constants.py | 2 +- .../tests/deterministic/dtest_main.py | 13 +- .../tests/deterministic/dtest_manager.py | 1 + .../tests/deterministic/unit/fastlane_bot | 1 - .../unit => }/test_dtest_tx_helper.py | 0 .../unit => }/test_dtest_wallet.py | 0 log.txt | 1 + resources/NBTest/conftest.py | 5 + 11 files changed, 14 insertions(+), 225 deletions(-) rename fastlane_bot/tests/{deterministic => }/_data/static_pool_data_testing.csv (100%) delete mode 100644 fastlane_bot/tests/deterministic/_data/test_results.json delete mode 100644 fastlane_bot/tests/deterministic/_data/test_strategies.json delete mode 120000 fastlane_bot/tests/deterministic/unit/fastlane_bot rename fastlane_bot/tests/{deterministic/unit => }/test_dtest_tx_helper.py (100%) rename fastlane_bot/tests/{deterministic/unit => }/test_dtest_wallet.py (100%) create mode 100644 log.txt create mode 100644 resources/NBTest/conftest.py diff --git a/fastlane_bot/tests/deterministic/_data/static_pool_data_testing.csv b/fastlane_bot/tests/_data/static_pool_data_testing.csv similarity index 100% rename from fastlane_bot/tests/deterministic/_data/static_pool_data_testing.csv rename to fastlane_bot/tests/_data/static_pool_data_testing.csv diff --git a/fastlane_bot/tests/deterministic/_data/test_results.json b/fastlane_bot/tests/deterministic/_data/test_results.json deleted file mode 100644 index 5d01b25a3..000000000 --- a/fastlane_bot/tests/deterministic/_data/test_results.json +++ /dev/null @@ -1,160 +0,0 @@ -{ - "test_data": { - "1": { - "type": "multi", - "profit_gas_token": 0.113, - "profit_usd": 255.8626, - "flashloan": [ - { - "token": "WETH", - "amount": 1.1466, - "profit": 0.113 - } - ], - "trades": [ - { - "trade_index": 0, - "exchange": "uniswap_v2", - "tkn_in": { - "WETH": "0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2" - }, - "amount_in": 1.1466, - "tkn_out": { - "USDC": "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48" - }, - "amt_out": 2587.6497, - "cid0": "fe98c78ce1" - }, - { - "trade_index": 1, - "exchange": "carbon_v1", - "tkn_in": { - "USDC": "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48" - }, - "amount_in": 2587.6497, - "tkn_out": { - "WETH": "0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2" - }, - "amt_out": 1.2596 - } - ] - }, - "2": { - "type": "multi", - "profit_gas_token": 0.1817, - "profit_usd": 411.3437, - "flashloan": [ - { - "token": "LINK", - "amount": 51.3308, - "profit": 28.1742 - } - ], - "trades": [ - { - "trade_index": 0, - "exchange": "uniswap_v3", - "tkn_in": { - "LINK": "0x514910771AF9Ca656af840dff83E8264EcF986CA" - }, - "amount_in": 51.3308, - "tkn_out": { - "WETH": "0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2" - }, - "amt_out": 0.33, - "cid0": "b800399517" - }, - { - "trade_index": 1, - "exchange": "carbon_v1", - "tkn_in": { - "WETH": "0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2" - }, - "amount_in": 0.33, - "tkn_out": { - "LINK": "0x514910771AF9Ca656af840dff83E8264EcF986CA" - }, - "amt_out": 79.505 - } - ] - }, - "3": { - "type": "multi", - "profit_gas_token": 0.1293, - "profit_usd": 292.8006, - "flashloan": [ - { - "token": "USDC", - "amount": 2697.3, - "profit": 292.8006 - } - ], - "trades": [ - { - "trade_index": 0, - "exchange": "carbon_v1", - "tkn_in": { - "USDC": "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48" - }, - "amount_in": 2697.3, - "tkn_out": { - "DAI": "0x6B175474E89094C44Da98b954EedeAC495271d0F" - }, - "amt_out": 2990.7069 - }, - { - "trade_index": 1, - "exchange": "pancakeswap_v3", - "tkn_in": { - "DAI": "0x6B175474E89094C44Da98b954EedeAC495271d0F" - }, - "amount_in": 2990.7069, - "tkn_out": { - "USDC": "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48" - }, - "amt_out": 2990.1006, - "cid0": "d198cf907e" - } - ] - }, - "4": { - "type": "multi", - "profit_gas_token": 0.0198, - "profit_usd": 44.7633, - "flashloan": [ - { - "token": "WETH", - "amount": 1.3329, - "profit": 0.0198 - } - ], - "trades": [ - { - "trade_index": 0, - "exchange": "carbon_v1", - "tkn_in": { - "WETH": "0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2" - }, - "amount_in": 1.3329, - "tkn_out": { - "WBTC": "0x2260FAC5E5542a773Aa44fBCfeDf7C193bc2C599" - }, - "amt_out": 0.0737 - }, - { - "trade_index": 1, - "exchange": "pancakeswap_v2", - "tkn_in": { - "WBTC": "0x2260FAC5E5542a773Aa44fBCfeDf7C193bc2C599" - }, - "amount_in": 0.0737, - "tkn_out": { - "WETH": "0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2" - }, - "amt_out": 1.3526, - "cid0": "66c0a836b8" - } - ] - } - } -} \ No newline at end of file diff --git a/fastlane_bot/tests/deterministic/_data/test_strategies.json b/fastlane_bot/tests/deterministic/_data/test_strategies.json deleted file mode 100644 index b60e11464..000000000 --- a/fastlane_bot/tests/deterministic/_data/test_strategies.json +++ /dev/null @@ -1,56 +0,0 @@ -{ - "test_strategies": { - "1": { - "token0": "0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE", - "token1": "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48", - "y0": 1267460000029073000, - "z0": 1267460000029073000, - "A0": 0, - "B0": 4411844567835374, - "y1": 7634017977, - "z1": 7649999999, - "A1": 0, - "B1": 11433567603, - "wallet": "0x28C6c06298d514Db089934071355E5743bf21d60" - }, - "2": { - "token0": "0x514910771AF9Ca656af840dff83E8264EcF986CA", - "token1": "0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE", - "y0": 80000000000000000000, - "z0": 80000000000000000000, - "A0": 0, - "B0": 1399247190132272, - "y1": 0, - "z1": 0, - "A1": 0, - "B1": 17102934719008, - "wallet": "0x28C6c06298d514Db089934071355E5743bf21d60" - }, - "3": { - "token0": "0x6B175474E89094C44Da98b954EedeAC495271d0F", - "token1": "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48", - "y0": 3000000000000000000000, - "z0": 3000000000000000000000, - "A0": 0, - "B0": 6052452418541427, - "y1": 0, - "z1": 0, - "A1": 0, - "B1": 0, - "wallet": "0x28C6c06298d514Db089934071355E5743bf21d60" - }, - "4": { - "token0": "0x2260FAC5E5542a773Aa44fBCfeDf7C193bc2C599", - "token1": "0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE", - "y0": 100000000, - "z0": 100000000, - "A0": 17694999, - "B0": 645747883, - "y1": 0, - "z1": 0, - "A1": 0, - "B1": 0, - "wallet": "0x28C6c06298d514Db089934071355E5743bf21d60" - } - } -} \ No newline at end of file diff --git a/fastlane_bot/tests/deterministic/dtest_constants.py b/fastlane_bot/tests/deterministic/dtest_constants.py index 4e6f55b54..f268441fb 100644 --- a/fastlane_bot/tests/deterministic/dtest_constants.py +++ b/fastlane_bot/tests/deterministic/dtest_constants.py @@ -21,7 +21,7 @@ DEFAULT_FROM_BLOCK = 1000000 TENDERLY_RPC_KEY = "fb866397-29bd-4886-8406-a2cc7b7c5b1f" # https://virtual.mainnet.rpc.tenderly.co/9ea4ceb3-d0f5-4faf-959e-f51cf1f6b52b, from_block: 19325893, fb866397-29bd-4886-8406-a2cc7b7c5b1f REAL_DATA_DIR = "fastlane_bot/data/blockchain_data" -TEST_DATA_DIR = "fastlane_bot/tests/deterministic/_data" +TEST_DATA_DIR = "fastlane_bot/tests/_data" binance14 = "0x28C6c06298d514Db089934071355E5743bf21d60" TOKENS_MODIFICATIONS = { "0x0": { diff --git a/fastlane_bot/tests/deterministic/dtest_main.py b/fastlane_bot/tests/deterministic/dtest_main.py index 939a4de8b..51b686090 100644 --- a/fastlane_bot/tests/deterministic/dtest_main.py +++ b/fastlane_bot/tests/deterministic/dtest_main.py @@ -39,14 +39,13 @@ def get_logger(args: argparse.Namespace) -> logging.Logger: return logger -def set_test_state_task(args: argparse.Namespace): +def set_test_state_task(mgr: TestManager): """ Sets the test state based on the static_pool_data_testing.csv file. Args: - args: argparse.Namespace, the command line arguments + mgr: TestManager, the test manager """ - args.logger.info("\nRunning set_test_state_task...") test_pools = TestPool.load_test_pools() @@ -55,8 +54,8 @@ def set_test_state_task(args: argparse.Namespace): for index, test_pools_row in test_pools.iterrows() ] pools = [pool for pool in pools if pool.is_supported] - builder = TestPoolParamsBuilder(args.w3) - builder.update_pools_by_exchange(args, builder, pools, args.w3) + builder = TestPoolParamsBuilder(mgr.w3) + builder.update_pools_by_exchange(mgr.args, builder, pools, mgr.w3) def get_carbon_strategies_and_delete_task( @@ -192,14 +191,14 @@ def main(args: argparse.Namespace): test_manager.delete_old_logs(args) if args.task == "set_test_state": - set_test_state_task(test_manager.w3) + set_test_state_task(test_manager) elif args.task == "get_carbon_strategies_and_delete": get_carbon_strategies_and_delete_task(test_manager, args) elif args.task == "run_tests_on_mode": _extracted_run_tests_on_mode(test_manager, args) elif args.task == "end_to_end": get_carbon_strategies_and_delete_task(test_manager, args) - set_test_state_task(test_manager.w3) + set_test_state_task(test_manager) _extracted_run_tests_on_mode(test_manager, args) else: raise ValueError(f"Task {args.task} not recognized") diff --git a/fastlane_bot/tests/deterministic/dtest_manager.py b/fastlane_bot/tests/deterministic/dtest_manager.py index 2697f46f3..bf953a101 100644 --- a/fastlane_bot/tests/deterministic/dtest_manager.py +++ b/fastlane_bot/tests/deterministic/dtest_manager.py @@ -60,6 +60,7 @@ def __init__(self, args: argparse.Namespace): ) self.carbon_controller = carbon_controller + self.args = args @property def logs_path(self) -> str: diff --git a/fastlane_bot/tests/deterministic/unit/fastlane_bot b/fastlane_bot/tests/deterministic/unit/fastlane_bot deleted file mode 120000 index 57929bed7..000000000 --- a/fastlane_bot/tests/deterministic/unit/fastlane_bot +++ /dev/null @@ -1 +0,0 @@ -../../../../fastlane_bot \ No newline at end of file diff --git a/fastlane_bot/tests/deterministic/unit/test_dtest_tx_helper.py b/fastlane_bot/tests/test_dtest_tx_helper.py similarity index 100% rename from fastlane_bot/tests/deterministic/unit/test_dtest_tx_helper.py rename to fastlane_bot/tests/test_dtest_tx_helper.py diff --git a/fastlane_bot/tests/deterministic/unit/test_dtest_wallet.py b/fastlane_bot/tests/test_dtest_wallet.py similarity index 100% rename from fastlane_bot/tests/deterministic/unit/test_dtest_wallet.py rename to fastlane_bot/tests/test_dtest_wallet.py diff --git a/log.txt b/log.txt new file mode 100644 index 000000000..ad2fed490 --- /dev/null +++ b/log.txt @@ -0,0 +1 @@ +Searching for main.py in /Users/mikewcasale/Documents/GitHub/bancor/fastlane-bot \ No newline at end of file diff --git a/resources/NBTest/conftest.py b/resources/NBTest/conftest.py new file mode 100644 index 000000000..58a2e8e45 --- /dev/null +++ b/resources/NBTest/conftest.py @@ -0,0 +1,5 @@ +"""Inside this file, define a collect_ignore list that specifies the file paths to ignore when running the tests. For +example, to ignore all files in the tests directory that start with the prefix "dtest_", you can use the following +code:""" +collect_ignore = ['dtest_*.py', + ] \ No newline at end of file From 9f495a64feb3f435ae82eba40f58802789fbe6d5 Mon Sep 17 00:00:00 2001 From: Mike Casale <46603283+mikewcasale@users.noreply.github.com> Date: Thu, 28 Mar 2024 05:23:31 -0700 Subject: [PATCH 24/41] addresses comment https://github.com/bancorprotocol/fastlane-bot/pull/399#discussion_r1535343192 --- fastlane_bot/tests/deterministic/dtest_manager.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/fastlane_bot/tests/deterministic/dtest_manager.py b/fastlane_bot/tests/deterministic/dtest_manager.py index bf953a101..4d3964ea3 100644 --- a/fastlane_bot/tests/deterministic/dtest_manager.py +++ b/fastlane_bot/tests/deterministic/dtest_manager.py @@ -33,15 +33,13 @@ class TestManager: """ A class to manage Web3 contracts and Carbon strategies. + + Args: + args (argparse.Namespace): The command-line arguments. """ def __init__(self, args: argparse.Namespace): - """ - Initializes the TestManager. - - Args: - args (argparse.Namespace): The command-line arguments. - """ + self.w3 = Web3(Web3.HTTPProvider(args.rpc_url, {"timeout": 60})) assert self.w3.is_connected(), "Web3 connection failed" From 8541f60b5e2418177edda8de05454bcfd118dc1c Mon Sep 17 00:00:00 2001 From: Mike Casale <46603283+mikewcasale@users.noreply.github.com> Date: Thu, 28 Mar 2024 05:29:21 -0700 Subject: [PATCH 25/41] addresses comment about file docstrings new format see https://github.com/bancorprotocol/fastlane-bot/pull/399#discussion_r1535344378 --- .../tests/deterministic/dtest_cmd_line_args.py | 7 +++++++ fastlane_bot/tests/deterministic/dtest_constants.py | 1 + fastlane_bot/tests/deterministic/dtest_main.py | 11 ++++++++--- fastlane_bot/tests/deterministic/dtest_manager.py | 3 ++- fastlane_bot/tests/deterministic/dtest_pool.py | 1 + .../tests/deterministic/dtest_pool_params_builder.py | 1 + fastlane_bot/tests/deterministic/dtest_strategy.py | 1 + fastlane_bot/tests/deterministic/dtest_token.py | 1 + fastlane_bot/tests/deterministic/dtest_tx_helper.py | 1 + fastlane_bot/tests/deterministic/dtest_wallet.py | 1 + 10 files changed, 24 insertions(+), 4 deletions(-) diff --git a/fastlane_bot/tests/deterministic/dtest_cmd_line_args.py b/fastlane_bot/tests/deterministic/dtest_cmd_line_args.py index e3cfa5fdc..9025330e8 100644 --- a/fastlane_bot/tests/deterministic/dtest_cmd_line_args.py +++ b/fastlane_bot/tests/deterministic/dtest_cmd_line_args.py @@ -1,3 +1,10 @@ +""" +This file contains the TestCommandLineArgs class, which is a dataclass that represents the command-line arguments for the main.py. + +(c) Copyright Bprotocol foundation 2024. +All rights reserved. +Licensed under MIT License. +""" from dataclasses import dataclass from fastlane_bot.tools.cpc import T diff --git a/fastlane_bot/tests/deterministic/dtest_constants.py b/fastlane_bot/tests/deterministic/dtest_constants.py index f268441fb..863b18a65 100644 --- a/fastlane_bot/tests/deterministic/dtest_constants.py +++ b/fastlane_bot/tests/deterministic/dtest_constants.py @@ -2,6 +2,7 @@ This file contains constants used in the deterministic tests. (c) Copyright Bprotocol foundation 2024. +All rights reserved. Licensed under MIT License. """ diff --git a/fastlane_bot/tests/deterministic/dtest_main.py b/fastlane_bot/tests/deterministic/dtest_main.py index 51b686090..8d4402069 100644 --- a/fastlane_bot/tests/deterministic/dtest_main.py +++ b/fastlane_bot/tests/deterministic/dtest_main.py @@ -1,3 +1,10 @@ +"""This file contains the main function for the deterministic tests. Used to run the specified task based on the +command line arguments defined in `run_deterministic_tests.py`. + +(c) Copyright Bprotocol foundation 2024. +All rights reserved. +Licensed under MIT License. +""" import argparse import logging import os @@ -5,10 +12,8 @@ import time from typing import Dict -from web3 import Web3 - -from fastlane_bot.tests.deterministic.dtest_constants import KNOWN_UNABLE_TO_DELETE from fastlane_bot.tests.deterministic.dtest_cmd_line_args import TestCommandLineArgs +from fastlane_bot.tests.deterministic.dtest_constants import KNOWN_UNABLE_TO_DELETE from fastlane_bot.tests.deterministic.dtest_manager import TestManager from fastlane_bot.tests.deterministic.dtest_pool import TestPool from fastlane_bot.tests.deterministic.dtest_pool_params_builder import TestPoolParamsBuilder diff --git a/fastlane_bot/tests/deterministic/dtest_manager.py b/fastlane_bot/tests/deterministic/dtest_manager.py index 4d3964ea3..ce037723b 100644 --- a/fastlane_bot/tests/deterministic/dtest_manager.py +++ b/fastlane_bot/tests/deterministic/dtest_manager.py @@ -2,6 +2,7 @@ A module to manage Carbon strategies. (c) Copyright Bprotocol foundation 2024. +All rights reserved. Licensed under MIT License. """ import argparse @@ -39,7 +40,7 @@ class TestManager: """ def __init__(self, args: argparse.Namespace): - + self.w3 = Web3(Web3.HTTPProvider(args.rpc_url, {"timeout": 60})) assert self.w3.is_connected(), "Web3 connection failed" diff --git a/fastlane_bot/tests/deterministic/dtest_pool.py b/fastlane_bot/tests/deterministic/dtest_pool.py index d6bfdfe4b..bb04655d9 100644 --- a/fastlane_bot/tests/deterministic/dtest_pool.py +++ b/fastlane_bot/tests/deterministic/dtest_pool.py @@ -2,6 +2,7 @@ This module contains the Wallet class, which is a dataclass that represents a wallet on the given network. (c) Copyright Bprotocol foundation 2024. +All rights reserved. Licensed under MIT License. """ import argparse diff --git a/fastlane_bot/tests/deterministic/dtest_pool_params_builder.py b/fastlane_bot/tests/deterministic/dtest_pool_params_builder.py index ad4420d23..4363881f9 100644 --- a/fastlane_bot/tests/deterministic/dtest_pool_params_builder.py +++ b/fastlane_bot/tests/deterministic/dtest_pool_params_builder.py @@ -3,6 +3,7 @@ encode them into a string that can be used to update the storage of the pool contract. (c) Copyright Bprotocol foundation 2024. +All rights reserved. Licensed under MIT License. """ import re diff --git a/fastlane_bot/tests/deterministic/dtest_strategy.py b/fastlane_bot/tests/deterministic/dtest_strategy.py index d67b2db00..2980821d0 100644 --- a/fastlane_bot/tests/deterministic/dtest_strategy.py +++ b/fastlane_bot/tests/deterministic/dtest_strategy.py @@ -2,6 +2,7 @@ This file contains the Strategy class, which is used to represent a strategy in the deterministic tests. (c) Copyright Bprotocol foundation 2024. +All rights reserved. Licensed under MIT License. """ import argparse diff --git a/fastlane_bot/tests/deterministic/dtest_token.py b/fastlane_bot/tests/deterministic/dtest_token.py index 8aed1aeb6..3aca55552 100644 --- a/fastlane_bot/tests/deterministic/dtest_token.py +++ b/fastlane_bot/tests/deterministic/dtest_token.py @@ -2,6 +2,7 @@ Token class to store token address and contract object for interacting with the token on the blockchain. (c) Copyright Bprotocol foundation 2024. +All rights reserved. Licensed under MIT License. """ from dataclasses import dataclass diff --git a/fastlane_bot/tests/deterministic/dtest_tx_helper.py b/fastlane_bot/tests/deterministic/dtest_tx_helper.py index 0d18784ec..fcb9366b4 100644 --- a/fastlane_bot/tests/deterministic/dtest_tx_helper.py +++ b/fastlane_bot/tests/deterministic/dtest_tx_helper.py @@ -3,6 +3,7 @@ and clean and extract the transaction data. (c) Copyright Bprotocol foundation 2024. +All rights reserved. Licensed under MIT License. """ import argparse diff --git a/fastlane_bot/tests/deterministic/dtest_wallet.py b/fastlane_bot/tests/deterministic/dtest_wallet.py index ad9b026c2..3d53d63ff 100644 --- a/fastlane_bot/tests/deterministic/dtest_wallet.py +++ b/fastlane_bot/tests/deterministic/dtest_wallet.py @@ -2,6 +2,7 @@ This module contains the Wallet class, which is a dataclass that represents a wallet on the Ethereum network. (c) Copyright Bprotocol foundation 2024. +All rights reserved. Licensed under MIT License. """ from dataclasses import dataclass From af5c22ffd0f13072be0d17a5ee3d3e11e23e955b Mon Sep 17 00:00:00 2001 From: Mike Casale <46603283+mikewcasale@users.noreply.github.com> Date: Thu, 28 Mar 2024 05:37:12 -0700 Subject: [PATCH 26/41] changes print statements to use the logger, addresses comment https://github.com/bancorprotocol/fastlane-bot/pull/399#discussion_r1535388533 --- .../tests/deterministic/dtest_main.py | 2 +- .../dtest_pool_params_builder.py | 24 +++++++++---------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/fastlane_bot/tests/deterministic/dtest_main.py b/fastlane_bot/tests/deterministic/dtest_main.py index 8d4402069..66ab1b53b 100644 --- a/fastlane_bot/tests/deterministic/dtest_main.py +++ b/fastlane_bot/tests/deterministic/dtest_main.py @@ -59,7 +59,7 @@ def set_test_state_task(mgr: TestManager): for index, test_pools_row in test_pools.iterrows() ] pools = [pool for pool in pools if pool.is_supported] - builder = TestPoolParamsBuilder(mgr.w3) + builder = TestPoolParamsBuilder(mgr.args, mgr.w3) builder.update_pools_by_exchange(mgr.args, builder, pools, mgr.w3) diff --git a/fastlane_bot/tests/deterministic/dtest_pool_params_builder.py b/fastlane_bot/tests/deterministic/dtest_pool_params_builder.py index 4363881f9..10e914414 100644 --- a/fastlane_bot/tests/deterministic/dtest_pool_params_builder.py +++ b/fastlane_bot/tests/deterministic/dtest_pool_params_builder.py @@ -6,9 +6,10 @@ All rights reserved. Licensed under MIT License. """ +import argparse import re from dataclasses import dataclass -from typing import Dict, List, Tuple +from typing import Any, Dict, List, Tuple import eth_abi from web3 import Web3 @@ -32,8 +33,9 @@ class TestPoolParamsBuilder: This class is used to build the parameters for the test pool. """ - def __init__(self, w3: Web3): + def __init__(self, args: argparse.Namespace, w3: Web3): self.w3 = w3 + self.args = args @staticmethod def convert_to_bool(value: str or int) -> bool: @@ -44,19 +46,17 @@ def convert_to_bool(value: str or int) -> bool: return value.lower() in ["true", "1"] return bool(value) - @staticmethod - def safe_int_conversion(value: any) -> int or None: + def safe_int_conversion(self, value: Any) -> int or None: """ This method is used to convert a value to an integer. """ try: return int(value) except (ValueError, TypeError): - print(f"Error converting {value} to int") + self.args.logger.error(f"Error converting {value} to int") return None - @staticmethod - def append_zeros(value: any, type_str: str) -> str: + def append_zeros(self, value: any, type_str: str) -> str: """ This method is used to append zeros to a value based on the type. """ @@ -72,7 +72,7 @@ def append_zeros(value: any, type_str: str) -> str: length = int(re.search(r"\d+", type_str).group()) // 4 result = "0" * (length - len(hex_value)) + hex_value except Exception as e: - print(f"Error building append_zeros {str(e)}") + self.args.logger.error(f"Error building append_zeros {str(e)}") return result def build_type_val_dict( @@ -113,7 +113,7 @@ def get_latest_block_timestamp(self): try: return int(self.w3.eth.get_block("latest")["timestamp"]) except Exception as e: - print(f"Error fetching latest block timestamp: {e}") + self.args.logger.error(f"Error fetching latest block timestamp: {e}") return None def encode_params(self, type_val_dict: Dict, param_list_single: List[str]) -> str: @@ -135,7 +135,7 @@ def encode_params(self, type_val_dict: Dict, param_list_single: List[str]) -> st ) return "0x" + "0" * (64 - len(result)) + result except Exception as e: - print(f"Error encoding params: {e}, {type_val_dict}") + self.args.logger.error(f"Error encoding params: {e}, {type_val_dict}") return None def get_update_params_dict(self, pool: TestPool) -> Dict: @@ -171,8 +171,8 @@ def set_storage_at(self, pool_address: str, update_params_dict_single: Dict): update_params_dict_single["encoded_params"], ], ) - print(f"[set_storage_at] {pool_address}, {update_params_dict_single['slot']}") - print( + self.args.logger.debug(f"[set_storage_at] {pool_address}, {update_params_dict_single['slot']}") + self.args.logger.debug( f"[set_storage_at] Updated storage parameters for {pool_address} at slot {update_params_dict_single['slot']}" ) From f68f95b3e768fbd422df7ac72eeaac06148b41d3 Mon Sep 17 00:00:00 2001 From: Mike Casale <46603283+mikewcasale@users.noreply.github.com> Date: Thu, 28 Mar 2024 05:38:15 -0700 Subject: [PATCH 27/41] addresses comment https://github.com/bancorprotocol/fastlane-bot/pull/399#discussion_r1535449099 --- fastlane_bot/tests/deterministic/dtest_token.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastlane_bot/tests/deterministic/dtest_token.py b/fastlane_bot/tests/deterministic/dtest_token.py index 3aca55552..21f3b99ce 100644 --- a/fastlane_bot/tests/deterministic/dtest_token.py +++ b/fastlane_bot/tests/deterministic/dtest_token.py @@ -21,7 +21,7 @@ class TestToken: A class to represent a token on the blockchain. Attributes: - address: str or Address + address (str or Address): The address of the token. """ address: str or Address # Address after __post_init__, str before From 4b8e5dc0c7810e71cc659726192a84688ad60ec4 Mon Sep 17 00:00:00 2001 From: Mike Casale <46603283+mikewcasale@users.noreply.github.com> Date: Thu, 28 Mar 2024 05:46:16 -0700 Subject: [PATCH 28/41] Update the docstring for the script --- run_deterministic_tests.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/run_deterministic_tests.py b/run_deterministic_tests.py index 19c087d82..377609989 100644 --- a/run_deterministic_tests.py +++ b/run_deterministic_tests.py @@ -5,7 +5,7 @@ --rpc_url --network --arb_mode ` --timeout_minutes --from_block --create_new_testnet -The `--task` argument specifies the task to run. The options are: +The `--task` argument specifies the task to run. The default task is `end_to_end`. The options are: - `set_test_state`: Set the test state based on the static_pool_data_testing.csv file. - `get_carbon_strategies_and_delete`: Get the carbon strategies and delete them. - `run_tests_on_mode`: Run tests on the specified arbitrage mode. @@ -13,30 +13,31 @@ The `--rpc_url` argument specifies the URL for the RPC endpoint. -The `--network` argument specifies the network to test. The options are: +The `--network` argument specifies the network to test. Default is `ethereum`. The options are: - `ethereum`: Ethereum network. -The `--arb_mode` argument specifies the arbitrage mode to test. The options are: +The `--arb_mode` argument specifies the arbitrage mode to test. Default is `multi`. The options are: - `single`: Single arbitrage mode. - `multi`: Multi arbitrage mode. - `triangle`: Triangle arbitrage mode. - `multi_triangle`: Multi triangle arbitrage mode. -The `--timeout_minutes` argument specifies the timeout for the tests (in minutes). +The `--timeout_minutes` argument specifies the timeout for the tests (in minutes). The default is 10 minutes. -The `--from_block` argument specifies the block number to start from. +The `--from_block` argument specifies the block number to start from. The default is 1000000. -The `--create_new_testnet` argument specifies whether to create a new testnet. The options are: +The `--create_new_testnet` argument specifies whether to create a new testnet. The default is `False`. The options are: - `True`: Create a new testnet. - `False`: Do not create a new testnet. -All data used in the tests is stored in the `fastlane_bot/tests/deterministic/_data` directory. +All data used in the tests is stored in the `fastlane_bot/tests/_data` directory. Note: This script uses the function `get_default_main_args` which returns the default command line arguments for the `main` function in the `main.py` file. If these arguments change in main.py then they should be updated in the `get_default_main_args` function as well. (c) Copyright Bprotocol foundation 2024. +All rights reserved. Licensed under MIT License. """ import argparse From 3832f0f1786524b55a82d590e91a9f0e9544c18d Mon Sep 17 00:00:00 2001 From: Mike Casale <46603283+mikewcasale@users.noreply.github.com> Date: Thu, 28 Mar 2024 05:52:49 -0700 Subject: [PATCH 29/41] addresses comment about cleaner constants imports https://github.com/bancorprotocol/fastlane-bot/pull/399#discussion_r1535456288 --- .../tests/deterministic/dtest_main.py | 4 +-- .../tests/deterministic/dtest_manager.py | 24 +++++++----------- .../tests/deterministic/dtest_pool.py | 9 +++---- .../tests/deterministic/dtest_strategy.py | 25 +++++++------------ .../tests/deterministic/dtest_token.py | 4 +-- .../tests/deterministic/dtest_tx_helper.py | 6 ++--- 6 files changed, 27 insertions(+), 45 deletions(-) diff --git a/fastlane_bot/tests/deterministic/dtest_main.py b/fastlane_bot/tests/deterministic/dtest_main.py index 66ab1b53b..b83d590b9 100644 --- a/fastlane_bot/tests/deterministic/dtest_main.py +++ b/fastlane_bot/tests/deterministic/dtest_main.py @@ -13,7 +13,7 @@ from typing import Dict from fastlane_bot.tests.deterministic.dtest_cmd_line_args import TestCommandLineArgs -from fastlane_bot.tests.deterministic.dtest_constants import KNOWN_UNABLE_TO_DELETE +from fastlane_bot.tests.deterministic import dtest_constants as constants from fastlane_bot.tests.deterministic.dtest_manager import TestManager from fastlane_bot.tests.deterministic.dtest_pool import TestPool from fastlane_bot.tests.deterministic.dtest_pool_params_builder import TestPoolParamsBuilder @@ -89,7 +89,7 @@ def get_carbon_strategies_and_delete_task( # These strategies cannot be deleted on Ethereum assert all( - x in KNOWN_UNABLE_TO_DELETE for x in undeleted_strategies + x in constants.KNOWN_UNABLE_TO_DELETE for x in undeleted_strategies ), f"Strategies not deleted that are unknown: {undeleted_strategies}" diff --git a/fastlane_bot/tests/deterministic/dtest_manager.py b/fastlane_bot/tests/deterministic/dtest_manager.py index ce037723b..4cb1d11b6 100644 --- a/fastlane_bot/tests/deterministic/dtest_manager.py +++ b/fastlane_bot/tests/deterministic/dtest_manager.py @@ -20,13 +20,7 @@ from web3.contract import Contract from fastlane_bot.data.abi import CARBON_CONTROLLER_ABI -from fastlane_bot.tests.deterministic.dtest_constants import ( - DEFAULT_GAS, - DEFAULT_GAS_PRICE, - ETH_ADDRESS, - TEST_DATA_DIR, - TOKENS_MODIFICATIONS, -) +from fastlane_bot.tests.deterministic import dtest_constants as constants from fastlane_bot.tests.deterministic.dtest_cmd_line_args import TestCommandLineArgs from fastlane_bot.tests.deterministic.dtest_strategy import TestStrategy @@ -304,7 +298,7 @@ def set_balance_via_faucet( """ token_address = w3.to_checksum_address(token_address) wallet = w3.to_checksum_address(wallet) - if token_address in {ETH_ADDRESS}: + if token_address in {constants.ETH_ADDRESS}: method = "tenderly_setBalance" params = [[wallet], w3.to_hex(amount_wei)] else: @@ -371,7 +365,7 @@ def modify_tokens_for_deletion(self, args: argparse.Namespace) -> None: Args: args (argparse.Namespace): The command-line arguments. """ - for token_name, details in TOKENS_MODIFICATIONS.items(): + for token_name, details in constants.TOKENS_MODIFICATIONS.items(): args.logger.debug(f"Modifying {token_name} token..., details: {details}") self.modify_token( args, @@ -396,8 +390,8 @@ def create_strategy(self, args: argparse.Namespace, strategy: TestStrategy) -> s tx_params = { "from": strategy.wallet.address, "nonce": strategy.wallet.nonce, - "gasPrice": DEFAULT_GAS_PRICE, - "gas": DEFAULT_GAS, + "gasPrice": constants.DEFAULT_GAS_PRICE, + "gas": constants.DEFAULT_GAS, } if strategy.value: tx_params["value"] = strategy.value @@ -434,8 +428,8 @@ def delete_strategy(self, strategy_id: int, wallet: Address) -> int: tx_params = { "from": wallet, "nonce": nonce, - "gasPrice": DEFAULT_GAS_PRICE, - "gas": DEFAULT_GAS, + "gasPrice": constants.DEFAULT_GAS_PRICE, + "gas": constants.DEFAULT_GAS, } tx_hash = self.carbon_controller.functions.deleteStrategy(strategy_id).transact( tx_params @@ -496,7 +490,7 @@ def get_test_strategies(args: argparse.Namespace) -> Dict: Gets test strategies from a JSON file. """ test_strategies_path = os.path.normpath( - f"{TEST_DATA_DIR}/test_strategies.json" + f"{constants.TEST_DATA_DIR}/test_strategies.json" ) with open(test_strategies_path) as file: test_strategies = json.load(file)["test_strategies"] @@ -546,7 +540,7 @@ def write_strategy_txhashs_to_json(test_strategy_txhashs: Dict): test_strategy_txhashs (dict): The test strategy txhashs. """ test_strategy_txhashs_path = os.path.normpath( - f"{TEST_DATA_DIR}/test_strategy_txhashs.json" + f"{constants.TEST_DATA_DIR}/test_strategy_txhashs.json" ) with open(test_strategy_txhashs_path, "w") as f: json.dump(test_strategy_txhashs, f) diff --git a/fastlane_bot/tests/deterministic/dtest_pool.py b/fastlane_bot/tests/deterministic/dtest_pool.py index bb04655d9..816bb8b80 100644 --- a/fastlane_bot/tests/deterministic/dtest_pool.py +++ b/fastlane_bot/tests/deterministic/dtest_pool.py @@ -14,10 +14,7 @@ from web3 import Web3 from web3.types import RPCEndpoint -from fastlane_bot.tests.deterministic.dtest_constants import ( - SUPPORTED_EXCHANGES, - TEST_DATA_DIR, -) +from fastlane_bot.tests.deterministic import dtest_constants as constants from fastlane_bot.tests.deterministic.dtest_token import TestTokenBalance @@ -81,7 +78,7 @@ def is_supported(self): """ Returns True if the pool is supported, otherwise False. """ - return self.exchange_type in SUPPORTED_EXCHANGES + return self.exchange_type in constants.SUPPORTED_EXCHANGES def set_balance_via_faucet(self, args: argparse.Namespace, w3: Web3, token_id: int): @@ -114,6 +111,6 @@ def set_balance_via_faucet(self, args: argparse.Namespace, def load_test_pools(): # Import pool data static_pool_data_testing_path = os.path.normpath( - f"{TEST_DATA_DIR}/static_pool_data_testing.csv" + f"{constants.TEST_DATA_DIR}/static_pool_data_testing.csv" ) return pd.read_csv(static_pool_data_testing_path, dtype=str) diff --git a/fastlane_bot/tests/deterministic/dtest_strategy.py b/fastlane_bot/tests/deterministic/dtest_strategy.py index 2980821d0..6e43d297f 100644 --- a/fastlane_bot/tests/deterministic/dtest_strategy.py +++ b/fastlane_bot/tests/deterministic/dtest_strategy.py @@ -12,14 +12,7 @@ from web3 import Web3 from fastlane_bot.data.abi import ERC20_ABI -from fastlane_bot.tests.deterministic.dtest_constants import ( - BNT_ADDRESS, - DEFAULT_GAS, - DEFAULT_GAS_PRICE, - TEST_MODE_AMT, - USDC_ADDRESS, - USDT_ADDRESS, -) +from fastlane_bot.tests.deterministic import dtest_constants as constants from fastlane_bot.tests.deterministic.dtest_token import TestToken from fastlane_bot.tests.deterministic.dtest_wallet import TestWallet @@ -78,16 +71,16 @@ def get_token_approval( """ token = self.token0 if token_id == 0 else self.token1 if token.address in [ - BNT_ADDRESS, - USDC_ADDRESS, - USDT_ADDRESS, + constants.BNT_ADDRESS, + constants.USDC_ADDRESS, + constants.USDT_ADDRESS, ]: function_call = token.contract.functions.approve( approval_address, 0 ).transact( { - "gasPrice": DEFAULT_GAS_PRICE, - "gas": DEFAULT_GAS, + "gasPrice": constants.DEFAULT_GAS_PRICE, + "gas": constants.DEFAULT_GAS, "from": self.wallet.address, "nonce": self.wallet.nonce, } @@ -103,11 +96,11 @@ def get_token_approval( args.logger.debug(f"tx_hash = {tx_hash}") function_call = token.contract.functions.approve( - approval_address, TEST_MODE_AMT + approval_address, constants.TEST_MODE_AMT ).transact( { - "gasPrice": DEFAULT_GAS_PRICE, - "gas": DEFAULT_GAS, + "gasPrice": constants.DEFAULT_GAS_PRICE, + "gas": constants.DEFAULT_GAS, "from": self.wallet.address, "nonce": self.wallet.nonce, } diff --git a/fastlane_bot/tests/deterministic/dtest_token.py b/fastlane_bot/tests/deterministic/dtest_token.py index 21f3b99ce..95756fd8a 100644 --- a/fastlane_bot/tests/deterministic/dtest_token.py +++ b/fastlane_bot/tests/deterministic/dtest_token.py @@ -12,7 +12,7 @@ from web3 import Web3 from web3.contract import Contract -from fastlane_bot.tests.deterministic.dtest_constants import ETH_ADDRESS +from fastlane_bot.tests.deterministic import dtest_constants as constants @dataclass @@ -32,7 +32,7 @@ def __post_init__(self): @property def is_eth(self): - return self.address == ETH_ADDRESS + return self.address == constants.ETH_ADDRESS @dataclass diff --git a/fastlane_bot/tests/deterministic/dtest_tx_helper.py b/fastlane_bot/tests/deterministic/dtest_tx_helper.py index fcb9366b4..9e7e326c0 100644 --- a/fastlane_bot/tests/deterministic/dtest_tx_helper.py +++ b/fastlane_bot/tests/deterministic/dtest_tx_helper.py @@ -14,9 +14,7 @@ import time from typing import Dict, List -from fastlane_bot.tests.deterministic.dtest_constants import ( - TEST_DATA_DIR, -) +from fastlane_bot.tests.deterministic import dtest_constants as constants class TestTxHelper: @@ -175,7 +173,7 @@ def load_json_file(file_name: str, args: argparse.Namespace) -> Dict: dict: The data from the json file. """ file_path = ( - os.path.normpath(f"{TEST_DATA_DIR}/{file_name}") + os.path.normpath(f"{constants.TEST_DATA_DIR}/{file_name}") if "/" not in file_name else os.path.normpath(file_name) ) From 573ef3e1930c85990dab0cd77cb4f244f5665b82 Mon Sep 17 00:00:00 2001 From: Mike Casale <46603283+mikewcasale@users.noreply.github.com> Date: Thu, 28 Mar 2024 05:58:01 -0700 Subject: [PATCH 30/41] addresses comment https://github.com/bancorprotocol/fastlane-bot/pull/399#discussion_r1535459564 --- fastlane_bot/tests/deterministic/dtest_constants.py | 1 + fastlane_bot/tests/deterministic/dtest_tx_helper.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/fastlane_bot/tests/deterministic/dtest_constants.py b/fastlane_bot/tests/deterministic/dtest_constants.py index 863b18a65..78dbb77d3 100644 --- a/fastlane_bot/tests/deterministic/dtest_constants.py +++ b/fastlane_bot/tests/deterministic/dtest_constants.py @@ -14,6 +14,7 @@ ) ETH_ADDRESS = "0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE" SUPPORTED_EXCHANGES = ["uniswap_v2", "uniswap_v3", "pancakeswap_v2", "pancakeswap_v3"] +CARBON_V1_FORKS = ["carbon_v1"] # See usage in dtest_tx_helper.py if developing multi-carbon tenderly tests BNT_ADDRESS = "0x1F573D6Fb3F13d689FF844B4cE37794d79a7FF1C" USDC_ADDRESS = "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48" USDT_ADDRESS = "0xdAC17F958D2ee523a2206206994597C13D831ec7" diff --git a/fastlane_bot/tests/deterministic/dtest_tx_helper.py b/fastlane_bot/tests/deterministic/dtest_tx_helper.py index 9e7e326c0..561076fdd 100644 --- a/fastlane_bot/tests/deterministic/dtest_tx_helper.py +++ b/fastlane_bot/tests/deterministic/dtest_tx_helper.py @@ -133,7 +133,7 @@ def clean_tx_data(tx_data: Dict) -> Dict: return tx_data for trade in tx_data["trades"]: - if trade["exchange"] == "carbon_v1" and "cid0" in trade: + if trade["exchange"] in constants.CARBON_V1_FORKS and "cid0" in trade: del trade["cid0"] return tx_data From 23865a50f5f07946675562a2c0a8bb0cf39d3155 Mon Sep 17 00:00:00 2001 From: Mike Casale <46603283+mikewcasale@users.noreply.github.com> Date: Thu, 28 Mar 2024 06:05:08 -0700 Subject: [PATCH 31/41] addressing comment https://github.com/bancorprotocol/fastlane-bot/pull/399#discussion_r1535462036 --- fastlane_bot/tests/deterministic/dtest_tx_helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastlane_bot/tests/deterministic/dtest_tx_helper.py b/fastlane_bot/tests/deterministic/dtest_tx_helper.py index 561076fdd..147377e55 100644 --- a/fastlane_bot/tests/deterministic/dtest_tx_helper.py +++ b/fastlane_bot/tests/deterministic/dtest_tx_helper.py @@ -121,7 +121,7 @@ def tx_scanner(self, args: argparse.Namespace) -> List: @staticmethod def clean_tx_data(tx_data: Dict) -> Dict: """ - This method takes a transaction data dictionary and removes the cid0 key from the trades. + This method takes a transaction data dictionary and removes the cid0 key from the trades. Note that the same dict object is modified in place. Args: tx_data (dict): The transaction data. From 26c5e96cf97a2f4feace06d0814591cda0f06310 Mon Sep 17 00:00:00 2001 From: Mike Casale <46603283+mikewcasale@users.noreply.github.com> Date: Thu, 28 Mar 2024 06:28:43 -0700 Subject: [PATCH 32/41] this has been addressed - https://github.com/bancorprotocol/fastlane-bot/pull/399#discussion_r1535479189 --- fastlane_bot/tests/deterministic/dtest_main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastlane_bot/tests/deterministic/dtest_main.py b/fastlane_bot/tests/deterministic/dtest_main.py index b83d590b9..ecd42580c 100644 --- a/fastlane_bot/tests/deterministic/dtest_main.py +++ b/fastlane_bot/tests/deterministic/dtest_main.py @@ -24,7 +24,7 @@ def get_logger(args: argparse.Namespace) -> logging.Logger: """ Get the logger for the script. """ - logger = logging.getLogger(__name__) + logger = logging.getLogger(args.task) logger.setLevel(args.loglevel) logger.handlers.clear() # Clear existing handlers to avoid duplicate logging From 66b327a76d8f6f3ecfe46b5eb16e181ca20035a4 Mon Sep 17 00:00:00 2001 From: Mike Casale <46603283+mikewcasale@users.noreply.github.com> Date: Thu, 28 Mar 2024 06:34:13 -0700 Subject: [PATCH 33/41] addressing comment https://github.com/bancorprotocol/fastlane-bot/pull/399#discussion_r1535496835 --- fastlane_bot/tests/deterministic/dtest_strategy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastlane_bot/tests/deterministic/dtest_strategy.py b/fastlane_bot/tests/deterministic/dtest_strategy.py index 6e43d297f..b5f7fe9bb 100644 --- a/fastlane_bot/tests/deterministic/dtest_strategy.py +++ b/fastlane_bot/tests/deterministic/dtest_strategy.py @@ -69,7 +69,7 @@ def get_token_approval( Returns: str: The transaction hash. """ - token = self.token0 if token_id == 0 else self.token1 + token = [self.token0, self.token1][token_id] if token.address in [ constants.BNT_ADDRESS, constants.USDC_ADDRESS, From fc1fe825cb927d631a9dcdfc1e9c95f53a458fbc Mon Sep 17 00:00:00 2001 From: Mike Casale <46603283+mikewcasale@users.noreply.github.com> Date: Thu, 28 Mar 2024 09:39:22 -0700 Subject: [PATCH 34/41] addressing comments and moving constant --- fastlane_bot/tests/deterministic/dtest_constants.py | 4 ++++ fastlane_bot/tests/deterministic/dtest_manager.py | 12 ++++++------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/fastlane_bot/tests/deterministic/dtest_constants.py b/fastlane_bot/tests/deterministic/dtest_constants.py index 78dbb77d3..b8722122b 100644 --- a/fastlane_bot/tests/deterministic/dtest_constants.py +++ b/fastlane_bot/tests/deterministic/dtest_constants.py @@ -5,6 +5,7 @@ All rights reserved. Licensed under MIT License. """ +import os KNOWN_UNABLE_TO_DELETE = { 68737038118029569619601670701217178714718: ("pDFS", "ETH"), @@ -24,6 +25,9 @@ TENDERLY_RPC_KEY = "fb866397-29bd-4886-8406-a2cc7b7c5b1f" # https://virtual.mainnet.rpc.tenderly.co/9ea4ceb3-d0f5-4faf-959e-f51cf1f6b52b, from_block: 19325893, fb866397-29bd-4886-8406-a2cc7b7c5b1f REAL_DATA_DIR = "fastlane_bot/data/blockchain_data" TEST_DATA_DIR = "fastlane_bot/tests/_data" +MULTICHAIN_ADDRESSES_PATH = os.path.normpath( + "fastlane_bot/data/multichain_addresses.csv" +) binance14 = "0x28C6c06298d514Db089934071355E5743bf21d60" TOKENS_MODIFICATIONS = { "0x0": { diff --git a/fastlane_bot/tests/deterministic/dtest_manager.py b/fastlane_bot/tests/deterministic/dtest_manager.py index 4cb1d11b6..c61d675d6 100644 --- a/fastlane_bot/tests/deterministic/dtest_manager.py +++ b/fastlane_bot/tests/deterministic/dtest_manager.py @@ -38,13 +38,9 @@ def __init__(self, args: argparse.Namespace): self.w3 = Web3(Web3.HTTPProvider(args.rpc_url, {"timeout": 60})) assert self.w3.is_connected(), "Web3 connection failed" - multichain_addresses_path = os.path.normpath( - "fastlane_bot/data/multichain_addresses.csv" - ) - # Get the Carbon Controller Address for the network carbon_controller_address = self.get_carbon_controller_address( - multichain_addresses_path=multichain_addresses_path, network=args.network + multichain_addresses_path=constants.MULTICHAIN_ADDRESSES_PATH, network=args.network ) # Initialize the Carbon Controller contract @@ -57,13 +53,15 @@ def __init__(self, args: argparse.Namespace): @property def logs_path(self) -> str: - """TODO TESTS: This should be read from dtest_constants""" + """TODO TESTS: This should be read from TestCommandLineArgs""" return os.path.normpath("./logs_dtest/logs/*") def get_carbon_controller(self, address: Address or str) -> Contract: """ Gets the Carbon Controller contract on the given network. + TODO: This needs to be updated when tests are added for multiple carbon deployments + Args: address (Address or str): The address. @@ -79,6 +77,8 @@ def get_carbon_controller_address( """ Gets the Carbon Controller contract address on the given network. + TODO: This needs to be updated when tests are added for multiple carbon deployments + Args: multichain_addresses_path (str): The path to the multichain addresses file. network (str): The network. From dcccf98f37b972011a51a14eb112aae93257e9c1 Mon Sep 17 00:00:00 2001 From: Mike Casale <46603283+mikewcasale@users.noreply.github.com> Date: Thu, 28 Mar 2024 09:42:08 -0700 Subject: [PATCH 35/41] addressing comment https://github.com/bancorprotocol/fastlane-bot/pull/399#discussion_r1535351018 --- fastlane_bot/tests/deterministic/dtest_manager.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/fastlane_bot/tests/deterministic/dtest_manager.py b/fastlane_bot/tests/deterministic/dtest_manager.py index c61d675d6..d4fcbf675 100644 --- a/fastlane_bot/tests/deterministic/dtest_manager.py +++ b/fastlane_bot/tests/deterministic/dtest_manager.py @@ -127,10 +127,13 @@ def create_new_testnet() -> Tuple[str, int]: "syncState": False, } - response = requests.post(url, headers=headers, data=json.dumps(data)) + try: + response = requests.post(url, headers=headers, data=json.dumps(data)) - uri = f"{response.json()['container']['connectivityConfig']['endpoints'][0]['uri']}" - from_block = int(response.json()["container"]["networkConfig"]["blockNumber"]) + uri = f"{response.json()['container']['connectivityConfig']['endpoints'][0]['uri']}" + from_block = int(response.json()["container"]["networkConfig"]["blockNumber"]) + except Exception as e: + raise Exception(f"[create_new_testnet] Error creating new testnet: {e}") from e return uri, from_block From d5c63005c59392192c669763907903c740febfc2 Mon Sep 17 00:00:00 2001 From: Mike Casale <46603283+mikewcasale@users.noreply.github.com> Date: Thu, 28 Mar 2024 09:44:39 -0700 Subject: [PATCH 36/41] addressing comment https://github.com/bancorprotocol/fastlane-bot/pull/399#discussion_r1535365865 --- fastlane_bot/tests/deterministic/__init__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/fastlane_bot/tests/deterministic/__init__.py b/fastlane_bot/tests/deterministic/__init__.py index e69de29bb..a67b42ecd 100644 --- a/fastlane_bot/tests/deterministic/__init__.py +++ b/fastlane_bot/tests/deterministic/__init__.py @@ -0,0 +1,7 @@ +""" +This package contains the deterministic modules. + +(c) Copyright Bprotocol foundation 2024. +All rights reserved. +Licensed under MIT License. +""" \ No newline at end of file From ed7a3ea8c239ab643519e93c6ced93936144dbaf Mon Sep 17 00:00:00 2001 From: Nicholas Welch Date: Thu, 23 May 2024 14:47:32 +1000 Subject: [PATCH 37/41] data files for det testing --- fastlane_bot/tests/_data/test_results.json | 291 ++++++++++++++++++ fastlane_bot/tests/_data/test_strategies.json | 56 ++++ 2 files changed, 347 insertions(+) create mode 100644 fastlane_bot/tests/_data/test_results.json create mode 100644 fastlane_bot/tests/_data/test_strategies.json diff --git a/fastlane_bot/tests/_data/test_results.json b/fastlane_bot/tests/_data/test_results.json new file mode 100644 index 000000000..39cc9700d --- /dev/null +++ b/fastlane_bot/tests/_data/test_results.json @@ -0,0 +1,291 @@ +{ + "test_data": { + "1": { + "type": "multi", + "profit_gas_token": "0.1130", + "profit_usd": "255.8626", + "flashloan_struct": [ + { + "platformId": 7, + "sourceTokens": [ + "0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2" + ], + "sourceAmounts": [ + 1146601000000000000 + ] + } + ], + "route_struct": [ + { + "platformId": 3, + "sourceToken": "0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2", + "targetToken": "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48", + "sourceAmount": 1146601000000000000, + "minTargetAmount": 2587649724, + "customAddress": "0x7a250d5630B4cF539739dF2C5dAcb4c659F2488D", + "customInt": 0, + "customData": "0x" + }, + { + "platformId": 6, + "sourceToken": "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48", + "targetToken": "0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE", + "sourceAmount": 0, + "minTargetAmount": 1259617647370498383, + "customAddress": "0xC537e898CD774e2dCBa3B14Ea6f34C93d5eA45e1", + "customInt": 0 + }, + { + "platformId": 10, + "sourceToken": "0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE", + "targetToken": "0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2", + "sourceAmount": 0, + "minTargetAmount": 0, + "customAddress": "0x0000000000000000000000000000000000000000", + "customInt": 0, + "customData": "0x" + } + ], + "trades": [ + { + "trade_index": 0, + "exchange": "uniswap_v2", + "tkn_in": { + "WETH": "0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2" + }, + "amount_in": "1.1466", + "tkn_out": { + "USDC": "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48" + }, + "amt_out": "2587.6497", + "cid0": "fe98c78ce1" + }, + { + "trade_index": 1, + "exchange": "carbon_v1", + "tkn_in": { + "USDC": "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48" + }, + "amount_in": "2587.6497", + "tkn_out": { + "ETH": "0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE" + }, + "amt_out": "1.2596" + } + ] + } + , + "2": { + "type": "multi", + "profit_gas_token": "0.1817", + "profit_usd": "411.3004", + "flashloan_struct": [ + { + "platformId": 2, + "sourceTokens": [ + "0x514910771AF9Ca656af840dff83E8264EcF986CA" + ], + "sourceAmounts": [ + 51330779000000000000 + ] + } + ], + "route_struct": [ + { + "platformId": 4, + "sourceToken": "0x514910771AF9Ca656af840dff83E8264EcF986CA", + "targetToken": "0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2", + "sourceAmount": 51330779000000000000, + "minTargetAmount": 330002553680218980, + "customAddress": "0xE592427A0AEce92De3Edee1F18E0157C05861564", + "customInt": 3000, + "customData": "0x0000000000000000000000000000000000000000000000000000000000000000" + }, + { + "platformId": 10, + "sourceToken": "0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2", + "targetToken": "0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE", + "sourceAmount": 0, + "minTargetAmount": 330002553680218980, + "customAddress": "0x0000000000000000000000000000000000000000", + "customInt": 0, + "customData": "0x" + }, + { + "platformId": 6, + "sourceToken": "0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE", + "targetToken": "0x514910771AF9Ca656af840dff83E8264EcF986CA", + "sourceAmount": 0, + "minTargetAmount": 79504976899565541953, + "customAddress": "0xC537e898CD774e2dCBa3B14Ea6f34C93d5eA45e1", + "customInt": 0 + } + ], + "trades": [ + { + "trade_index": 0, + "exchange": "uniswap_v3", + "tkn_in": { + "LINK": "0x514910771AF9Ca656af840dff83E8264EcF986CA" + }, + "amount_in": "51.3308", + "tkn_out": { + "WETH": "0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2" + }, + "amt_out": "0.3300", + "cid0": "b800399517" + }, + { + "trade_index": 1, + "exchange": "carbon_v1", + "tkn_in": { + "ETH": "0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE" + }, + "amount_in": "0.3300", + "tkn_out": { + "LINK": "0x514910771AF9Ca656af840dff83E8264EcF986CA" + }, + "amt_out": "79.5050" + } + ] + }, + "3": { + "type": "multi", + "profit_gas_token": "0.1293", + "profit_usd": "292.8006", + "flashloan_struct": [ + { + "platformId": 7, + "sourceTokens": [ + "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48" + ], + "sourceAmounts": [ + 2697300000 + ] + } + ], + "route_struct": [ + { + "platformId": 6, + "sourceToken": "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48", + "targetToken": "0x6B175474E89094C44Da98b954EedeAC495271d0F", + "sourceAmount": 2697300000, + "minTargetAmount": 2990706899399958137440, + "customAddress": "0xC537e898CD774e2dCBa3B14Ea6f34C93d5eA45e1", + "customInt": 0 + }, + { + "platformId": 4, + "sourceToken": "0x6B175474E89094C44Da98b954EedeAC495271d0F", + "targetToken": "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48", + "sourceAmount": 0, + "minTargetAmount": 2990100648, + "customAddress": "0x1b81D678ffb9C0263b24A97847620C99d213eB14", + "customInt": 100, + "customData": "0x0000000000000000000000000000000000000000000000000000000000000000" + } + ], + "trades": [ + { + "trade_index": 0, + "exchange": "carbon_v1", + "tkn_in": { + "USDC": "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48" + }, + "amount_in": "2697.3000", + "tkn_out": { + "DAI": "0x6B175474E89094C44Da98b954EedeAC495271d0F" + }, + "amt_out": "2990.7069" + }, + { + "trade_index": 1, + "exchange": "pancakeswap_v3", + "tkn_in": { + "DAI": "0x6B175474E89094C44Da98b954EedeAC495271d0F" + }, + "amount_in": "2990.7069", + "tkn_out": { + "USDC": "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48" + }, + "amt_out": "2990.1006", + "cid0": "d198cf907e" + } + ] + }, + "4": { + "type": "multi", + "profit_gas_token": "0.0198", + "profit_usd": "44.7633", + "flashloan_struct": [ + { + "platformId": 7, + "sourceTokens": [ + "0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2" + ], + "sourceAmounts": [ + 1332872317026955600 + ] + } + ], + "route_struct": [ + { + "platformId": 10, + "sourceToken": "0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2", + "targetToken": "0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE", + "sourceAmount": 1332872317026955600, + "minTargetAmount": 1332872317026955600, + "customAddress": "0x0000000000000000000000000000000000000000", + "customInt": 0, + "customData": "0x" + }, + { + "platformId": 6, + "sourceToken": "0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE", + "targetToken": "0x2260FAC5E5542a773Aa44fBCfeDf7C193bc2C599", + "sourceAmount": 0, + "minTargetAmount": 7374732, + "customAddress": "0xC537e898CD774e2dCBa3B14Ea6f34C93d5eA45e1", + "customInt": 0 + }, + { + "platformId": 3, + "sourceToken": "0x2260FAC5E5542a773Aa44fBCfeDf7C193bc2C599", + "targetToken": "0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2", + "sourceAmount": 0, + "minTargetAmount": 1352646741423507228, + "customAddress": "0xEfF92A263d31888d860bD50809A8D171709b7b1c", + "customInt": 0, + "customData": "0x" + } + ], + "trades": [ + { + "trade_index": 0, + "exchange": "carbon_v1", + "tkn_in": { + "ETH": "0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE" + }, + "amount_in": "1.3329", + "tkn_out": { + "WBTC": "0x2260FAC5E5542a773Aa44fBCfeDf7C193bc2C599" + }, + "amt_out": "0.0737" + }, + { + "trade_index": 1, + "exchange": "pancakeswap_v2", + "tkn_in": { + "WBTC": "0x2260FAC5E5542a773Aa44fBCfeDf7C193bc2C599" + }, + "amount_in": "0.0737", + "tkn_out": { + "WETH": "0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2" + }, + "amt_out": "1.3526", + "cid0": "66c0a836b8" + } + ] + } + } +} \ No newline at end of file diff --git a/fastlane_bot/tests/_data/test_strategies.json b/fastlane_bot/tests/_data/test_strategies.json new file mode 100644 index 000000000..b60e11464 --- /dev/null +++ b/fastlane_bot/tests/_data/test_strategies.json @@ -0,0 +1,56 @@ +{ + "test_strategies": { + "1": { + "token0": "0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE", + "token1": "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48", + "y0": 1267460000029073000, + "z0": 1267460000029073000, + "A0": 0, + "B0": 4411844567835374, + "y1": 7634017977, + "z1": 7649999999, + "A1": 0, + "B1": 11433567603, + "wallet": "0x28C6c06298d514Db089934071355E5743bf21d60" + }, + "2": { + "token0": "0x514910771AF9Ca656af840dff83E8264EcF986CA", + "token1": "0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE", + "y0": 80000000000000000000, + "z0": 80000000000000000000, + "A0": 0, + "B0": 1399247190132272, + "y1": 0, + "z1": 0, + "A1": 0, + "B1": 17102934719008, + "wallet": "0x28C6c06298d514Db089934071355E5743bf21d60" + }, + "3": { + "token0": "0x6B175474E89094C44Da98b954EedeAC495271d0F", + "token1": "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48", + "y0": 3000000000000000000000, + "z0": 3000000000000000000000, + "A0": 0, + "B0": 6052452418541427, + "y1": 0, + "z1": 0, + "A1": 0, + "B1": 0, + "wallet": "0x28C6c06298d514Db089934071355E5743bf21d60" + }, + "4": { + "token0": "0x2260FAC5E5542a773Aa44fBCfeDf7C193bc2C599", + "token1": "0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE", + "y0": 100000000, + "z0": 100000000, + "A0": 17694999, + "B0": 645747883, + "y1": 0, + "z1": 0, + "A1": 0, + "B1": 0, + "wallet": "0x28C6c06298d514Db089934071355E5743bf21d60" + } + } +} \ No newline at end of file From 834d4291a4d34f102e61c3ee7113d2f300eaf5f4 Mon Sep 17 00:00:00 2001 From: Nicholas Welch Date: Thu, 23 May 2024 14:51:47 +1000 Subject: [PATCH 38/41] small changes for deterministic tests --- fastlane_bot/data/abi.py | 14 +++++++++++++ .../deterministic/dtest_cmd_line_args.py | 2 +- .../tests/deterministic/dtest_tx_helper.py | 20 ++++++++++--------- 3 files changed, 26 insertions(+), 10 deletions(-) diff --git a/fastlane_bot/data/abi.py b/fastlane_bot/data/abi.py index 3c98f7000..b114d5c32 100644 --- a/fastlane_bot/data/abi.py +++ b/fastlane_bot/data/abi.py @@ -155,6 +155,20 @@ "stateMutability": "view", "inputs": [], "outputs": [{"internalType": "uint32", "name": "", "type": "uint32"}] + }, + { + "type": "function", + "name": "deleteStrategy", + "stateMutability": "nonpayable", + "inputs": [{"internalType": "uint256","name": "strategyId","type": "uint256"}], + "outputs": [] + }, + { + "type":"function", + "name":"createStrategy", + "stateMutability":"payable", + "inputs":[{"internalType":"Token","name":"token0","type":"address"},{"internalType":"Token","name":"token1","type":"address"},{"components":[{"internalType":"uint128","name":"y","type":"uint128"},{"internalType":"uint128","name":"z","type":"uint128"},{"internalType":"uint64","name":"A","type":"uint64"},{"internalType":"uint64","name":"B","type":"uint64"}],"internalType":"structOrder[2]","name":"orders","type":"tuple[2]"}], + "outputs":[{"internalType":"uint256","name":"","type":"uint256"}] } ] diff --git a/fastlane_bot/tests/deterministic/dtest_cmd_line_args.py b/fastlane_bot/tests/deterministic/dtest_cmd_line_args.py index 9025330e8..b2d6bb317 100644 --- a/fastlane_bot/tests/deterministic/dtest_cmd_line_args.py +++ b/fastlane_bot/tests/deterministic/dtest_cmd_line_args.py @@ -46,11 +46,11 @@ class TestCommandLineArgs: pool_data_update_frequency: int = -1 use_specific_exchange_for_target_tokens: str = None prefix_path: str = "" - version_check_frequency: int = 1 self_fund: str = "False" read_only: str = "False" is_args_test: str = "False" rpc_url: str = None + pool_finder_period: int = -1 @staticmethod def args_to_command_line(args): diff --git a/fastlane_bot/tests/deterministic/dtest_tx_helper.py b/fastlane_bot/tests/deterministic/dtest_tx_helper.py index 147377e55..5e11f11b4 100644 --- a/fastlane_bot/tests/deterministic/dtest_tx_helper.py +++ b/fastlane_bot/tests/deterministic/dtest_tx_helper.py @@ -113,7 +113,7 @@ def tx_scanner(self, args: argparse.Namespace) -> List: args.logger.debug(f"len(pool_data): {len(pool_data)}") transactions = self.read_transaction_files(most_recent_log_folder) - successful_txs = [tx for tx in transactions if "'status': 1" in tx] + successful_txs = [tx for tx in transactions if '"tx_status": "succeeded",' in tx] args.logger.debug(f"Found {len(successful_txs)} successful transactions.") return successful_txs @@ -132,9 +132,15 @@ def clean_tx_data(tx_data: Dict) -> Dict: if not tx_data: return tx_data + # filter out the non-deterministic factors derieved from unique strategyids for trade in tx_data["trades"]: if trade["exchange"] in constants.CARBON_V1_FORKS and "cid0" in trade: del trade["cid0"] + for trade in tx_data["route_struct"]: + if "deadline" in trade: + del trade['deadline'] + if trade["platformId"] == 6 and "customData" in trade: + del trade["customData"] return tx_data @staticmethod @@ -151,14 +157,10 @@ def get_tx_data(strategy_id: int, txt_all_successful_txs: List) -> Dict: dict: The transaction data for the given strategy_id. """ for tx in txt_all_successful_txs: - if str(strategy_id) in tx: + if hex(strategy_id).replace("0x","") in tx: return json.loads( - tx.split( - """ - -""" - )[-1] - ) + tx + )['log_dict'] @staticmethod def load_json_file(file_name: str, args: argparse.Namespace) -> Dict: @@ -290,7 +292,7 @@ def wait_for_txs(self, args: argparse.Namespace) -> Dict: dict: The test strategy txhashs. """ test_strategy_txhashs = self.load_json_file("test_strategy_txhashs.json", args) - sleep_seconds = int(35 * len(test_strategy_txhashs.keys()) + 15) + sleep_seconds = int(20 * len(test_strategy_txhashs.keys()) + 15) args.logger.debug(f"sleep_seconds: {sleep_seconds}") time.sleep(sleep_seconds) return test_strategy_txhashs From 5c1c1375ba6b7780a0bad86c01b5ffdfcb881b49 Mon Sep 17 00:00:00 2001 From: Nicholas Welch Date: Thu, 23 May 2024 14:52:58 +1000 Subject: [PATCH 39/41] fixed merge mistake --- fastlane_bot/config/multicaller.py | 108 ++++++----------------------- 1 file changed, 22 insertions(+), 86 deletions(-) diff --git a/fastlane_bot/config/multicaller.py b/fastlane_bot/config/multicaller.py index 9a73b4678..76b031bf6 100644 --- a/fastlane_bot/config/multicaller.py +++ b/fastlane_bot/config/multicaller.py @@ -44,89 +44,25 @@ class MultiCaller: __DATE__ = "2022-09-26" __VERSION__ = "0.0.2" - - def __init__(self, contract: MultiProviderContractWrapper or web3.contract.Contract, - web3: Web3, - block_identifier: Any = 'latest', multicall_address = "0x5BA1e12693Dc8F9c48aAD8770482f4739bEeD696"): - self._contract_calls: List[Callable] = [] - self.contract = contract - self.block_identifier = block_identifier - self.web3 = web3 - self.MULTICALL_CONTRACT_ADDRESS = self.web3.to_checksum_address(multicall_address) - - def __enter__(self) -> 'MultiCaller': - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - pass - - def add_call(self, fn: Callable, *args, **kwargs) -> None: - self._contract_calls.append(partial(fn, *args, **kwargs)) - - def multicall(self) -> List[Any]: - calls_for_aggregate = [] - output_types_list = [] - _calls_for_aggregate = {} - _output_types_list = {} - for fn in self._contract_calls: - fn_name = str(fn).split('functools.partial(')[0] - output_types = get_output_types_from_abi(self.contract.abi, fn_name) - if fn_name in _calls_for_aggregate: - _calls_for_aggregate[fn_name].append({ - 'target': self.contract.address, - 'callData': fn()._encode_transaction_data() - }) - _output_types_list[fn_name].append(output_types) - else: - _calls_for_aggregate[fn_name] = [{ - 'target': self.contract.address, - 'callData': fn()._encode_transaction_data() - }] - _output_types_list[fn_name] = [output_types] - - for fn_list in _calls_for_aggregate.keys(): - calls_for_aggregate += (_calls_for_aggregate[fn_list]) - output_types_list += (_output_types_list[fn_list]) - - _encoded_data = [] - - function_keys = _calls_for_aggregate.keys() - for fn_list in function_keys: - _encoded_data.append(self.web3.eth.contract( - abi=MULTICALL_ABI, - address=self.MULTICALL_CONTRACT_ADDRESS - ).functions.aggregate(_calls_for_aggregate[fn_list]).call(block_identifier=self.block_identifier)) - - if len(_encoded_data) > 0 and not isinstance(_encoded_data[0], list): - raise TypeError(f"Expected encoded_data to be a list, got {type(_encoded_data[0])} instead.") - - encoded_data = self.web3.eth.contract( - abi=MULTICALL_ABI, - address=self.MULTICALL_CONTRACT_ADDRESS - ).functions.aggregate(calls_for_aggregate).call(block_identifier=self.block_identifier) - - if not isinstance(encoded_data, list): - raise TypeError(f"Expected encoded_data to be a list, got {type(encoded_data)} instead.") - - encoded_data = encoded_data[1] - decoded_data_list = [] - for output_types, encoded_output in zip(output_types_list, encoded_data): - decoded_data = decode(output_types, encoded_output) - decoded_data_list.append(decoded_data) - - return_data = [i[0] for i in decoded_data_list if len(i) == 1] - return_data += [i[1] for i in decoded_data_list if len(i) > 1] - - # Handling for Bancor POL - combine results into a Tuple - if "tokenPrice" in function_keys and "amountAvailableForTrading" in function_keys: - new_return = [] - returned_items = int(len(return_data)) - total_pools = int(returned_items / 2) - assert returned_items % 2 == 0, f"[multicaller.py multicall] non-even number of returned calls for Bancor POL {returned_items}" - total_pools = int(total_pools) - - for idx in range(total_pools): - new_return.append((return_data[idx][0], return_data[idx][1], return_data[idx + total_pools])) - return_data = new_return - - return return_data + def __init__(self, web3: Any, multicall_contract_address: str): + self.multicall_contract = web3.eth.contract(abi=MULTICALL_ABI, address=multicall_contract_address) + self.contract_calls: List[ContractFunction] = [] + self.output_types_list: List[List[str]] = [] + + def add_call(self, call: ContractFunction): + self.contract_calls.append({'target': call.address, 'callData': call._encode_transaction_data()}) + self.output_types_list.append([collapse_if_tuple(item) for item in call.abi['outputs']]) + + def run_calls(self, block_identifier: Any = 'latest') -> List[Any]: + encoded_data = self.multicall_contract.functions.tryAggregate( + False, + self.contract_calls + ).call(block_identifier=block_identifier) + + result_list = [ + decode(output_types, encoded_output[1]) if encoded_output[0] else (None,) + for output_types, encoded_output in zip(self.output_types_list, encoded_data) + ] + + # Convert every single-value tuple into a single value + return [result if len(result) > 1 else result[0] for result in result_list] From c97312f3099e78c8da4e72bfd0f1c38d616691d5 Mon Sep 17 00:00:00 2001 From: Nicholas Welch Date: Thu, 23 May 2024 14:54:16 +1000 Subject: [PATCH 40/41] update bot logging w structs --- fastlane_bot/bot.py | 161 ++++++++++++++---------------- fastlane_bot/helpers/txhelpers.py | 9 +- 2 files changed, 82 insertions(+), 88 deletions(-) diff --git a/fastlane_bot/bot.py b/fastlane_bot/bot.py index f40e33cca..0ec5e3f27 100644 --- a/fastlane_bot/bot.py +++ b/fastlane_bot/bot.py @@ -358,7 +358,7 @@ def _run( ) return - tx_hash, tx_receipt = self._handle_trade_instructions(CCm, arb_mode, r, replay_from_block) + tx_hash, tx_receipt, log_dict = self._handle_trade_instructions(CCm, arb_mode, r, replay_from_block) if tx_hash: tx_status = ["failed", "succeeded"][tx_receipt["status"]] if tx_receipt else "pending" @@ -368,7 +368,13 @@ def _run( if logging_path: filename = f"tx_{datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}.txt" with open(os.path.join(logging_path, filename), "w") as f: - f.write(f"{tx_hash} {tx_status}: {tx_details}") + log_entry = { + "tx_hash": tx_hash, + "tx_status": tx_status, + "tx_details": tx_details, + "log_dict": log_dict + } + json.dump(log_entry, f, indent=4) def validate_optimizer_trades(self, arb_opp, arb_finder): """ @@ -817,7 +823,7 @@ def _handle_trade_instructions( self.ConfigObj.logger.info( f"[bot._handle_trade_instructions] Opportunity with profit: {num_format(best_profit_gastkn)} does not meet minimum profit: {self.ConfigObj.DEFAULT_MIN_PROFIT_GAS_TOKEN}, discarding." ) - return None, None + return None, None, None # Log the flashloan amount self.ConfigObj.logger.debug( @@ -855,89 +861,17 @@ def _handle_trade_instructions( maximize_last_trade_per_tkn(route_struct=route_struct_processed) - # Get the cids - cids = [(ti.cid,ti.strategy_id) for ti in ordered_trade_instructions_objects] - - # Check if the network is tenderly and submit the transaction accordingly - if self.ConfigObj.NETWORK == self.ConfigObj.NETWORK_TENDERLY: - return ( - self._validate_and_submit_transaction_tenderly( - ConfigObj=self.ConfigObj, - flashloan_struct=flashloan_struct, - route_struct=route_struct_maximized, - src_amount=flashloan_amount_wei, - src_address=flashloan_token_address, - ), - cids, - route_struct_maximized, - log_dict, - ) - - # Log the route_struct - self.handle_logging_for_trade_instructions( - 4, # The log id - flashloan_amount=flashloan_amount_wei, - flashloan_token_symbol=fl_token_symbol, - flashloan_token_address=flashloan_token_address, - route_struct=route_struct_maximized, - best_trade_instructions_dic=best_trade_instructions_dic, - ) - - # Get the tx helpers class - tx_helpers = TxHelpers(ConfigObj=self.ConfigObj) - - # Return the validate and submit transaction - return ( - tx_helpers.validate_and_submit_transaction( - route_struct=route_struct_maximized, - src_amt=flashloan_amount_wei, - src_address=flashloan_token_address, - expected_profit_gastkn=best_profit_gastkn, - expected_profit_usd=best_profit_usd, - safety_override=False, - verbose=True, - log_object=log_dict, - flashloan_struct=flashloan_struct, - ), - cids, - route_struct, - log_dict, + # Use helper function to update the log dict + log_dict = self.update_log_dict( + arb_mode, + best_profit_gastkn, + best_profit_usd, + flashloan_struct, + route_struct_processed, + split_calculated_trade_instructions, ) - def handle_logging_for_trade_instructions(self, log_id: int, **kwargs): - """ - Handles logging for trade instructions based on log_id. - - Parameters - ---------- - log_id : int - The ID for log type. - **kwargs : dict - Additional parameters required for logging. - - Returns - ------- - None - """ - log_actions = { - 1: self.log_best_profit, - 2: self.log_calculated_arb, - 3: self.log_flashloan_amount, - 4: self.log_flashloan_details, - } - log_action = log_actions.get(log_id) - if log_action: - log_action(**kwargs) - - def log_best_profit(self, best_profit: Optional[float] = None): - """ - Logs the best profit. - - Parameters - ---------- - best_profit : Optional[float], optional - The best profit, by default None - """ + # Log the flashloan details self.ConfigObj.logger.debug( f"[bot._handle_trade_instructions] Flashloan of {fl_token_symbol}, amount: {flashloan_amount_wei}" ) @@ -959,6 +893,7 @@ def log_best_profit(self, best_profit: Optional[float] = None): expected_profit_gastkn=best_profit_gastkn, expected_profit_usd=best_profit_usd, flashloan_struct=flashloan_struct, + log_dict=log_dict ) def get_tokens_in_exchange( @@ -1024,3 +959,61 @@ def run( ) except self.NoArbAvailable as e: self.ConfigObj.logger.info(e) + + @staticmethod + def update_log_dict( + arb_mode: str, + best_profit_gastkn: Decimal, + best_profit_usd: Decimal, + flashloan_struct: List[Any], + route_struct: List[Any], + calculated_trade_instructions: List[Any], + ) -> Dict[str, Any]: + """ + Update the log dictionary. + + Parameters + ---------- + arb_mode: str + The arbitrage mode. + best_profit: Decimal + The best profit. + best_profit_usd: Decimal + The profit in USD. + flashloan_struct: List[Any] + The flashloan struct. + route_struct: List[Any] + The route struct. + calculated_trade_instructions: List[Any] + The calculated trade instructions. + + Returns + ------- + dict + The updated log dictionary. + """ + log_dict = { + "type": arb_mode, + "profit_gas_token": num_format(best_profit_gastkn), + "profit_usd": num_format(best_profit_usd), + "flashloan_struct": flashloan_struct, + "route_struct": route_struct, + "trades": [], + } + + for idx, trade in enumerate(calculated_trade_instructions): + tknin = {trade.tknin_symbol: trade.tknin} if trade.tknin_symbol != trade.tknin else trade.tknin + tknout = {trade.tknout_symbol: trade.tknout} if trade.tknout_symbol != trade.tknout else trade.tknout + log_dict["trades"].append( + { + "trade_index": idx, + "exchange": trade.exchange_name, + "tkn_in": tknin, + "amount_in": num_format(trade.amtin), + "tkn_out": tknout, + "amt_out": num_format(trade.amtout), + "cid0": trade.cid[-10:], + } + ) + + return log_dict \ No newline at end of file diff --git a/fastlane_bot/helpers/txhelpers.py b/fastlane_bot/helpers/txhelpers.py index 0fdf166fc..7fc675af3 100644 --- a/fastlane_bot/helpers/txhelpers.py +++ b/fastlane_bot/helpers/txhelpers.py @@ -62,7 +62,8 @@ def validate_and_submit_transaction( src_address: str, expected_profit_gastkn: Decimal, expected_profit_usd: Decimal, - flashloan_struct: List[Dict] + flashloan_struct: List[Dict], + log_dict: Dict ) -> Tuple[Optional[str], Optional[dict]]: """ This method validates and submits a transaction to the arb contract. @@ -104,7 +105,7 @@ def validate_and_submit_transaction( self._update_transaction(tx) except Exception as e: self.cfg.logger.info(f"Transaction {dumps(tx, indent=4)}\nFailed with {e}") - return None, None + return None, None, None tx["gas"] += self.cfg.DEFAULT_GAS_SAFETY_OFFSET @@ -132,9 +133,9 @@ def validate_and_submit_transaction( self.cfg.logger.info(f"Waiting for transaction {tx_hash} receipt") tx_receipt = self._wait_for_transaction_receipt(tx_hash) self.cfg.logger.info(f"Transaction receipt: {dumps(tx_receipt, indent=4)}") - return tx_hash, tx_receipt + return tx_hash, tx_receipt, log_dict - return None, None + return None, None, None def check_and_approve_tokens(self, tokens: List): """ From 101550f74b6727c60e05d3d2266b669439342959 Mon Sep 17 00:00:00 2001 From: Nicholas Welch Date: Thu, 23 May 2024 15:11:22 +1000 Subject: [PATCH 41/41] fix merge mistake --- requirements.txt | 90 +++++++++++++++++++++++++++++++++++++----------- 1 file changed, 69 insertions(+), 21 deletions(-) diff --git a/requirements.txt b/requirements.txt index 8c94bafdc..7cec2637d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,21 +1,69 @@ -psutil~=5.9.6 -packaging==21.3 -requests~=2.31.0 -python-dateutil~=2.8.2 -typing-extensions~=4.7.1 -python-dotenv~=0.16.0 -joblib~=1.2.0 -pandas~=1.5.2 -alchemy-sdk~=0.1.1 -pyarrow~=11.0.0 -networkx~=3.0 -cvxpy~=1.3.1 -matplotlib~=3.7.1 -dataclass_wizard~=0.22.2 -hexbytes~=0.3.1 -setuptools~=67.6.1 -protobuf~=4.24.4 -tqdm~=4.64.1 -web3~=6.11.2 -nest-asyncio~=1.5.8 -eth-abi~=5.0.0 +aiohttp==3.9.3 ; python_version >= "3.8" and python_version < "4.0" +aiosignal==1.3.1 ; python_version >= "3.8" and python_version < "4.0" +alchemy-sdk==0.1.1 ; python_version >= "3.8" and python_version < "4.0" +async-timeout==4.0.3 ; python_version >= "3.8" and python_version < "3.11" +attrs==23.2.0 ; python_version >= "3.8" and python_version < "4.0" +backoff==2.2.1 ; python_version >= "3.8" and python_version < "4.0" +bitarray==2.9.2 ; python_version >= "3.8" and python_version < "4" +certifi==2024.2.2 ; python_version >= "3.8" and python_version < "4.0" +charset-normalizer==3.3.2 ; python_version >= "3.8" and python_version < "4.0" +ckzg==1.0.0 ; python_version >= "3.8" and python_version < "4" +colorama==0.4.6 ; python_version >= "3.8" and python_version < "4.0" and platform_system == "Windows" +contourpy==1.1.1 ; python_version >= "3.8" and python_version < "4.0" +cycler==0.12.1 ; python_version >= "3.8" and python_version < "4.0" +cytoolz==0.12.3 ; python_version >= "3.8" and python_version < "4" and implementation_name == "cpython" +dataclass-wizard==0.22.3 ; python_version >= "3.8" and python_version < "4.0" +eth-abi==5.1.0 ; python_version >= "3.8" and python_version < "4" +eth-account==0.11.2 ; python_version >= "3.8" and python_version < "4" +eth-hash==0.7.0 ; python_version >= "3.8" and python_version < "4" +eth-hash[pycryptodome]==0.7.0 ; python_version >= "3.8" and python_version < "4" +eth-keyfile==0.8.0 ; python_version >= "3.8" and python_version < "4" +eth-keys==0.5.0 ; python_version >= "3.8" and python_version < "4" +eth-rlp==1.0.1 ; python_version >= "3.8" and python_version < "4" +eth-typing==4.1.0 ; python_version >= "3.8" and python_version < "4" +eth-utils==4.1.0 ; python_version >= "3.8" and python_version < "4" +fonttools==4.51.0 ; python_version >= "3.8" and python_version < "4.0" +frozenlist==1.4.1 ; python_version >= "3.8" and python_version < "4.0" +hexbytes==0.3.1 ; python_version >= "3.8" and python_version < "4" +idna==3.7 ; python_version >= "3.8" and python_version < "4.0" +importlib-resources==6.4.0 ; python_version >= "3.8" and python_version < "3.10" +joblib==1.4.0 ; python_version >= "3.8" and python_version < "4.0" +jsonschema-specifications==2023.12.1 ; python_version >= "3.8" and python_version < "4.0" +jsonschema==4.21.1 ; python_version >= "3.8" and python_version < "4.0" +kiwisolver==1.4.5 ; python_version >= "3.8" and python_version < "4.0" +lru-dict==1.2.0 ; python_version >= "3.8" and python_version < "4.0" +matplotlib==3.7.5 ; python_version >= "3.8" and python_version < "4.0" +multidict==6.0.5 ; python_version >= "3.8" and python_version < "4.0" +nest-asyncio==1.6.0 ; python_version >= "3.8" and python_version < "4.0" +networkx==3.1 ; python_version >= "3.8" and python_version < "4.0" +numpy==1.24.4 ; python_version >= "3.8" and python_version < "4.0" +packaging==21.3 ; python_version >= "3.8" and python_version < "4.0" +pandas==1.5.3 ; python_version >= "3.8" and python_version < "4.0" +parsimonious==0.10.0 ; python_version >= "3.8" and python_version < "4" +pillow==10.3.0 ; python_version >= "3.8" and python_version < "4.0" +pkgutil-resolve-name==1.3.10 ; python_version >= "3.8" and python_version < "3.9" +protobuf==4.25.3 ; python_version >= "3.8" and python_version < "4.0" +psutil==5.9.8 ; python_version >= "3.8" and python_version < "4.0" +pyarrow==11.0.0 ; python_version >= "3.8" and python_version < "4.0" +pycryptodome==3.20.0 ; python_version >= "3.8" and python_version < "4" +pyparsing==3.1.2 ; python_version >= "3.8" and python_version < "4.0" +python-dateutil==2.9.0.post0 ; python_version >= "3.8" and python_version < "4.0" +python-dotenv==0.16.0 ; python_version >= "3.8" and python_version < "4.0" +pytz==2024.1 ; python_version >= "3.8" and python_version < "4.0" +pyunormalize==15.1.0 ; python_version >= "3.8" and python_version < "4.0" +pywin32==306 ; python_version >= "3.8" and python_version < "4.0" and platform_system == "Windows" +referencing==0.34.0 ; python_version >= "3.8" and python_version < "4.0" +regex==2023.12.25 ; python_version >= "3.8" and python_version < "4" +requests==2.31.0 ; python_version >= "3.8" and python_version < "4.0" +rlp==4.0.0 ; python_version >= "3.8" and python_version < "4" +rpds-py==0.18.0 ; python_version >= "3.8" and python_version < "4.0" +setuptools==67.8.0 ; python_version >= "3.8" and python_version < "4.0" +six==1.16.0 ; python_version >= "3.8" and python_version < "4.0" +toolz==0.12.1 ; python_version >= "3.8" and python_version < "4" and (implementation_name == "pypy" or implementation_name == "cpython") +tqdm==4.66.2 ; python_version >= "3.8" and python_version < "4.0" +typing-extensions==4.11.0 ; python_version >= "3.8" and python_version < "4.0" +urllib3==2.2.1 ; python_version >= "3.8" and python_version < "4.0" +web3==6.16.0 ; python_version >= "3.8" and python_version < "4.0" +websockets==12.0 ; python_version >= "3.8" and python_version < "4.0" +yarl==1.9.4 ; python_version >= "3.8" and python_version < "4.0" +zipp==3.18.1 ; python_version >= "3.8" and python_version < "3.10"