From 5585ebe469a1670d78a02a3835ee98f2dd5990ca Mon Sep 17 00:00:00 2001 From: Mike Casale <46603283+mikewcasale@users.noreply.github.com> Date: Wed, 9 Aug 2023 11:05:10 -0700 Subject: [PATCH] Removes flaky tests - should be added back later. --- .../NBTest/NBTest_902_ValidatorSlow.ipynb | 1769 ----------------- resources/NBTest/NBTest_902_ValidatorSlow.py | 291 --- 2 files changed, 2060 deletions(-) delete mode 100644 resources/NBTest/NBTest_902_ValidatorSlow.ipynb delete mode 100644 resources/NBTest/NBTest_902_ValidatorSlow.py diff --git a/resources/NBTest/NBTest_902_ValidatorSlow.ipynb b/resources/NBTest/NBTest_902_ValidatorSlow.ipynb deleted file mode 100644 index 46d2b023d..000000000 --- a/resources/NBTest/NBTest_902_ValidatorSlow.ipynb +++ /dev/null @@ -1,1769 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 9, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "ConstantProductCurve v2.14 (23/May/2023)\n", - "CarbonBot v3-b2.2 (20/June/2023)\n", - "UniswapV2 v0.0.1 (2023-07-03)\n", - "UniswapV3 v0.0.1 (2023-07-03)\n", - "SushiswapV2 v0.0.1 (2023-07-03)\n", - "CarbonV1 v0.0.1 (2023-07-03)\n", - "BancorV3 v0.0.1 (2023-07-03)\n", - "Version = 3-b2.2 [requirements >= 3.0 is met]\n" - ] - } - ], - "source": [ - "# coding=utf-8\n", - "\"\"\"\n", - "This module contains the tests for the exchanges classes\n", - "\"\"\"\n", - "from fastlane_bot import Bot, Config\n", - "from fastlane_bot.bot import CarbonBot\n", - "from fastlane_bot.tools.cpc import ConstantProductCurve\n", - "from fastlane_bot.tools.cpc import ConstantProductCurve as CPC\n", - "from fastlane_bot.events.exchanges import UniswapV2, UniswapV3, SushiswapV2, CarbonV1, BancorV3\n", - "from fastlane_bot.events.interface import QueryInterface\n", - "from fastlane_bot.helpers.poolandtokens import PoolAndTokens\n", - "from fastlane_bot.helpers import TradeInstruction, TxReceiptHandler, TxRouteHandler, TxSubmitHandler, TxHelpers, TxHelper\n", - "from fastlane_bot.events.managers.manager import Manager\n", - "from fastlane_bot.events.interface import QueryInterface\n", - "from joblib import Parallel, delayed\n", - "import pytest\n", - "import math\n", - "import json\n", - "print(\"{0.__name__} v{0.__VERSION__} ({0.__DATE__})\".format(CPC))\n", - "print(\"{0.__name__} v{0.__VERSION__} ({0.__DATE__})\".format(Bot))\n", - "print(\"{0.__name__} v{0.__VERSION__} ({0.__DATE__})\".format(UniswapV2))\n", - "print(\"{0.__name__} v{0.__VERSION__} ({0.__DATE__})\".format(UniswapV3))\n", - "print(\"{0.__name__} v{0.__VERSION__} ({0.__DATE__})\".format(SushiswapV2))\n", - "print(\"{0.__name__} v{0.__VERSION__} ({0.__DATE__})\".format(CarbonV1))\n", - "print(\"{0.__name__} v{0.__VERSION__} ({0.__DATE__})\".format(BancorV3))\n", - "from fastlane_bot.testing import *\n", - "from fastlane_bot.modes import triangle_single_bancor3\n", - "#plt.style.use('seaborn-dark')\n", - "plt.rcParams['figure.figsize'] = [12,6]\n", - "from fastlane_bot import __VERSION__\n", - "require(\"3.0\", __VERSION__)" - ], - "metadata": { - "collapsed": false, - "ExecuteTime": { - "end_time": "2023-08-06T12:29:53.196201Z", - "start_time": "2023-08-06T12:29:53.124133Z" - } - }, - "id": "56224b770915d14" - }, - { - "cell_type": "markdown", - "id": "4c4e2283", - "metadata": {}, - "source": [ - "# Multi Mode [NB039]" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "a51e5ec2", - "metadata": { - "ExecuteTime": { - "end_time": "2023-08-06T12:29:54.497241Z", - "start_time": "2023-08-06T12:29:53.143Z" - } - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2023-08-06 05:29:54,070 [fastlane:INFO] - Time taken to add initial pools: 0.06647491455078125\n", - "2023-08-06 05:29:54,070 [fastlane:INFO] - Time taken to add initial pools: 0.06647491455078125\n", - "2023-08-06 05:29:54,073 [fastlane:INFO] - Initializing the bot...\n", - "2023-08-06 05:29:54,073 [fastlane:INFO] - Initializing the bot...\n", - "2023-08-06 05:29:54,191 [fastlane:INFO] - Removed 3242 unmapped uniswap_v2/sushi pools. 1897 uniswap_v2/sushi pools remaining\n", - "2023-08-06 05:29:54,191 [fastlane:INFO] - Removed 3242 unmapped uniswap_v2/sushi pools. 1897 uniswap_v2/sushi pools remaining\n", - "2023-08-06 05:29:54,192 [fastlane:INFO] - Unmapped uniswap_v2/sushi pools:\n", - "2023-08-06 05:29:54,192 [fastlane:INFO] - Unmapped uniswap_v2/sushi pools:\n", - "2023-08-06 05:29:54,379 [fastlane:INFO] - uniswap_v3: 0\n", - "2023-08-06 05:29:54,379 [fastlane:INFO] - uniswap_v3: 0\n", - "2023-08-06 05:29:54,380 [fastlane:INFO] - uniswap_v2: 3242\n", - "2023-08-06 05:29:54,380 [fastlane:INFO] - uniswap_v2: 3242\n", - "2023-08-06 05:29:54,380 [fastlane:INFO] - sushiswap_v2: 0\n", - "2023-08-06 05:29:54,380 [fastlane:INFO] - sushiswap_v2: 0\n", - "2023-08-06 05:29:54,382 [fastlane:INFO] - uniswap_v3: 636\n", - "2023-08-06 05:29:54,382 [fastlane:INFO] - uniswap_v3: 636\n", - "2023-08-06 05:29:54,382 [fastlane:INFO] - sushiswap_v2: 78\n", - "2023-08-06 05:29:54,382 [fastlane:INFO] - sushiswap_v2: 78\n", - "2023-08-06 05:29:54,383 [fastlane:INFO] - uniswap_v2: 0\n", - "2023-08-06 05:29:54,383 [fastlane:INFO] - uniswap_v2: 0\n", - "2023-08-06 05:29:54,383 [fastlane:INFO] - bancor_v2: 0\n", - "2023-08-06 05:29:54,383 [fastlane:INFO] - bancor_v2: 0\n", - "2023-08-06 05:29:54,383 [fastlane:INFO] - bancor_v3: 34\n", - "2023-08-06 05:29:54,383 [fastlane:INFO] - bancor_v3: 34\n", - "2023-08-06 05:29:54,384 [fastlane:INFO] - carbon_v1: 220\n", - "2023-08-06 05:29:54,384 [fastlane:INFO] - carbon_v1: 220\n", - "2023-08-06 05:29:54,415 [fastlane:INFO] - uniswap_v3_zero_liquidity_pools: 858\n", - "2023-08-06 05:29:54,415 [fastlane:INFO] - uniswap_v3_zero_liquidity_pools: 858\n", - "2023-08-06 05:29:54,415 [fastlane:INFO] - sushiswap_v2_zero_liquidity_pools: 34\n", - "2023-08-06 05:29:54,415 [fastlane:INFO] - sushiswap_v2_zero_liquidity_pools: 34\n", - "2023-08-06 05:29:54,416 [fastlane:INFO] - uniswap_v2_zero_liquidity_pools: 0\n", - "2023-08-06 05:29:54,416 [fastlane:INFO] - uniswap_v2_zero_liquidity_pools: 0\n", - "2023-08-06 05:29:54,416 [fastlane:INFO] - bancor_v2_zero_liquidity_pools: 0\n", - "2023-08-06 05:29:54,416 [fastlane:INFO] - bancor_v2_zero_liquidity_pools: 0\n", - "2023-08-06 05:29:54,417 [fastlane:INFO] - bancor_v3_zero_liquidity_pools: 37\n", - "2023-08-06 05:29:54,417 [fastlane:INFO] - bancor_v3_zero_liquidity_pools: 37\n", - "2023-08-06 05:29:54,417 [fastlane:INFO] - carbon_v1_zero_liquidity_pools: 0\n", - "2023-08-06 05:29:54,417 [fastlane:INFO] - carbon_v1_zero_liquidity_pools: 0\n", - "2023-08-06 05:29:54,418 [fastlane:INFO] - Removed 0 unsupported exchanges. 968 pools remaining\n", - "2023-08-06 05:29:54,418 [fastlane:INFO] - Removed 0 unsupported exchanges. 968 pools remaining\n", - "2023-08-06 05:29:54,419 [fastlane:INFO] - Pools remaining per exchange:\n", - "2023-08-06 05:29:54,419 [fastlane:INFO] - Pools remaining per exchange:\n", - "2023-08-06 05:29:54,419 [fastlane:INFO] - carbon_v1: 220\n", - "2023-08-06 05:29:54,419 [fastlane:INFO] - carbon_v1: 220\n", - "2023-08-06 05:29:54,419 [fastlane:INFO] - sushiswap_v2: 78\n", - "2023-08-06 05:29:54,419 [fastlane:INFO] - sushiswap_v2: 78\n", - "2023-08-06 05:29:54,420 [fastlane:INFO] - uniswap_v2: 0\n", - "2023-08-06 05:29:54,420 [fastlane:INFO] - uniswap_v2: 0\n", - "2023-08-06 05:29:54,420 [fastlane:INFO] - bancor_v3: 34\n", - "2023-08-06 05:29:54,420 [fastlane:INFO] - bancor_v3: 34\n", - "2023-08-06 05:29:54,420 [fastlane:INFO] - uniswap_v3: 636\n", - "2023-08-06 05:29:54,420 [fastlane:INFO] - uniswap_v3: 636\n" - ] - } - ], - "source": [ - "C = cfg = Config.new(config=Config.CONFIG_MAINNET)\n", - "C.DEFAULT_MIN_PROFIT_BNT = 0.02\n", - "C.DEFAULT_MIN_PROFIT = 0.02\n", - "cfg.DEFAULT_MIN_PROFIT_BNT = 0.02\n", - "cfg.DEFAULT_MIN_PROFIT = 0.02\n", - "assert (C.NETWORK == C.NETWORK_MAINNET)\n", - "assert (C.PROVIDER == C.PROVIDER_ALCHEMY)\n", - "setup_bot = CarbonBot(ConfigObj=C)\n", - "pools = None\n", - "with open('fastlane_bot/data/tests/latest_pool_data_testing.json') as f:\n", - " pools = json.load(f)\n", - "pools = [pool for pool in pools]\n", - "pools[0]\n", - "static_pools = pools\n", - "state = pools\n", - "exchanges = list({ex['exchange_name'] for ex in state})\n", - "db = QueryInterface(state=state, ConfigObj=C, exchanges=exchanges)\n", - "setup_bot.db = db\n", - "\n", - "static_pool_data_filename = \"static_pool_data\"\n", - "\n", - "static_pool_data = pd.read_csv(f\"fastlane_bot/data/{static_pool_data_filename}.csv\", low_memory=False)\n", - " \n", - "uniswap_v2_event_mappings = pd.read_csv(\"fastlane_bot/data/uniswap_v2_event_mappings.csv\", low_memory=False)\n", - " \n", - "tokens = pd.read_csv(\"fastlane_bot/data/tokens.csv\", low_memory=False)\n", - " \n", - "exchanges = \"carbon_v1,bancor_v3,uniswap_v3,uniswap_v2,sushiswap_v2\"\n", - "\n", - "exchanges = exchanges.split(\",\")\n", - "\n", - "\n", - "alchemy_max_block_fetch = 20\n", - "static_pool_data[\"cid\"] = [\n", - " cfg.w3.keccak(text=f\"{row['descr']}\").hex()\n", - " for index, row in static_pool_data.iterrows()\n", - " ]\n", - "# Filter out pools that are not in the supported exchanges\n", - "static_pool_data = [\n", - " row for index, row in static_pool_data.iterrows()\n", - " if row[\"exchange_name\"] in exchanges\n", - "]\n", - "\n", - "static_pool_data = pd.DataFrame(static_pool_data)\n", - "static_pool_data['exchange_name'].unique()\n", - "# Initialize data fetch manager\n", - "mgr = Manager(\n", - " web3=cfg.w3,\n", - " cfg=cfg,\n", - " pool_data=static_pool_data.to_dict(orient=\"records\"),\n", - " SUPPORTED_EXCHANGES=exchanges,\n", - " alchemy_max_block_fetch=alchemy_max_block_fetch,\n", - " uniswap_v2_event_mappings=uniswap_v2_event_mappings,\n", - " tokens=tokens.to_dict(orient=\"records\"),\n", - ")\n", - "\n", - "# Add initial pools for each row in the static_pool_data\n", - "start_time = time.time()\n", - "Parallel(n_jobs=-1, backend=\"threading\")(\n", - " delayed(mgr.add_pool_to_exchange)(row) for row in mgr.pool_data\n", - ")\n", - "cfg.logger.info(f\"Time taken to add initial pools: {time.time() - start_time}\")\n", - "\n", - "# check if any duplicate cid's exist in the pool data\n", - "mgr.deduplicate_pool_data()\n", - "cids = [pool[\"cid\"] for pool in mgr.pool_data]\n", - "assert len(cids) == len(set(cids)), \"duplicate cid's exist in the pool data\"\n", - "def init_bot(mgr: Manager) -> CarbonBot:\n", - " \"\"\"\n", - " Initializes the bot.\n", - "\n", - " Parameters\n", - " ----------\n", - " mgr : Manager\n", - " The manager object.\n", - "\n", - " Returns\n", - " -------\n", - " CarbonBot\n", - " The bot object.\n", - " \"\"\"\n", - " mgr.cfg.logger.info(\"Initializing the bot...\")\n", - " bot = CarbonBot(ConfigObj=mgr.cfg)\n", - " bot.db = db\n", - " bot.db.mgr = mgr\n", - " assert isinstance(\n", - " bot.db, QueryInterface\n", - " ), \"QueryInterface not initialized correctly\"\n", - " return bot\n", - "bot = init_bot(mgr)\n", - "# add data cleanup steps from main.py\n", - "bot.db.handle_token_key_cleanup()\n", - "bot.db.remove_unmapped_uniswap_v2_pools()\n", - "bot.db.remove_zero_liquidity_pools()\n", - "bot.db.remove_unsupported_exchanges()\n", - "tokens = bot.db.get_tokens()\n", - "ADDRDEC = {t.key: (t.address, int(t.decimals)) for t in tokens if not math.isnan(t.decimals)}\n", - "flashloan_tokens = bot.setup_flashloan_tokens(None)\n", - "CCm = bot.setup_CCm(None)\n", - "pools = db.get_pool_data_with_tokens()\n", - "\n", - "arb_mode = \"multi\"" - ] - }, - { - "cell_type": "markdown", - "id": "a70cf16f", - "metadata": {}, - "source": [ - "## Test_MIN_PROFIT" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "c8f41237", - "metadata": { - "ExecuteTime": { - "end_time": "2023-08-06T12:29:54.497413Z", - "start_time": "2023-08-06T12:29:54.494079Z" - } - }, - "outputs": [], - "source": [ - "assert(cfg.DEFAULT_MIN_PROFIT_BNT <= 0.02), f\"[TestMultiMode], DEFAULT_MIN_PROFIT_BNT must be <= 0.02 for this Notebook to run, currently set to {cfg.DEFAULT_MIN_PROFIT_BNT}\"\n", - "assert(C.DEFAULT_MIN_PROFIT_BNT <= 0.02), f\"[TestMultiMode], DEFAULT_MIN_PROFIT_BNT must be <= 0.02 for this Notebook to run, currently set to {cfg.DEFAULT_MIN_PROFIT_BNT}\"" - ] - }, - { - "cell_type": "markdown", - "id": "eddfdea7", - "metadata": {}, - "source": [ - "## Test_validator_in_out" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "id": "aaaf7423", - "metadata": { - "ExecuteTime": { - "end_time": "2023-08-06T12:29:54.500387Z", - "start_time": "2023-08-06T12:29:54.496715Z" - } - }, - "outputs": [], - "source": [ - "arb_finder = bot._get_arb_finder(\"multi\")\n", - "assert arb_finder.__name__ == \"FindArbitrageMultiPairwise\", f\"[TestMultiMode] Expected arb_finder class name name = FindArbitrageMultiPairwise, found {arb_finder.__name__}\"" - ] - }, - { - "cell_type": "markdown", - "id": "96ef3bf0", - "metadata": {}, - "source": [ - "## Test_validator_multi" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "id": "db7ec9ec", - "metadata": { - "ExecuteTime": { - "end_time": "2023-08-06T12:29:54.788652Z", - "start_time": "2023-08-06T12:29:54.508442Z" - } - }, - "outputs": [], - "source": [ - "arb_finder = bot._get_arb_finder(\"multi\")\n", - "finder = arb_finder(\n", - " flashloan_tokens=flashloan_tokens,\n", - " CCm=CCm,\n", - " mode=\"bothin\",\n", - " result=bot.AO_CANDIDATES,\n", - " ConfigObj=bot.ConfigObj,\n", - " )\n", - "r = finder.find_arbitrage()\n", - "\n", - "arb_opp = r[0]\n", - "\n", - "validated = bot.validate_optimizer_trades(arb_opp=arb_opp, arb_mode=\"multi\", arb_finder=finder)\n", - "\n", - "\n", - "\n", - "assert arb_opp == validated" - ] - }, - { - "cell_type": "markdown", - "id": "5b195f52", - "metadata": {}, - "source": [ - "## Test_validator_single" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "id": "78d0dbc1", - "metadata": { - "ExecuteTime": { - "end_time": "2023-08-06T12:29:58.990699Z", - "start_time": "2023-08-06T12:29:54.793412Z" - } - }, - "outputs": [ - { - "data": { - "text/plain": "0it [00:00, ?it/s]", - "application/vnd.jupyter.widget-view+json": { - "version_major": 2, - "version_minor": 0, - "model_id": "56850cdfceff41aab83fda57bbd21b77" - } - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/plain": " 0%| | 0/2 [00:00 CarbonBot: - """ - Initializes the bot. - - Parameters - ---------- - mgr : Manager - The manager object. - - Returns - ------- - CarbonBot - The bot object. - """ - mgr.cfg.logger.info("Initializing the bot...") - bot = CarbonBot(ConfigObj=mgr.cfg) - bot.db = db - bot.db.mgr = mgr - assert isinstance( - bot.db, QueryInterface - ), "QueryInterface not initialized correctly" - return bot - - -bot = init_bot(mgr) -# add data cleanup steps from main.py -bot.db.handle_token_key_cleanup() -bot.db.remove_unmapped_uniswap_v2_pools() -bot.db.remove_zero_liquidity_pools() -bot.db.remove_unsupported_exchanges() -tokens = bot.db.get_tokens() -ADDRDEC = { - t.key: (t.address, int(t.decimals)) for t in tokens if not math.isnan(t.decimals) -} -flashloan_tokens = bot.setup_flashloan_tokens(None) -CCm = bot.setup_CCm(None) -pools = db.get_pool_data_with_tokens() - -arb_mode = "multi" -# - - -# ## Test_MIN_PROFIT - -assert ( - cfg.DEFAULT_MIN_PROFIT_BNT <= 0.02 -), f"[TestMultiMode], DEFAULT_MIN_PROFIT_BNT must be <= 0.02 for this Notebook to run, currently set to {cfg.DEFAULT_MIN_PROFIT_BNT}" -assert ( - C.DEFAULT_MIN_PROFIT_BNT <= 0.02 -), f"[TestMultiMode], DEFAULT_MIN_PROFIT_BNT must be <= 0.02 for this Notebook to run, currently set to {cfg.DEFAULT_MIN_PROFIT_BNT}" - -# ## Test_validator_in_out - -arb_finder = bot._get_arb_finder("multi") -assert ( - arb_finder.__name__ == "FindArbitrageMultiPairwise" -), f"[TestMultiMode] Expected arb_finder class name name = FindArbitrageMultiPairwise, found {arb_finder.__name__}" - -# ## Test_validator_multi - -# + -arb_finder = bot._get_arb_finder("multi") -finder = arb_finder( - flashloan_tokens=flashloan_tokens, - CCm=CCm, - mode="bothin", - result=bot.AO_CANDIDATES, - ConfigObj=bot.ConfigObj, -) -r = finder.find_arbitrage() - -arb_opp = r[0] - -validated = bot.validate_optimizer_trades( - arb_opp=arb_opp, arb_mode="multi", arb_finder=finder -) - - -assert arb_opp == validated -# - - -# ## Test_validator_single - -# + -arb_mode = "single" -arb_finder = bot._get_arb_finder(arb_mode) -finder = arb_finder( - flashloan_tokens=flashloan_tokens, - CCm=CCm, - mode="bothin", - result=bot.AO_CANDIDATES, - ConfigObj=bot.ConfigObj, -) -r = finder.find_arbitrage() - -arb_opp = r[0] - -validated = bot.validate_optimizer_trades( - arb_opp=arb_opp, arb_mode=arb_mode, arb_finder=finder -) - - -assert arb_opp == validated -# - - -# ## Test_validator_bancor_v3 - -# + -arb_mode = "bancor_v3" - -arb_finder = bot._get_arb_finder(arb_mode) -finder = arb_finder( - flashloan_tokens=flashloan_tokens, - CCm=CCm, - mode="bothin", - result=bot.AO_CANDIDATES, - ConfigObj=bot.ConfigObj, -) -r = finder.find_arbitrage() - -arb_opp = r[0] - -validated = bot.validate_optimizer_trades( - arb_opp=arb_opp, arb_mode=arb_mode, arb_finder=finder -) - - -assert arb_opp == validated -# - - -# ## Test_validator_multi_triangle - -# + -arb_mode = "multi_triangle" -arb_finder = bot._get_arb_finder(arb_mode) -finder = arb_finder( - flashloan_tokens=flashloan_tokens, - CCm=CCm, - mode="bothin", - result=bot.AO_CANDIDATES, - ConfigObj=bot.ConfigObj, -) -r = finder.find_arbitrage() - -arb_opp = r[0] - -validated = bot.validate_optimizer_trades( - arb_opp=arb_opp, arb_mode=arb_mode, arb_finder=finder -) - - -assert arb_opp == validated