From 44343b8871fa294350024be2696c7476ddb23744 Mon Sep 17 00:00:00 2001 From: Joseph Gimba <86230531+Joewizy@users.noreply.github.com> Date: Thu, 28 Nov 2024 00:10:20 +0100 Subject: [PATCH 01/10] Created pipeline for dashboard_app --- .github/workflows/dashboard_app_pylint.yml | 44 ++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 .github/workflows/dashboard_app_pylint.yml diff --git a/.github/workflows/dashboard_app_pylint.yml b/.github/workflows/dashboard_app_pylint.yml new file mode 100644 index 00000000..fca27430 --- /dev/null +++ b/.github/workflows/dashboard_app_pylint.yml @@ -0,0 +1,44 @@ +name: Pylint Check - Dashboard App + +on: [push, pull_request] + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Set up Python 3.11 + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pylint + + - name: Debug Information + run: | + echo "Current directory:" + pwd + echo "\nDirectory structure:" + ls -R + echo "\nGit root directory:" + git rev-parse --show-toplevel + echo "\nSearching for Python files:" + find . -name "*.py" + echo "\nGit ls-files output:" + git ls-files + echo "\nSpecific path search:" + git ls-files 'dashboard_app/*.py' + git ls-files './dashboard_app/*.py' + git ls-files 'apps/dashboard_app/*.py' + git ls-files './apps/dashboard_app/*.py' + + - name: Run Pylint + run: | + files=$(git ls-files 'dashboard_app/*.py') + if [ -n "$files" ]; then + pylint $files --disable=all --enable=C0114,C0115,C0116,C0301 + fi From 61d4b672ff8b415b23123f6b70876041a248401f Mon Sep 17 00:00:00 2001 From: Joseph Gimba <86230531+Joewizy@users.noreply.github.com> Date: Thu, 28 Nov 2024 00:18:23 +0100 Subject: [PATCH 02/10] Created pipeline for dashboard_app --- .github/workflows/dashboard_app_pylint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dashboard_app_pylint.yml b/.github/workflows/dashboard_app_pylint.yml index fca27430..81aad05c 100644 --- a/.github/workflows/dashboard_app_pylint.yml +++ b/.github/workflows/dashboard_app_pylint.yml @@ -1,4 +1,4 @@ -name: Pylint Check - Dashboard App +name: Dashboard App on: [push, pull_request] From 7b5b0397917eaa7138bceceb719f4336409e1ea9 Mon Sep 17 00:00:00 2001 From: Joewizy Date: Thu, 28 Nov 2024 18:45:39 +0100 Subject: [PATCH 03/10] Added pylint pipeline for dashboard_app --- apps/dashboard_app/charts/__init__.py | 4 ++ apps/dashboard_app/charts/main.py | 18 +++++++- .../dashboard_app/charts/main_chart_figure.py | 41 ++++++++++++++--- apps/dashboard_app/charts/utils.py | 31 ++++++++----- apps/dashboard_app/dashboard.py | 4 ++ apps/dashboard_app/data_conector.py | 7 +++ apps/dashboard_app/helpers/ekubo.py | 29 +++++++----- apps/dashboard_app/helpers/load_data.py | 6 ++- apps/dashboard_app/helpers/loans_table.py | 15 ++++++- apps/dashboard_app/helpers/protocol_stats.py | 35 ++++++++++++--- apps/dashboard_app/helpers/settings.py | 14 +++++- apps/dashboard_app/helpers/tools.py | 45 +++++++++++++------ 12 files changed, 194 insertions(+), 55 deletions(-) diff --git a/apps/dashboard_app/charts/__init__.py b/apps/dashboard_app/charts/__init__.py index a5f7e917..0ef6c745 100644 --- a/apps/dashboard_app/charts/__init__.py +++ b/apps/dashboard_app/charts/__init__.py @@ -1 +1,5 @@ +""" +Imports the Dashboard class from the main module in the dashboard_app.charts package. +""" + from dashboard_app.charts.main import Dashboard diff --git a/apps/dashboard_app/charts/main.py b/apps/dashboard_app/charts/main.py index 48995625..7d47a62b 100644 --- a/apps/dashboard_app/charts/main.py +++ b/apps/dashboard_app/charts/main.py @@ -1,3 +1,7 @@ +""" +This module defines the Dashboard class for rendering a DeRisk dashboard using Streamlit. +""" + import streamlit as st from dashboard_app.charts.main_chart_figure import get_main_chart_figure @@ -16,13 +20,16 @@ class Dashboard: + """ + A class representing a dashboard for managing protocol names. + """ PROTOCOL_NAMES = [ "zkLend", # "Nostra Alpha", # "Nostra Mainnet", ] - def __init__(self, zklend_state): + def __init__(self, ): """ Initialize the dashboard. :param zklend_state: ZkLendState @@ -44,6 +51,9 @@ def __init__(self, zklend_state): self.collateral_token_price = 0 def load_sidebar(self): + """ + Creates an interactive sidebar for selecting multiple protocols, debt and collateral token. + """ col1, _ = st.columns([1, 3]) with col1: self.protocols = st.multiselect( @@ -73,6 +83,9 @@ def load_sidebar(self): ) def load_main_chart(self): + """ + Generates a chart that visualizes liquidable debt against available supply. + """ ( protocol_main_chart_data_mapping, protocol_loans_data_mapping, @@ -116,6 +129,9 @@ def load_main_chart(self): st.plotly_chart(figure_or_data=figure, use_container_width=True) def run(self): + """ + This function executes/runs the load_sidebar() and load_main_chart() function. + """ # Load sidebar with protocol settings self.load_sidebar() self.load_main_chart() diff --git a/apps/dashboard_app/charts/main_chart_figure.py b/apps/dashboard_app/charts/main_chart_figure.py index da6963f0..e3df8f79 100644 --- a/apps/dashboard_app/charts/main_chart_figure.py +++ b/apps/dashboard_app/charts/main_chart_figure.py @@ -1,9 +1,17 @@ +""" +This module includes functions to generate financial charts using token price and liquidity data. +""" + import math import pandas as pd import plotly.express import plotly.graph_objs +from shared.amms import SwapAmm +from shared.state import State +from shared.types import Prices + from dashboard_app.helpers.settings import TOKEN_SETTINGS from dashboard_app.helpers.tools import ( get_collateral_token_range, @@ -11,9 +19,6 @@ get_prices, get_underlying_address, ) -from shared.amms import SwapAmm -from shared.state import State -from shared.types import Prices AMMS = ("10kSwap", "MySwap", "SithSwap", "JediSwap") @@ -25,6 +30,11 @@ def get_main_chart_data( collateral_token_underlying_symbol: str, debt_token_underlying_symbol: str, ) -> pd.DataFrame: + """ + Generates financial chart data based on token prices, liquidity, and debt information. + Takes five parameters and + Returns: A DataFrame containing calculated token prices and liquidable debt data. + """ collateral_token_underlying_address = get_underlying_address( token_parameters=state.token_parameters.collateral, underlying_symbol=collateral_token_underlying_symbol, @@ -64,6 +74,10 @@ def get_main_chart_data( data[f"{amm}_debt_token_supply"] = 0 def compute_supply_at_price(collateral_token_price: float): + """ + Computes the token supply for each AMM at a given collateral token price and + Returns: the token supplied by AMM and total supply across all AMMs. + """ supplies = { amm: swap_amms.get_supply_at_price( collateral_token_underlying_symbol=collateral_token_underlying_symbol, @@ -79,7 +93,7 @@ def compute_supply_at_price(collateral_token_price: float): supplies_and_totals = data["collateral_token_price"].apply(compute_supply_at_price) for amm in AMMS: data[f"{amm}_debt_token_supply"] = supplies_and_totals.apply( - lambda x: x[0][amm] + lambda x, amm=amm: x[0][amm] ) data["debt_token_supply"] = supplies_and_totals.apply(lambda x: x[1]) @@ -92,6 +106,10 @@ def get_main_chart_figure( debt_token: str, collateral_token_price: float, ) -> plotly.graph_objs.Figure: + """ + Generates a plotly figure for the main chart the function takes in four paramters and + Returns: A Plotly figure object for the chart. + """ color_map_protocol = { "liquidable_debt_at_interval_zkLend": "#fff7bc", # light yellow "liquidable_debt_at_interval_Nostra Alpha": "#fec44f", # yellow @@ -103,7 +121,7 @@ def get_main_chart_figure( customdata = get_custom_data(data) # Add bars for each protocol and the total liquidable debt - for col in color_map_protocol.keys(): + for col in color_map_protocol.items(): try: figure.add_trace( plotly.graph_objs.Bar( @@ -143,11 +161,12 @@ def get_main_chart_figure( # Update layout for the stacked bar plot and the separate trace figure.update_layout( barmode="stack", - title=f"Liquidable debt and the corresponding supply of {debt_token} at various price intervals of {collateral_token}", + title=f"Liquidable debt and the corresponding supply of {debt_token}" + f" at various price intervals of {collateral_token}", xaxis_title=f"{collateral_token} Price (USD)", yaxis_title="Volume (USD)", legend_title="Legend", - yaxis2=dict(overlaying="y", side="left", matches="y"), + yaxis2={"overlaying": 'y', "side": 'left', "matches": 'y'}, ) # Add the vertical line and shaded region for the current price @@ -177,6 +196,10 @@ def get_bar_chart_figures( ) -> tuple[ plotly.graph_objs.Figure, plotly.graph_objs.Figure, plotly.graph_objs.Figure ]: + """ + Generates a bar chart figures for supply, collateral, and debt stats and then + Returns: A tuple of three objects (supply, collateral, and debt charts). + """ underlying_addresses_to_decimals = { x.address: int(math.log10(x.decimal_factor)) for x in TOKEN_SETTINGS.values() } @@ -281,6 +304,10 @@ def get_bar_chart_figures( def get_specific_loan_usd_amounts( loan: pd.DataFrame, ) -> tuple[pd.DataFrame, pd.DataFrame]: + """ + This function gets the loan amount in usd and then it + Returns: A tuple containing two DataFrames with specific loan USD amounts. + """ underlying_addresses_to_decimals = { x.address: int(math.log10(x.decimal_factor)) for x in TOKEN_SETTINGS.values() } diff --git a/apps/dashboard_app/charts/utils.py b/apps/dashboard_app/charts/utils.py index cd099ed4..08aebd72 100644 --- a/apps/dashboard_app/charts/utils.py +++ b/apps/dashboard_app/charts/utils.py @@ -1,3 +1,7 @@ +""" +This moudel process and transform liquidity, loan, and chart data for protocols. +""" + import logging import math from collections import defaultdict @@ -77,19 +81,21 @@ def parse_token_amounts(raw_token_amounts: str) -> dict[str, float]: def create_stablecoin_bundle(data: dict[str, pd.DataFrame]) -> dict[str, pd.DataFrame]: """ - Creates a stablecoin bundle by merging relevant DataFrames for collateral tokens and debt tokens. + Creates a stablecoin bundle by merging relevant DataFrames for collateral tokens and debt + tokens. - For each collateral token specified in `src.settings.COLLATERAL_TOKENS`, this function finds the - relevant stablecoin pairs from the provided `data` dictionary and merges the corresponding DataFrames - based on the 'collateral_token_price' column. It combines the debt and liquidity data for multiple - stablecoin pairs and adds the result back to the `data` dictionary under a new key. + For each collateral token specified in `src.settings.COLLATERAL_TOKENS`, this function finds + the relevant stablecoin pairs from the provided `data` dictionary and merges the corresponding + Dataframes based on the 'collateral_token_price' column. It combines the debt and liquidity + data for multiple stablecoin pairs and adds the result back to the `data` dictionary under a + new key. Parameters: - data (dict[str, pandas.DataFrame]): A dictionary where the keys are token pairs and the values are - corresponding DataFrames containing price and supply data. + data (dict[str, pandas.DataFrame]): A dictionary where the keys are token pairs and the values + are corresponding DataFrames containing price and supply data. - Returns: - dict[str, pandas.DataFrame]: The updated dictionary with the newly created stablecoin bundle added. + Returns: dict[str, pandas.DataFrame]: + The updated dictionary with the newly created stablecoin bundle added. """ # Iterate over all collateral tokens defined in the settings @@ -109,7 +115,7 @@ def create_stablecoin_bundle(data: dict[str, pd.DataFrame]) -> dict[str, pd.Data if df.empty: # Log a warning if the DataFrame is empty and skip to the next pair - logging.warning(f"Empty DataFrame for pair: {pair}") + logging.warning("Empty DataFrame for pair: %s", pair) continue if combined_df is None: @@ -232,10 +238,11 @@ def transform_main_chart_data( for protocol in protocols: protocol_main_chart_data = protocol_main_chart_data_mapping[protocol] if protocol_main_chart_data is None or protocol_main_chart_data.empty: - logging.warning(f"No data for pair {current_pair} from {protocol}") + logging.warning("No data for pair %s from %s", current_pair, protocol) collateral_token, debt_token = current_pair.split("-") st.subheader( - f":warning: No liquidable debt for the {collateral_token} collateral token and the {debt_token} debt token exists on the {protocol} protocol." + f":warning: No liquidable debt for the {collateral_token} collateral token and " + f"the {debt_token} debt token exists on the {protocol} protocol." ) continue diff --git a/apps/dashboard_app/dashboard.py b/apps/dashboard_app/dashboard.py index 668e0576..3fa54cd1 100644 --- a/apps/dashboard_app/dashboard.py +++ b/apps/dashboard_app/dashboard.py @@ -1,3 +1,7 @@ +""" +This script loads data and runs the dashboard. +""" + import logging from dashboard_app.charts import Dashboard from dashboard_app.helpers.load_data import DashboardDataHandler diff --git a/apps/dashboard_app/data_conector.py b/apps/dashboard_app/data_conector.py index 55cedde2..3b915213 100644 --- a/apps/dashboard_app/data_conector.py +++ b/apps/dashboard_app/data_conector.py @@ -1,3 +1,7 @@ +""" +This is module connects to a PostgreSQL database and fetches data. +""" + import os import pandas as pd @@ -8,6 +12,9 @@ class DataConnector: + """ + Handles database connection and fetches data. + """ REQUIRED_VARS = ("DB_USER", "DB_PASSWORD", "DB_HOST", "DB_PORT", "DB_NAME") ZKLEND_SQL_QUERY = """ SELECT diff --git a/apps/dashboard_app/helpers/ekubo.py b/apps/dashboard_app/helpers/ekubo.py index f3e5df69..cf1803cf 100644 --- a/apps/dashboard_app/helpers/ekubo.py +++ b/apps/dashboard_app/helpers/ekubo.py @@ -1,3 +1,8 @@ +""" +A module that interacts with an API to fetch the latest +liquidity data and applies it to a dataframe. +""" + import logging import time @@ -6,6 +11,10 @@ class EkuboLiquidity: + """ + Fetches data from a liquidity API and send it to the dataframe which updates the + liquidity of a token pair. + """ URL = "http://178.32.172.153/orderbook/" DEX = "Ekubo" LOWER_BOUND_VALUE = 0.95 @@ -61,7 +70,7 @@ def apply_liquidity_to_dataframe( data=liquidity, price=price, price_diff=price_diff, - bids=True if bids_or_asks["type"] == "bids" else False, + bids = bids_or_asks["type"] == "bids", ), ) self.data["debt_token_supply"] += self.data["Ekubo_debt_token_supply"] @@ -81,14 +90,13 @@ def fetch_liquidity(self, bids: bool = True) -> dict[str, str | list[float]]: :return: dict[str, str | list[float]] """ - params = self.params_for_bids + params = self.params_for_bids if bids else self.params_for_asks if not bids: - params = self.params_for_asks logging.warning( "Using collateral token as base token and debt token as quote token." ) - response = requests.get(self.URL, params=params) + response = requests.get(self.URL, params=params, timeout=10) if response.ok: liquidity = response.json() @@ -99,13 +107,12 @@ def fetch_liquidity(self, bids: bool = True) -> dict[str, str | list[float]]: data["prices"], data["quantities"] = zip(*liquidity[data["type"]]) except ValueError: time.sleep(300 if bids else 5) - self.fetch_liquidity(bids=True) - else: - return data - else: - time.sleep(300 if bids else 5) - self.fetch_liquidity( - bids=False if bids else True, + return self.fetch_liquidity(bids=True) + return data + + time.sleep(300 if bids else 5) + return self.fetch_liquidity( + bids= not bids, ) def _get_available_liquidity( diff --git a/apps/dashboard_app/helpers/load_data.py b/apps/dashboard_app/helpers/load_data.py index 75c37b00..c2ff852d 100644 --- a/apps/dashboard_app/helpers/load_data.py +++ b/apps/dashboard_app/helpers/load_data.py @@ -1,3 +1,7 @@ +""" +This module loads and handle the data. +""" + import asyncio import logging import math @@ -70,7 +74,7 @@ def _init_zklend_state() -> ZkLendState: zklend_state.interest_rate_models.debt = zklend_interest_rate_data["debt"].iloc[ 0 ] - logger.info(f"Initialized ZkLend state in {monotonic() - start:.2f}s") + logger.info("Initialized ZkLend state in %.2fs", monotonic() - start) return zklend_state diff --git a/apps/dashboard_app/helpers/loans_table.py b/apps/dashboard_app/helpers/loans_table.py index c71a0178..01582b5a 100644 --- a/apps/dashboard_app/helpers/loans_table.py +++ b/apps/dashboard_app/helpers/loans_table.py @@ -1,3 +1,7 @@ +""" +This module organizes and handles the loan data in a tabular manner. +""" + import pandas as pd from data_handler.handlers.loan_states.nostra_alpha.events import NostraAlphaState from data_handler.handlers.loan_states.nostra_mainnet.events import NostraMainnetState @@ -7,6 +11,10 @@ def get_protocol(state: State) -> str: + """ + Takes a parameter of State which gets the loan entities and + returns the string. + """ # TODO: Improve the inference. if isinstance(state, ZkLendState): return "zkLend" @@ -49,7 +57,7 @@ def get_loans_table_data( debt_interest_rate_model=state.interest_rate_models.debt, prices=prices, ) - if isinstance(state, NostraAlphaState) or isinstance(state, NostraMainnetState): + if isinstance(state, (NostraAlphaState, NostraMainnetState)): risk_adjusted_debt_usd = loan_entity.compute_debt_usd( risk_adjusted=True, debt_token_parameters=state.token_parameters.debt, @@ -105,6 +113,11 @@ def get_supply_function_call_parameters( protocol: str, token_addresses: list[str], ) -> tuple[list[str], str]: + """ + Takes two parameters of name of protocol and token address and then + Returns a tuple which the first element is the list of token address and + the second is supply function name + """ if protocol == "zkLend": return token_addresses, "felt_total_supply" if protocol in {"Nostra Alpha", "Nostra Mainnet"}: diff --git a/apps/dashboard_app/helpers/protocol_stats.py b/apps/dashboard_app/helpers/protocol_stats.py index a1ab6d70..867f2dbe 100644 --- a/apps/dashboard_app/helpers/protocol_stats.py +++ b/apps/dashboard_app/helpers/protocol_stats.py @@ -1,3 +1,7 @@ +""" +This module handles the collection and computation of statistics related to the protocol. +""" + import asyncio from collections import defaultdict from decimal import Decimal @@ -36,9 +40,12 @@ def get_general_stats( { "Protocol": protocol, "Number of active users": number_of_active_users, - # At the moment, Hashstack V0 and Hashstack V1 are the only protocols for which the number of active - # loans doesn't equal the number of active users. The reason is that Hashstack V0 and Hashstack V1 - # allow for liquidations on the loan level, whereas other protocols use user-level liquidations. + # At the moment, Hashstack V0 and Hashstack V1 are the only protocols + # for which the number of active + # loans doesn't equal the number of active users. + # The reason is that Hashstack V0 and Hashstack V1 + # allow for liquidations on the loan level, whereas other + # protocols use user-level liquidations. "Number of active loans": state.compute_number_of_active_loan_entities(), "Number of active borrowers": number_of_active_borrowers, "Total debt (USD)": round(loan_stats[protocol]["Debt (USD)"].sum(), 4), @@ -68,7 +75,7 @@ def get_supply_stats( for state in states: protocol = get_protocol(state=state) token_supplies = {} - for token in TOKEN_SETTINGS: + for token in TOKEN_SETTINGS.items(): ( addresses, selector, @@ -119,11 +126,16 @@ def get_supply_stats( def get_collateral_stats( states: list[State], ) -> pd.DataFrame: + """ + Get collateral stats for the dashboard. + :param states: States zklend, nostra_alpha, nostra_mainnet + :return: DataFrame with collateral stats + """ data = [] for state in states: protocol = get_protocol(state=state) token_collaterals = defaultdict(float) - for token in TOKEN_SETTINGS: + for token in TOKEN_SETTINGS.items(): # TODO: save zkLend amounts under token_addresses? if protocol == "zkLend": token_addresses = [ @@ -172,11 +184,16 @@ def get_collateral_stats( def get_debt_stats( states: list[State], ) -> pd.DataFrame: + """ + Get debts for the dashboard. + :param states: States zklend, nostra_alpha, nostra_mainnet + :return: DataFrame with debt stats + """ data = [] for state in states: protocol = get_protocol(state=state) token_debts = defaultdict(float) - for token in TOKEN_SETTINGS: + for token in TOKEN_SETTINGS.items(): # TODO: save zkLend amounts under token_addresses? if protocol == "zkLend": token_addresses = [ @@ -229,6 +246,12 @@ def get_utilization_stats( supply_stats: pd.DataFrame, debt_stats: pd.DataFrame, ) -> pd.DataFrame: + """ + Get utilization stats for the dashboard. + :param stats: DataFrame containing + general_stats, supply_stats, debt_stat. + :return: DataFrame with utilization stats + """ data = pd.DataFrame( { "Protocol": general_stats["Protocol"], diff --git a/apps/dashboard_app/helpers/settings.py b/apps/dashboard_app/helpers/settings.py index 9cdc962d..e9fa8e2d 100644 --- a/apps/dashboard_app/helpers/settings.py +++ b/apps/dashboard_app/helpers/settings.py @@ -1,11 +1,20 @@ +""" +A module for setting up tokens. +""" + from dataclasses import dataclass @dataclass class TokenSettings: + """ + This class represents the structure and properties of a token it has a + symbol, address and decimal factor. + """ symbol: str # Source: Starkscan, e.g. - # https://starkscan.co/token/0x049d36570d4e46f48e99674bd3fcc84644ddd6b96f7c741b1562b82f9e004dc7 for ETH. + # https://starkscan.co/token/0x049d36570d4e46f48e99674bd3fcc84644ddd6b96f7c741b1562b82f9e004dc7 + # for ETH. decimal_factor: float address: str @@ -57,7 +66,8 @@ class TokenSettings: # TODO: Introduce other pairs. -# TODO: Define the addresses first, then map (static or dynamic (with special treatment of DAI)?) to symbols. +# TODO: Define the addresses first, then map +# TODO:(static or dynamic (with special treatment of DAI)?) to symbols. PAIRS: list[str] = [ "ETH-USDC", "ETH-USDT", diff --git a/apps/dashboard_app/helpers/tools.py b/apps/dashboard_app/helpers/tools.py index 841eb3e3..5b46ab65 100644 --- a/apps/dashboard_app/helpers/tools.py +++ b/apps/dashboard_app/helpers/tools.py @@ -1,3 +1,7 @@ +""" +A module that fetches data of token prices, liquidity etc. +""" + import logging import math import os @@ -35,17 +39,22 @@ def get_collateral_token_range( collateral_token_underlying_address: str, collateral_token_price: float, ) -> list[float]: - TARGET_NUMBER_OF_VALUES = 50 + """ + Generates a range of prices for a collateral token and + Returns: A list of float values representing the range of prices for the collateral token. + """ + target_number_of_values = 50 start_price = 0.0 stop_price = collateral_token_price * 1.2 # Calculate rough step size to get about 50 (target) values - raw_step_size = (stop_price - start_price) / TARGET_NUMBER_OF_VALUES + raw_step_size = (stop_price - start_price) / target_number_of_values # Round the step size to the closest readable value (1, 2, or 5 times powers of 10) magnitude = 10 ** math.floor(math.log10(raw_step_size)) # Base scale step_factors = [1, 2, 2.5, 5, 10] difference = [ abs(50 - stop_price / (k * magnitude)) for k in step_factors - ] # Stores the difference between the target value and number of values generated from each step factor. + ] # Stores the difference between the target value and + #number of values generated from each step factor. readable_step = ( step_factors[difference.index(min(difference))] * magnitude ) # Gets readable step from step factor with values closest to the target value. @@ -55,6 +64,9 @@ def get_collateral_token_range( async def get_symbol(token_address: str) -> str: + """ + Takes the address of a token and returns the symbol of that token + """ # DAI V2's symbol is `DAI` but we don't want to mix it with DAI = DAI V1. if ( token_address @@ -78,8 +90,8 @@ def get_prices(token_decimals: dict[str, int]) -> dict[str, float]: :param token_decimals: Token decimals. :return: Dict with token addresses as keys and token prices as values. """ - URL = "https://starknet.impulse.avnu.fi/v1/tokens/short" - response = requests.get(URL) + url = "https://starknet.impulse.avnu.fi/v1/tokens/short" + response = requests.get(url, timeout=10) if not response.ok: response.raise_for_status() @@ -106,13 +118,13 @@ def get_prices(token_decimals: dict[str, int]) -> dict[str, float]: token_info = token_info_map.get(token) if not token_info: - logging.error(f"Token {token} not found in response.") + logging.error("Token %s not found in response.", token) continue if decimals != token_info.get("decimals"): logging.error( - f"Decimal mismatch for token {token}: expected {decimals}, got {token_info.get('decimals')}" - ) + "Decimal mismatch for token %s: expected %d, got %d" + ,token, decimals, token_info.get('decimals')) continue prices[token] = token_info.get("currentPrice") @@ -120,12 +132,12 @@ def get_prices(token_decimals: dict[str, int]) -> dict[str, float]: return prices -def add_leading_zeros(hash: str) -> str: +def add_leading_zeros(address: str) -> str: """ Converts e.g. `0x436d8d078de345c11493bd91512eae60cd2713e05bcaa0bb9f0cba90358c6e` to `0x00436d8d078de345c11493bd91512eae60cd2713e05bcaa0bb9f0cba90358c6e`. """ - return "0x" + hash[2:].zfill(64) + return "0x" + address[2:].zfill(64) def get_addresses( @@ -133,6 +145,10 @@ def get_addresses( underlying_address: str | None = None, underlying_symbol: str | None = None, ) -> list[str]: + """ + get the token address based on underlying address or symbol and + Returns it. + """ # Up to 2 addresses can match the given `underlying_address` or `underlying_symbol`. if underlying_address: addresses = [ @@ -148,10 +164,8 @@ def get_addresses( ] else: raise ValueError( - "Both `underlying_address` = {} or `underlying_symbol` = {} are not specified.".format( - underlying_address, - underlying_symbol, - ) + f"Both `underlying_address` = {underlying_symbol} " + f"or `underlying_symbol` = {underlying_symbol} are not specified." ) assert len(addresses) <= 2 return addresses @@ -161,6 +175,9 @@ def get_underlying_address( token_parameters: TokenParameters, underlying_symbol: str, ) -> str: + """ + Retrieves the underlying address for a given underlying symbol. + """ # One underlying address at maximum can match the given `underlying_symbol`. underlying_addresses = { x.underlying_address From 98e7e8bcf98719d9d8a712da3a865945dbf5a43d Mon Sep 17 00:00:00 2001 From: Joseph Gimba <86230531+Joewizy@users.noreply.github.com> Date: Thu, 28 Nov 2024 22:32:49 +0100 Subject: [PATCH 04/10] Update dashboard_app_pylint.yml --- .github/workflows/dashboard_app_pylint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dashboard_app_pylint.yml b/.github/workflows/dashboard_app_pylint.yml index 81aad05c..b74e8cbe 100644 --- a/.github/workflows/dashboard_app_pylint.yml +++ b/.github/workflows/dashboard_app_pylint.yml @@ -1,6 +1,6 @@ name: Dashboard App -on: [push, pull_request] +on: [push] jobs: lint: From 48b93d6915338b27a86d7b5de29280a2d5f24013 Mon Sep 17 00:00:00 2001 From: Lulu <104063177+Luluameh@users.noreply.github.com> Date: Thu, 28 Nov 2024 07:49:07 +0000 Subject: [PATCH 05/10] docs: update documentation for all apps --- README.md | 70 +++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 60 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 6c9886ee..c51b5f6f 100644 --- a/README.md +++ b/README.md @@ -2,39 +2,89 @@ This project consist of a monorepo with components required for the implementation of DeRisk on Starknet. There are several components in this repository, each with its own purpose and functionality. The main components are: -- `data_handler` -- `web_app` -- `legacy_app` -- `dashboard_app` -- `shared` - Common code shared between the components +- [`data_handler`](./apps/data_handler/README.md) - Data processing and analysis component +- [`web_app`](./apps/web_app/README.md) - Main web application interface +- [`legacy_app`](./apps/legacy_app/README.md) - Legacy application functionality +- [`dashboard_app`](./apps/dashboard_app/README.md) - Analytics dashboard +- [`shared`](./apps/shared/README.md) - Common code shared between the components + +## Quick Start Guide + +### Prerequisites +- Docker installed on your machine (v19.03+ recommended). +- Docker Compose installed (v1.27+ recommended). + +### Data Handler + +The data handler component processes and manages data for the DeRisk platform. + +#### Local Development -## Data Handler 1. To set up this project run next command for local development in `derisk-research` directory: + +2. Environment Configuration: +```bash +cp apps/data_handler/.env.example apps/data_handler/.env.dev ``` +3. Start the Services: + +```bash docker-compose -f devops/dev/docker-compose.data-handler.yaml up --build ``` -2. To run test cases for this project run next command in `derisk-research` directory: +4. Stop the Services: +```bash +docker-compose -f devops/dev/docker-compose.data-handler.yaml down ``` + +5. To run test cases for this project run next command in `derisk-research` directory: +```bash make test_data_handler ``` +For detailed documentation, see the [Data Handler](./apps/data_handler/README.md) + + ## Legacy app + +The legacy app provides essential functionality for data visualization and analysis through a Streamlit interface. + +#### Local Development + 1. To set up this project run next command for local development in `derisk-research` directory: -``` +```bash make setup ``` + 2. To run streamlit app run next command in `derisk-research` directory: -``` +```bash make app ``` +3. Start Jupyter notebook (optional): +```bash +make notebook +``` +For detailed documentation, see [`legacy_app`](./apps/legacy_app/README.md) + ## Web app (Notification app) -1. To set up this project run next command for local development in `derisk-research` directory: +To set up this project run next command for local development in `derisk-research` directory: + +1. Environment Configuration: +```bash +cp apps/web_app/.env.example apps/web_app/.env.dev ``` + +2. Start the Services: +```bash docker-compose -f devops/dev/docker-compose.notification-app.yaml up --build ``` +3. Stop the Services: +```bash +docker-compose -f devops/dev/docker-compose.notification-app.yaml down +``` + ## Shared package (Common code shared between the components) 1. How to run test cases for shared package, run next command in root folder: From 251e60b8ce095803a338783c7231376d45d17ab1 Mon Sep 17 00:00:00 2001 From: Lulu <104063177+Luluameh@users.noreply.github.com> Date: Thu, 28 Nov 2024 18:45:55 +0000 Subject: [PATCH 06/10] docs: update documentation structure --- README.md | 12 +++++- apps/dashboard_app/README.md | 75 ++++++++++++++++++++++++++++++++++++ apps/data_handler/README.md | 22 +++++++---- apps/web_app/README.md | 30 +++++++++++---- 4 files changed, 123 insertions(+), 16 deletions(-) create mode 100644 apps/dashboard_app/README.md diff --git a/README.md b/README.md index c51b5f6f..a70d9585 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ There are several components in this repository, each with its own purpose and f ### Prerequisites - Docker installed on your machine (v19.03+ recommended). -- Docker Compose installed (v1.27+ recommended). +- Docker Compose installed (v2.0+ recommended). ### Data Handler @@ -85,6 +85,16 @@ docker-compose -f devops/dev/docker-compose.notification-app.yaml up --build docker-compose -f devops/dev/docker-compose.notification-app.yaml down ``` +## Dashboard App + +Interactive dashboard application for visualizing and analyzing DeRisk data. + +### Key Features +- Interactive data visualization +- Protocol statistics monitoring +- Loan portfolio analysis +- Real-time data updates +For detailed documentation, see the [Dashboard App](./apps/dashboard_app/README.md) ## Shared package (Common code shared between the components) 1. How to run test cases for shared package, run next command in root folder: diff --git a/apps/dashboard_app/README.md b/apps/dashboard_app/README.md new file mode 100644 index 00000000..c21a70d6 --- /dev/null +++ b/apps/dashboard_app/README.md @@ -0,0 +1,75 @@ +# DeRisk Dashboard App + +Interactive dashboard application for visualizing and analyzing DeRisk data. + + +## Prerequisites + +- Python 3.11+ +- Poetry +- Docker + +## Setup + +### Local Development + +1. Install dependencies: +```bash +./setup.sh +``` + +2. Run the dashboard: +```bash +poetry run python dashboard.py +``` + +### Docker Setup + +1. Build the image: + + +2. Run the container: + +3. stop container: + +## Key Features + +- Interactive data visualization +- Protocol statistics monitoring +- Loan portfolio analysis +- Real-time data updates + +### Adding New Charts + +1. Create chart module in `charts/` +2. Implement chart logic using main.py templates +3. Register in dashboard.py + +### Data Integration + +Use `data_connector.py` to add new data sources: + +```python +from data_connector import DataConnector + +connector = DataConnector() +data = connector.get_data() +``` + +## Testing + +```bash +poetry run pytest +``` + +## Contributing + +1. Fork the [repository](https://github.com/CarmineOptions/derisk-research) +2. Create feature branch +3. Submit pull request + +Read [contribution guide](CONTRIBUTING.md) + +## License + +[License details](LICENSE.txt) diff --git a/apps/data_handler/README.md b/apps/data_handler/README.md index f4d338a5..37e0d922 100644 --- a/apps/data_handler/README.md +++ b/apps/data_handler/README.md @@ -1,5 +1,6 @@ # DeRisk Data Handler +## Overview This project was created to make data public for Derisk Alert app. This app is not intended for you to use in production. It's just a research project. @@ -23,7 +24,7 @@ git clone https://github.com/CarmineOptions/derisk-research.git cd data_handler ``` -### 3. Set up `.env` file +### 3. Configure Environment Variables Create `.env` file or just rename `.env.example` --> `.env` @@ -57,26 +58,29 @@ docker-compose up -d --build docker-compose down ``` -## Data migrations +## Data migrations with Alembic In this project is using `alembic` for data migrations. -For generating new migration use this command: -In folder `apps` run these commands: + +### Generating Migrations +Navigate to the `apps` folder and generate a new migration using the following command: ```bash +cd apps alembic -c data_handler/alembic.ini revision --autogenerate -m "your message" ``` - +### Applying Migrations After generating new migration, you need to apply it: ```bash alembic -c data_handler/alembic.ini upgrade head ``` - +### Rolling Back Migrations For downgrading migration: ```bash alembic -c data_handler/alembic.ini downgrade -1 ``` +### Migration Utility Commands Useful commands: Purge all celery tasks: ```bash @@ -91,13 +95,15 @@ Go to bash docker-compose exec backend bash ``` -## How to run migration command: +## Running Migration Command: 1. Go to root folder `derisk-research` + +### Prepare the Environment 2. Run up db in docker: ``` docker-compose -f devops/dev/docker-compose.db.yaml up -d --remove-orphans ``` -3. Go to `data_hander` folder: +3. Navigate to the `data_hander` directory: ``` cd apps/data_handler ``` diff --git a/apps/web_app/README.md b/apps/web_app/README.md index ff850f9d..f41e88ac 100644 --- a/apps/web_app/README.md +++ b/apps/web_app/README.md @@ -1,15 +1,23 @@ # DeRisk Alert # How it works: + +This section provides an overview of how the DeRisk Alert system operates. ![A diagram that illustrates how this app works](docs/how-it-works.drawio.png) + +### Demo Video ### `Here is a demo video of how it works` [Click](https://drive.google.com/file/d/1TYwEx6PWvPerrJSfiePQzZEtbj53Yn_g/view?usp=sharing) +### Notification System ### When the `health ratio` is appropriate to the value you ### have set you will be notified via `Telegram` ![An image that illustrates a notification](docs/notification.png) -# Database ordering: +# Database Structure: + + +This diagram shows the ordering and structure of the database for the DeRisk Alert system: ![A diagram that illustrates the ordering of the DB](docs/db-ordering.png) ## Requirements @@ -18,20 +26,28 @@ - docker - docker-compose +## Telegram Bot Setup + +To receive alerts via Telegram, follow these steps to set up your Telegram bot: + ## How to get Telegram Token: -### 1. To get a token and create a chatbot, you need to find a bot named BotFather in the Telegram messenger. +### 1. Find BotFather: +To get a token and create a chatbot, you need to find a bot named BotFather in the Telegram messenger. ![An image that shows how to find bot father in telegram](docs/find-bot-father.jpg) -### 2. In the BotFather bot, you need to write the command `/newbot`. After that, BotFather will prompt you to enter: +### 2. Create Your Bot: +In the BotFather bot, you need to write the command `/newbot`. After that, BotFather will prompt you to enter: - the name of your bot that users will see; - uri of the bot, i.e. the link to the bot that will be added to the link https://t.me/{youruri}. ![An image that shows how to create a new bot](docs/newbot-botfather.jpg) -### 3. After the data is entered and it has passed validation, BotFather will respond with a message that will contain the API token of the created bot. +### 3. Retrieve Your Token: +After the data is entered and it has passed validation, BotFather will respond with a message that will contain the API token of the created bot. ![An image that shows how to get a created token](docs/get-token.jpg) -### 4. Done! At this moment, the bot has already been created, and it is possible to subscribe to it by finding it in Telegram search or by following the link. +### 4. Access Your Bot: +Done! At this moment, the bot has already been created, and it is possible to subscribe to it by finding it in Telegram search or by following the link. # Setup @@ -41,7 +57,7 @@ git clone https://github.com/CarmineOptions/derisk-research.git ``` -### 2. Go to `web_app/` +### 2. Navigate to the `web_app/` Directory ```bash @@ -77,7 +93,7 @@ DATA_HANDLER_URL=# url to data handler docker-compose up -d --build ``` -#### Stop your containers +#### 6. Stop your containers ```bash docker-compose down From 49af6e5f1a9f909043d4f27001a91a3fa79d8bc8 Mon Sep 17 00:00:00 2001 From: djeck1432 Date: Fri, 29 Nov 2024 08:29:21 +0100 Subject: [PATCH 07/10] remove unused folders --- apps/web_app/order_books/__init__.py | 0 apps/web_app/order_books/abstractions.py | 106 ------- apps/web_app/order_books/constants.py | 41 --- apps/web_app/order_books/ekubo/README.md | 34 --- apps/web_app/order_books/ekubo/__init__.py | 0 .../order_books/ekubo/api_connector.py | 265 ----------------- .../order_books/ekubo/docs/pool_liquidity.md | 32 --- apps/web_app/order_books/ekubo/docs/pools.md | 52 ---- .../order_books/ekubo/docs/token_lists.md | 22 -- .../order_books/ekubo/docs/token_prices.md | 26 -- apps/web_app/order_books/ekubo/histogram.py | 159 ---------- apps/web_app/order_books/ekubo/main.py | 244 ---------------- apps/web_app/order_books/haiko/README.md | 23 -- apps/web_app/order_books/haiko/__init__.py | 0 .../order_books/haiko/api_connector.py | 148 ---------- .../order_books/haiko/docs/block_info.md | 29 -- apps/web_app/order_books/haiko/docs/depth.md | 31 -- .../web_app/order_books/haiko/docs/markets.md | 89 ------ apps/web_app/order_books/haiko/docs/prices.md | 29 -- apps/web_app/order_books/haiko/docs/tokens.md | 35 --- apps/web_app/order_books/haiko/logger.py | 35 --- apps/web_app/order_books/haiko/main.py | 272 ------------------ apps/web_app/order_books/haiko/report.py | 114 -------- apps/web_app/utils/helpers.py | 76 ----- apps/web_app/utils/settings.py | 53 ---- apps/web_app/utils/state.py | 153 ---------- apps/web_app/utils/zklend.py | 138 --------- 27 files changed, 2206 deletions(-) delete mode 100644 apps/web_app/order_books/__init__.py delete mode 100644 apps/web_app/order_books/abstractions.py delete mode 100644 apps/web_app/order_books/constants.py delete mode 100644 apps/web_app/order_books/ekubo/README.md delete mode 100644 apps/web_app/order_books/ekubo/__init__.py delete mode 100644 apps/web_app/order_books/ekubo/api_connector.py delete mode 100644 apps/web_app/order_books/ekubo/docs/pool_liquidity.md delete mode 100644 apps/web_app/order_books/ekubo/docs/pools.md delete mode 100644 apps/web_app/order_books/ekubo/docs/token_lists.md delete mode 100644 apps/web_app/order_books/ekubo/docs/token_prices.md delete mode 100644 apps/web_app/order_books/ekubo/histogram.py delete mode 100644 apps/web_app/order_books/ekubo/main.py delete mode 100644 apps/web_app/order_books/haiko/README.md delete mode 100644 apps/web_app/order_books/haiko/__init__.py delete mode 100644 apps/web_app/order_books/haiko/api_connector.py delete mode 100644 apps/web_app/order_books/haiko/docs/block_info.md delete mode 100644 apps/web_app/order_books/haiko/docs/depth.md delete mode 100644 apps/web_app/order_books/haiko/docs/markets.md delete mode 100644 apps/web_app/order_books/haiko/docs/prices.md delete mode 100644 apps/web_app/order_books/haiko/docs/tokens.md delete mode 100644 apps/web_app/order_books/haiko/logger.py delete mode 100644 apps/web_app/order_books/haiko/main.py delete mode 100644 apps/web_app/order_books/haiko/report.py delete mode 100644 apps/web_app/utils/helpers.py delete mode 100644 apps/web_app/utils/settings.py delete mode 100644 apps/web_app/utils/state.py delete mode 100644 apps/web_app/utils/zklend.py diff --git a/apps/web_app/order_books/__init__.py b/apps/web_app/order_books/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/apps/web_app/order_books/abstractions.py b/apps/web_app/order_books/abstractions.py deleted file mode 100644 index 9403ab4f..00000000 --- a/apps/web_app/order_books/abstractions.py +++ /dev/null @@ -1,106 +0,0 @@ -from abc import ABC, abstractmethod -from datetime import datetime, timezone -from decimal import Decimal - -from order_books.constants import TOKEN_MAPPING - -from db.schemas import OrderBookModel - - -class OrderBookBase(ABC): - DEX: str = None - MIN_PRICE_RANGE = Decimal("0.1") - MAX_PRICE_RANGE = Decimal("1.90") - - def __init__(self, token_a: str, token_b: str): - self.token_a = token_a - self.token_b = token_b - self.asks = [] # List of tuples (price, quantity) - self.bids = [] # List of tuples (price, quantity) - self.timestamp = None - self.block = None - self.current_price = Decimal("0") - self.token_a_decimal = self.get_token_decimals(token_a) - self.token_b_decimal = self.get_token_decimals(token_b) - self.total_liquidity = Decimal("0") - - def get_token_decimals(self, token: str) -> Decimal: - """ - Get the token decimals - :return: tuple - The token decimals - """ - token_config = TOKEN_MAPPING.get(token) - if token_config: - return token_config.decimals - return Decimal("0") - - @abstractmethod - def fetch_price_and_liquidity(self) -> None: - """ - Fetches the price and liquidity data from the connector - """ - pass - - @abstractmethod - def _calculate_order_book(self, *args, **kwargs) -> tuple: - """ - Calculates order book data based on the liquidity data - """ - pass - - def calculate_price_range(self) -> tuple: - """ - Calculate the minimum and maximum price based on the current price. - :return: tuple - The minimum and maximum price range. - """ - min_price = self.current_price * self.MIN_PRICE_RANGE - max_price = self.current_price * self.MAX_PRICE_RANGE - return min_price, max_price - - @abstractmethod - def calculate_liquidity_amount(self, *args, **kwargs) -> Decimal: - """ - Calculates the liquidity amount based on the liquidity delta difference - """ - pass - - @staticmethod - def get_sqrt_ratio(tick: Decimal) -> Decimal: - """ - Get the square root ratio based on the tick. - :param tick: tick value - :return: square root ratio - """ - return (Decimal("1.000001").sqrt() ** tick) * (Decimal(2) ** 128) - - @abstractmethod - def tick_to_price(self, tick: Decimal) -> Decimal: - """ - Converts the tick value to price - """ - pass - - def get_order_book(self) -> dict: - """ - Returns the order book data - :return: dict - The order book data - """ - dt_now = datetime.now(timezone.utc) - - return { - "token_a": self.token_a, - "token_b": self.token_b, - "timestamp": int(dt_now.replace(tzinfo=timezone.utc).timestamp()), - "block": self.block, - "dex": self.DEX, - "asks": sorted(self.asks, key=lambda x: x[0]), - "bids": sorted(self.bids, key=lambda x: x[0]), - } - - def serialize(self) -> OrderBookModel: - """ - Serialize the order book data - :return: dict - The serialized order book data - """ - order_book_data = self.get_order_book() - return OrderBookModel(**order_book_data) diff --git a/apps/web_app/order_books/constants.py b/apps/web_app/order_books/constants.py deleted file mode 100644 index e058cf94..00000000 --- a/apps/web_app/order_books/constants.py +++ /dev/null @@ -1,41 +0,0 @@ -from dataclasses import dataclass - - -@dataclass -class TokenConfig: - name: str - decimals: int - - -TOKEN_MAPPING = { - "0x49d36570d4e46f48e99674bd3fcc84644ddd6b96f7c741b1562b82f9e004dc7": TokenConfig( - name="ETH", decimals=18 - ), - "0x53c91253bc9682c04929ca02ed00b3e423f6710d2ee7e0d5ebb06f3ecf368a8": TokenConfig( - name="USDC", decimals=6 - ), - "0x068f5c6a61780768455de69077e07e89787839bf8166decfbf92b645209c0fb8": TokenConfig( - name="USDT", decimals=6 - ), - "0x00da114221cb83fa859dbdb4c44beeaa0bb37c7537ad5ae66fe5e0efd20e6eb3": TokenConfig( - name="DAI", decimals=18 - ), - "0x03fe2b97c1fd336e750087d68b9b867997fd64a2661ff3ca5a7c771641e8e7ac": TokenConfig( - name="wBTC", decimals=8 - ), - "0x04718f5a0fc34cc1af16a1cdee98ffb20c31f5cd61d6ab07201858f4287c938d": TokenConfig( - name="STRK", decimals=18 - ), - "0x0719b5092403233201aa822ce928bd4b551d0cdb071a724edd7dc5e5f57b7f34": TokenConfig( - name="UNO", decimals=18 - ), - "0x00585c32b625999e6e5e78645ff8df7a9001cf5cf3eb6b80ccdd16cb64bd3a34": TokenConfig( - name="ZEND", decimals=18 - ), - "0x042b8f0484674ca266ac5d08e4ac6a3fe65bd3129795def2dca5c34ecc5f96d2": TokenConfig( - name="wstETH", decimals=18 - ), - "0x0124aeb495b947201f5fac96fd1138e326ad86195b98df6dec9009158a533b49": TokenConfig( - name="LORDS", decimals=18 - ), -} diff --git a/apps/web_app/order_books/ekubo/README.md b/apps/web_app/order_books/ekubo/README.md deleted file mode 100644 index ee23b617..00000000 --- a/apps/web_app/order_books/ekubo/README.md +++ /dev/null @@ -1,34 +0,0 @@ -# Ekubo - -This folder contains all interaction with [Ekubo API](https://docs.ekubo.org/integration-guides/reference/ekubo-api/api-endpoints). - -# Endpoints Documentation: -- [Get Pool Liquidity](docs/pool_liquidity.md) - Detailed information about liquidity in different pools. -- [Get Pools](docs/pools.md) - Detailed information about various pools. -- [Get Token Prices](docs/token_prices.md) - Information on how to retrieve current token prices. -- [Get List of Tokens](docs/token_lists.md) - List of tokens with detailed information. - - -# How to run -In this folder: `ekubo` run next command -```bash -python main.py -``` - -# How to check quality of data -In this folder: `ekubo` run next command to see the histogram of the data -```bash -python manage.py histogram -``` - -## Histogram - -To run the script to see histogram, use the following command: -!!! Pay attention, you should run it from `web_app` folder -```sh -python histogram.py --type asks -``` -or -```sh -python histogram.py --type bids -``` \ No newline at end of file diff --git a/apps/web_app/order_books/ekubo/__init__.py b/apps/web_app/order_books/ekubo/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/apps/web_app/order_books/ekubo/api_connector.py b/apps/web_app/order_books/ekubo/api_connector.py deleted file mode 100644 index 01cf3eb4..00000000 --- a/apps/web_app/order_books/ekubo/api_connector.py +++ /dev/null @@ -1,265 +0,0 @@ -import time - -from web_app.utils.abstractions import AbstractionAPIConnector - - -class EkuboAPIConnector(AbstractionAPIConnector): - API_URL = "https://mainnet-api.ekubo.org" - - @classmethod - def get_token_prices(cls, quote_token: str) -> dict: - """ - Get the prices of all other tokens in terms of the specified quote token. - - :param quote_token: The address of the quote token on StarkNet. - :type quote_token: str - :return: A dictionary containing the timestamp and a list of dictionaries for each token with price details. - Each token's dictionary includes its address, price, and trading volume. - :rtype: dict - - The response dictionary structure is as follows: - { - 'timestamp': int, # Unix timestamp in milliseconds when the data was recorded - 'prices': [ - { - 'token': str, # The address of the token on the blockchain - 'price': str, # The current price of the token expressed in terms of the quote token - 'k_volume': str # The trading volume for the token, expressed in the smallest unit counts - } - ] - } - - Example: - { - 'timestamp': 1715350510592, - 'prices': [ - { - 'token': '0x49d36570d4e46f48e99674bd3fcc84644ddd6b96f7c741b1562b82f9e004dc7', - 'price': '0.0481921', - 'k_volume': '1176822712207290630680641242' - } - ] - } - """ - endpoint = f"/price/{quote_token}" - return cls.send_get_request(endpoint) - - @classmethod - def get_pool_liquidity(cls, key_hash: str) -> list: - """ - Get the liquidity delta for each tick for the given pool key hash. The response includes an array - of objects, each containing details about the tick and the net liquidity delta difference. - - :param key_hash: The pool key hash in hexadecimal or decimal format. - :type key_hash: str - :return: An array of objects detailing the current liquidity chart for the pool. Each object - in the array includes the following: - - 'tick': The tick index as an integer. - - 'net_liquidity_delta_diff': The difference in net liquidity for the tick, represented as a string. - :rtype: list - - Example of returned data: - [ - { - "tick": -88719042, - "net_liquidity_delta_diff": "534939583228319557" - } - ] - - """ - endpoint = f"/pools/{key_hash}/liquidity" - return cls.send_get_request(endpoint) - - def get_list_tokens(self) -> list: - """ - Retrieves a list of tokens from the blockchain layer 2. Each token object - in the list provides comprehensive details - including its name, symbol, decimal precision, layer 2 address, and other attributes. - - :return: A list of dictionaries, each representing a token with the following attributes: - - 'name' (str): The name of the token, e.g., "Wrapped BTC". - - 'symbol' (str): The abbreviated symbol of the token, e.g., "WBTC". - - 'decimals' (int): The number of decimal places the token is divided into, e.g., 8. - - 'l2_token_address' (str): The address of the token on layer 2, in hexadecimal format. - - 'sort_order' (int): An integer specifying the order in which the token should be displayed - relative to others; lower numbers appear first. - - 'total_supply' (str): The total supply of the token, if known. This can be 'None' - if the total supply is not set or unlimited. - - 'logo_url' (str): A URL to the token's logo image. - - Example of returned data: - [ - { - "name": "Wrapped BTC", - "symbol": "WBTC", - "decimals": 8, - "l2_token_address": "0x03fe2b97c1fd336e750087d68b9b867997fd64a2661ff3ca5a7c771641e8e7ac", - "sort_order": 0, - "total_supply": "None", - "logo_url": "https://imagedelivery.net/0xPAQaDtnQhBs8IzYRIlNg/7dcb2db2-a7a7-44af-660b-8262e057a100/logo" - } - ] - - """ - endpoint = "/tokens" - return self.send_get_request(endpoint) - - def get_pools(self) -> list: - """ - Retrieves a list of detailed information about various pools. Each entry in the list is a dictionary - that provides comprehensive details about a pool, including the tokens involved, fees, - and other relevant metrics. - - :return: A list of dictionaries, each containing detailed information about a pool. The structure of - each dictionary is as follows: - - 'key_hash': The unique identifier of the pool in hexadecimal format. (str) - - 'token0': The address of the first token in the pool on the blockchain. (str) - - 'token1': The address of the second token in the pool on the blockchain. (str) - - 'fee': The fee associated with the pool transactions, in hexadecimal format. (str) - - 'tick_spacing': The minimum granularity of price movements in the pool. (int) - - 'extension': Additional information or features related to the pool, in hexadecimal format. (str) - - 'sqrt_ratio': The square root of the current price ratio between token0 and token1, - in hexadecimal format. (str) - - 'tick': The current position of the pool in its price range. (int) - - 'liquidity': The total liquidity available in the pool. (str) - - 'lastUpdate': Dictionary containing details about the latest update event: - - 'event_id': Identifier of the last update event. (str) - :rtype: list - - Example of returned data: - [ - { - "key_hash": "0x34756a876aa3288b724dc967d43ee72d9b9cf390023775f0934305233767156", - "token0": "0xda114221cb83fa859dbdb4c44beeaa0bb37c7537ad5ae66fe5e0efd20e6eb3", - "token1": "0x49210ffc442172463f3177147c1aeaa36c51d152c1b0630f2364c300d4f48ee", - "fee": "0x20c49ba5e353f80000000000000000", - "tick_spacing": 1000, - "extension": "0x0", - "sqrt_ratio": "0x5c358a19c219f31e027d5ac98e8d5656", - "tick": -2042238, - "liquidity": "35067659406163360487", - "lastUpdate": { - "event_id": "2747597966999556" - } - } - ] - """ - endpoint = "/pools" - response = self.send_get_request(endpoint) - if isinstance(response, dict) and response.get("error"): - # handling too many requests - time.sleep(2) - response = self.send_get_request(endpoint) - return response - - def get_pool_states(self) -> list: - """ - Fetches the current state of all pools. - - The method sends a GET request to the endpoint "/pools" and retrieves a list of pool states. - Each pool state is represented as a dictionary with the following keys: - - - key_hash (str): A unique hash identifier for the pool. - - token0 (str): The address of the first token in the pool. - - token1 (str): The address of the second token in the pool. - - fee (str): The fee tier of the pool. - - tick_spacing (int): The tick spacing for the pool. - - extension (str): Extension field (details unspecified). - - sqrt_ratio (str): The square root of the price ratio (Q64.96 format). - - tick (int): The current tick of the pool. - - liquidity (str): The current liquidity in the pool. - - lastUpdate (dict): A dictionary containing the last update information: - - event_id (str): The event identifier for the last update. - - Returns: - list: A list of dictionaries, each representing the state of a pool. - """ - endpoint = "/pools" - return self.send_get_request(endpoint) - - def get_pair_liquidity(self, tokenA: str, tokenB: str) -> dict: - """ - Fetches the liquidity information for a specified token pair. - - The method sends a GET request to the endpoint "/tokens/{tokenA}/{tokenB}/liquidity" - to retrieve the liquidity data for the given token pair. The response is a dictionary - containing the following keys: - - - net_liquidity_delta_diff (str): The net liquidity delta difference for the token pair. - - tick (int): The current tick for the liquidity of the token pair. - - Args: - tokenA (str): The address of the first token. - tokenB (str): The address of the second token. - - Returns: - dict: A dictionary containing the liquidity information for the token pair. - """ - endpoint = f"/tokens/{tokenA}/{tokenB}/liquidity" - return self.send_get_request(endpoint) - - def get_pair_states(self, tokenA: str, tokenB: str) -> dict: - """ - Fetches the state information for a specified token pair. - - The method sends a GET request to the endpoint "/pair/{tokenA}/{tokenB}" - to retrieve the state data for the given token pair. The response is a dictionary - containing the following keys: - - - timestamp (int): The timestamp of the data. - - tvlByToken (list): A list of dictionaries containing the total value locked (TVL) by token: - - token (str): The address of the token. - - balance (str): The balance of the token in the pool. - - volumeByToken (list): A list of dictionaries containing the volume by token: - - token (str): The address of the token. - - volume (str): The trading volume of the token. - - fees (str): The fees generated by the token. - - revenueByToken (list): A list of dictionaries containing the revenue by token: - - token (str): The address of the token. - - revenue (str): The revenue generated by the token. - - tvlDeltaByTokenByDate (list): A list of dictionaries containing the TVL delta by token by date: - - token (str): The address of the token. - - date (str): The date of the TVL delta. - - delta (str): The change in TVL for the token. - - volumeByTokenByDate (list): A list of dictionaries containing the volume by token by date: - - token (str): The address of the token. - - date (str): The date of the volume data. - - volume (str): The trading volume of the token. - - fees (str): The fees generated by the token. - - revenueByTokenByDate (list): A list of dictionaries containing the revenue by token by date: - - token (str): The address of the token. - - date (str): The date of the revenue data. - - revenue (str): The revenue generated by the token. - - topPools (list): A list of dictionaries containing the top pools: - - fee (str): The fee tier of the pool. - - tick_spacing (int): The tick spacing of the pool. - - extension (str): Extension field (details unspecified). - - volume0_24h (str): The 24-hour trading volume of token0. - - volume1_24h (str): The 24-hour trading volume of token1. - - fees0_24h (str): The 24-hour trading fees for token0. - - fees1_24h (str): The 24-hour trading fees for token1. - - tvl0_total (str): The total TVL of token0. - - tvl1_total (str): The total TVL of token1. - - tvl0_delta_24h (str): The 24-hour delta in TVL for token0. - - tvl1_delta_24h (str): The 24-hour delta in TVL for token1. - - Args: - tokenA (str): The address of the first token. - tokenB (str): The address of the second token. - - Returns: - dict: A dictionary containing the state information for the token pair. - """ - endpoint = f"/pair/{tokenA}/{tokenB}" - return self.send_get_request(endpoint) - - def get_pair_price(self, base_token: str, quote_token: str) -> dict: - """ - Fetches the price information for a specified token pair. - :param base_token: Base token address - :param quote_token: Quote token address - :return: - {'price': '0.9528189037', 'timestamp': '2024-05-18T10:41:37.091Z'} - """ - endpoint = f"/price/{base_token}/{quote_token}" - return self.send_get_request(endpoint) diff --git a/apps/web_app/order_books/ekubo/docs/pool_liquidity.md b/apps/web_app/order_books/ekubo/docs/pool_liquidity.md deleted file mode 100644 index b145313e..00000000 --- a/apps/web_app/order_books/ekubo/docs/pool_liquidity.md +++ /dev/null @@ -1,32 +0,0 @@ -# Get Pool Liquidity - -### Class Method -Use this class method to retrieve liquidity data for each tick associated with a given pool key hash: -`Ekubo.api_connector.EkuboAPIConnector.get_pool_liquidity(key_hash)` -To get `key_hash`, use the `get_pools()` method. - -### Data Structure -This endpoint retrieves the liquidity delta for each tick for a specified pool key hash. The response includes an array of objects, each detailing specific liquidity information for a tick. Each object in the array contains: -- **tick**: The index of the tick within the pool's range. - - Type: `int` -- **net_liquidity_delta_diff**: The net change in liquidity at this tick, expressed as a difference. - - Type: `str` - -### Parameters -- **key_hash**: The unique identifier of the pool, can be in hexadecimal or decimal format. - - Type: `str` - -### Return -- Returns an array of dictionaries, each providing details on the liquidity state for each tick. - -### Example of Returned Data -```json -[ - { - "tick": -88719042, - "net_liquidity_delta_diff": "534939583228319557" - } -] -``` - - diff --git a/apps/web_app/order_books/ekubo/docs/pools.md b/apps/web_app/order_books/ekubo/docs/pools.md deleted file mode 100644 index 0f3bde93..00000000 --- a/apps/web_app/order_books/ekubo/docs/pools.md +++ /dev/null @@ -1,52 +0,0 @@ -# Get Pools - -### Class Method -Use this class method to retrieve detailed information about various pools: -`Ekubo.api_connector.EkuboAPIConnector.get_pools()` - -### Data Structure -This endpoint retrieves a list of dictionaries, each containing comprehensive details about a pool. The structure of each dictionary in the list is as follows: -- **key_hash**: The unique identifier of the pool in hexadecimal format. - - Type: `str` -- **token0**: The address of the first token in the pool on the blockchain. - - Type: `str` -- **token1**: The address of the second token in the pool on the blockchain. - - Type: `str` -- **fee**: The fee associated with the pool transactions, in hexadecimal format. - - Type: `str` -- **tick_spacing**: The minimum granularity of price movements in the pool. - - Type: `int` -- **extension**: Additional information or features related to the pool, in hexadecimal format. - - Type: `str` -- **sqrt_ratio**: The square root of the current price ratio between token0 and token1, in hexadecimal format. - - Type: `str` -- **tick**: The current position of the pool in its price range. - - Type: `int` -- **liquidity**: The total liquidity available in the pool. - - Type: `str` -- **lastUpdate**: A dictionary containing details about the latest update event: - - **event_id**: Identifier of the last update event. - - Type: `str` - -### Return -- The method returns a list of dictionaries. Each dictionary provides detailed information about a specific pool. - -### Example of Returned Data -```json -[ - { - "key_hash": "0x34756a876aa3288b724dc967d43ee72d9b9cf390023775f0934305233767156", - "token0": "0xda114221cb83fa859dbdb4c44beeaa0bb37c7537ad5ae66fe5e0efd20e6eb3", - "token1": "0x49210ffc442172463f3177147c1aeaa36c51d152c1b0630f2364c300d4f48ee", - "fee": "0x20c49ba5e353f80000000000000000", - "tick_spacing": 1000, - "extension": "0x0", - "sqrt_ratio": "0x5c358a19c219f31e027d5ac98e8d5656", - "tick": -2042238, - "liquidity": "35067659406163360487", - "lastUpdate": { - "event_id": "2747597966999556" - } - } -] -``` \ No newline at end of file diff --git a/apps/web_app/order_books/ekubo/docs/token_lists.md b/apps/web_app/order_books/ekubo/docs/token_lists.md deleted file mode 100644 index bb0dd8a0..00000000 --- a/apps/web_app/order_books/ekubo/docs/token_lists.md +++ /dev/null @@ -1,22 +0,0 @@ -# Get List of Tokens - -### Class Method -Retrieve a list of tokens by using: -`Ekubo.api_connector.EkuboAPIConnector.get_list_tokens()` - -### Data Structure -This endpoint provides a list of tokens with detailed information. Each entry in the returned list is a dictionary containing: -- **name**: The name of the token, such as "Wrapped BTC". - - Type: `str` -- **symbol**: The abbreviated symbol of the token, such as "WBTC". - - Type: `int` -- **decimals**: The number of decimal places the token is divided into. - - Type: `str` -- **l2_token_address**: The address of the token on layer 2, in hexadecimal format. - - Type: `str` -- **sort_order**: An integer specifying the display order relative to other tokens; lower numbers appear first. - - Type: `int` -- **total_supply**: The total supply of the token, if known. It can be 'None' if the total supply is not set or unlimited. - - Type: `str` -- **logo_url**: A URL to the token's logo image. - - Type: `str` \ No newline at end of file diff --git a/apps/web_app/order_books/ekubo/docs/token_prices.md b/apps/web_app/order_books/ekubo/docs/token_prices.md deleted file mode 100644 index 235e80c4..00000000 --- a/apps/web_app/order_books/ekubo/docs/token_prices.md +++ /dev/null @@ -1,26 +0,0 @@ -# Get Token Prices - -### Usage Example -To get the `quote_token`, first retrieve the list of tokens using `get_list_tokens()`, and then use a specific token's details from the list to query `get_token_prices(quote_token)` for its current prices. - -### Class Method -Use this class method to retrieve current token prices: -`Ekubo.api_connector.EkuboAPIConnector.get_token_prices(quote_token)` -`Quote token` can be fetched from `get_list_tokens` method. This field is `l2_token_address` - -### Data Structure -This endpoint returns the current price of a token in terms of another token or currency. The response includes the following details: -- **timestamp**: Time at which the data was recorded, in milliseconds since the Unix epoch. - - Type: `int` -- **prices**: A list of dictionaries detailing the price of each token. - - Type: `list` of `dict` - -Each dictionary in the `prices` list contains: -- **token**: The address of the token on the blockchain. - - Type: `str` -- **price**: The current price of the token, expressed in another token or currency. - - Type: `str` -- **k_volume**: Volume of transactions for the token within a specified period, expressed in smallest unit counts (e.g., wei for Ethereum). - - Type: `str` - - diff --git a/apps/web_app/order_books/ekubo/histogram.py b/apps/web_app/order_books/ekubo/histogram.py deleted file mode 100644 index bf901d7b..00000000 --- a/apps/web_app/order_books/ekubo/histogram.py +++ /dev/null @@ -1,159 +0,0 @@ -import argparse -from typing import List - -import matplotlib.pyplot as plt - -from web_app.order_books.ekubo.main import EkuboOrderBook - -TOKEN_A = "0x49d36570d4e46f48e99674bd3fcc84644ddd6b96f7c741b1562b82f9e004dc7" # ETH -TOKEN_B = "0x53c91253bc9682c04929ca02ed00b3e423f6710d2ee7e0d5ebb06f3ecf368a8" # USDC - - -def fetch_order_book_and_current_price(token_a: str, token_b: str) -> tuple: - """ - Fetch the order book and current price for the given token pair. - :param token_a: Base token contract address - :param token_b: Quote token contract address - :return: Tuple containing the order book and current price - """ - order_book = EkuboOrderBook(token_a, token_b) - order_book.fetch_price_and_liquidity() - return order_book.get_order_book(), order_book.current_price - - -class Histogram: - """ - A class to create and display histograms for order book data. - - Attributes: - fig (plt.Figure): The matplotlib figure object. - ax (plt.Axes): The matplotlib axes object. - ask_prices (List[float]): List of ask prices. - ask_quantities (List[float]): List of ask quantities. - bid_prices (List[float]): List of bid prices. - bid_quantities (List[float]): List of bid quantities. - current_price (float): The current market price. - """ - - def __init__(self): - """Initialize the Histogram with an empty plot and collect order book data.""" - self.fig, self.ax = plt.subplots() - self._collect_data() - - def add_label(self, quantity_name: str, price_name: str) -> None: - """ - Add labels to the histogram. - - Args: - quantity_name (str): The name of the quantity. - price_name (str): The name of the price. - """ - self.ax.set_xlabel(f"Quantity ({quantity_name})") - self.ax.set_ylabel(f"Price ({price_name})") - self.ax.legend() - - def _collect_data(self) -> None: - """Collect order book data and convert to float.""" - data, current_price = fetch_order_book_and_current_price(TOKEN_A, TOKEN_B) - - ask_prices, ask_quantities = zip(*data["asks"]) - bid_prices, bid_quantities = zip(*data["bids"]) - - self.ask_prices = [float(price) for price in ask_prices] - self.ask_quantities = [float(quantity) for quantity in ask_quantities] - self.bid_prices = [float(price) for price in bid_prices] - self.bid_quantities = [float(quantity) for quantity in bid_quantities] - - self.current_price = float(current_price) - - def add_current_price_line(self, max_quantity: float = 0) -> None: - """ - Add a line representing the current price to the histogram. - - Args: - max_quantity (float): The maximum quantity to set the width of the line. Defaults to 0. - """ - self.ax.barh( - [self.current_price], - max_quantity, - color="black", - height=20, - label="Current Price", - ) - min_price = min(min(self.bid_prices), min(self.ask_prices), self.current_price) - max_price = max(max(self.bid_prices), max(self.ask_prices), self.current_price) - self.ax.set_ylim(min_price - 100, max_price + 100) # Adding some buffer - - def add_asks(self) -> None: - """Add ask prices and quantities to the histogram.""" - self.ax.barh( - self.ask_prices, self.ask_quantities, color="red", label="Asks", height=15 - ) - - def add_bids(self) -> None: - """Add bid prices and quantities to the histogram.""" - self.ax.barh( - self.bid_prices, self.bid_quantities, color="green", label="Bids", height=15 - ) - - def add_total_box_quantity( - self, quantities_name: str, sum_quantities: float - ) -> None: - """ - Add a text box displaying the total quantity. - - Args: - quantities_name (str): The name of the quantities. - sum_quantities (float): The sum of the quantities. - """ - total_quantity = round(sum_quantities, 4) - textstr = f"Total {quantities_name} Quantity: {total_quantity}" - props = dict(boxstyle="round", facecolor="wheat", alpha=0.5) - self.ax.text( - 0.95, - 1.05, - textstr, - transform=self.ax.transAxes, - fontsize=10, - verticalalignment="top", - horizontalalignment="right", - bbox=props, - ) - - def show_asks(self) -> None: - """Display the asks histogram with the current price line and total quantity.""" - self.add_current_price_line(max(self.ask_quantities)) - self.add_asks() - self.add_label("ETH", "USDC") - self.add_total_box_quantity("ETH", sum(self.ask_quantities)) - plt.show() - - def show_bids(self) -> None: - """Display the bids histogram with the current price line and total quantity.""" - self.add_current_price_line(max(self.bid_quantities)) - self.add_bids() - self.add_label("USDC", "ETH") - self.add_total_box_quantity("USDC", sum(self.bid_quantities)) - plt.show() - - -def main(): - parser = argparse.ArgumentParser(description="Display order book histograms.") - parser.add_argument( - "--type", - choices=["asks", "bids"], - required=True, - help="Type of histogram to display: 'asks' or 'bids'.", - ) - - args = parser.parse_args() - - histogram = Histogram() - if args.type == "asks": - histogram.show_asks() - elif args.type == "bids": - histogram.show_bids() - - -if __name__ == "__main__": - main() diff --git a/apps/web_app/order_books/ekubo/main.py b/apps/web_app/order_books/ekubo/main.py deleted file mode 100644 index 206df4b3..00000000 --- a/apps/web_app/order_books/ekubo/main.py +++ /dev/null @@ -1,244 +0,0 @@ -import logging -from decimal import Decimal, getcontext - -import pandas as pd -from order_books.abstractions import OrderBookBase -from order_books.constants import TOKEN_MAPPING - -from web_app.order_books.ekubo.api_connector import EkuboAPIConnector - -getcontext().prec = 18 - - -class EkuboOrderBook(OrderBookBase): - DEX = "Ekubo" - - def __init__(self, token_a: str, token_b: str) -> None: - """ - Initialize the EkuboOrderBook object. - :param token_a: BaseToken contract address - :param token_b: QuoteToken contract address - """ - super().__init__(token_a, token_b) - self.connector = EkuboAPIConnector() - - def set_current_price(self) -> str: - """ - Get the current price of the pair. - :return: str - The current price of the pair. - """ - price_data = self.connector.get_pair_price(self.token_a, self.token_b) - self.current_price = Decimal(price_data.get("price", "0")) - - def fetch_price_and_liquidity(self) -> None: - """ - Fetch the current price and liquidity of the pair from the Ekubo API. - """ - # Get pool liquidity - pools = self.connector.get_pools() - df = pd.DataFrame(pools) - # filter pool data by token_a and token_b - pool_df = df.loc[ - (df["token0"] == self.token_a) & (df["token1"] == self.token_b) - ] - - # set current price - self.set_current_price() - for index, row in list(pool_df.iterrows()): - key_hash = row["key_hash"] - # Fetch pool liquidity data - pool_liquidity = int(row["liquidity"]) - self.block = row["lastUpdate"]["event_id"] - - liquidity_response = self.connector.get_pool_liquidity(key_hash) - liquidity_data = liquidity_response["data"] - liquidity_data = sorted(liquidity_data, key=lambda x: x["tick"]) - - self._calculate_order_book( - liquidity_data, - pool_liquidity, - row, - ) - - def _calculate_order_book( - self, - liquidity_data: list, - pool_liquidity: int, - row: pd.Series, - ) -> None: - """ - Calculate the order book based on the liquidity data. - :param liquidity_data: list - List of liquidity data - :param pool_liquidity: pool liquidity - :param row: pd.Series - Pool data - """ - min_price, max_price = self.calculate_price_range() - if not liquidity_data: - return - - self.add_asks(liquidity_data, row) - self.add_bids(liquidity_data, row) - - # Filter asks and bids by price range - self.asks = [ - (price, supply) - for price, supply in self.asks - if min_price < price < max_price - ] - self.bids = [ - (price, supply) - for price, supply in self.bids - if min_price < price < max_price - ] - - def add_asks(self, liquidity_data: list[dict], row: pd.Series) -> None: - """ - Add `asks` to the order book. - :param liquidity_data: list of dict with tick and net_liquidity_delta_diff - :param row: pool row data - """ - ask_ticks = [i for i in liquidity_data if i["tick"] >= row["tick"]] - if not ask_ticks: - return - - glob_liq = Decimal(row["liquidity"]) - - # Calculate for current tick (loops start with the next one) - next_tick = ask_ticks[0]["tick"] - prev_tick = next_tick - row["tick_spacing"] - - prev_sqrt = self._get_pure_sqrt_ratio(prev_tick) - next_sqrt = self._get_pure_sqrt_ratio(next_tick) - - supply = abs( - ((glob_liq / prev_sqrt) - (glob_liq / next_sqrt)) - / 10**self.token_a_decimal - ) - price = self.tick_to_price(prev_tick) - self.asks.append((price, supply)) - - for index, tick in enumerate(ask_ticks): - if index == 0: - continue - glob_liq += Decimal(ask_ticks[index - 1]["net_liquidity_delta_diff"]) - prev_tick = Decimal(ask_ticks[index - 1]["tick"]) - - curr_tick = Decimal(tick["tick"]) - - prev_sqrt = self._get_pure_sqrt_ratio(prev_tick) - next_sqrt = self._get_pure_sqrt_ratio(curr_tick) - - supply = abs( - ((glob_liq / prev_sqrt) - (glob_liq / next_sqrt)) - / 10**self.token_a_decimal - ) - price = self.tick_to_price(prev_tick) - self.asks.append((price, supply)) - - def add_bids(self, liquidity_data: list[dict], row: pd.Series) -> None: - """ - Add `bids` to the order book. - :param liquidity_data: liquidity data list of dict with tick and net_liquidity_delta_diff - :param row: pool row data - """ - bid_ticks = [i for i in liquidity_data if i["tick"] <= row["tick"]][::-1] - if not bid_ticks: - return - - glob_liq = Decimal(row["liquidity"]) - - next_tick = bid_ticks[0]["tick"] - prev_tick = next_tick + row["tick_spacing"] - - prev_sqrt = self._get_pure_sqrt_ratio(prev_tick) - next_sqrt = self._get_pure_sqrt_ratio(next_tick) - - supply = abs( - ((glob_liq * prev_sqrt) - (glob_liq * next_sqrt)) - / 10**self.token_b_decimal - ) - price = self.tick_to_price(prev_tick) - self.bids.append((price, supply)) - - for index, tick in enumerate(bid_ticks): - if index == 0: - continue - glob_liq -= Decimal(bid_ticks[index - 1]["net_liquidity_delta_diff"]) - prev_tick = Decimal(bid_ticks[index - 1]["tick"]) - curr_tick = Decimal(tick["tick"]) - - prev_sqrt = self._get_pure_sqrt_ratio(prev_tick) - next_sqrt = self._get_pure_sqrt_ratio(curr_tick) - - supply = ( - abs(((glob_liq * prev_sqrt) - (glob_liq * next_sqrt))) - / 10**self.token_b_decimal - ) - price = self.tick_to_price(prev_tick) - self.bids.append((price, supply)) - - def _get_pure_sqrt_ratio(self, tick: Decimal) -> Decimal: - """ - Get the square root ratio based on the tick. - :param tick: tick value - :return: square root ratio - """ - return Decimal("1.000001").sqrt() ** tick - - @staticmethod - def sort_ticks_by_asks_and_bids( - sorted_liquidity_data: list, current_tick: int - ) -> tuple[list, list]: - """ - Sort tick by ask and bid - :param sorted_liquidity_data: list - List of sorted liquidity data - :param current_tick: int - Current tick - :return: list - List of sorted liquidity data - """ - sorted_liquidity_data = sorted(sorted_liquidity_data, key=lambda x: x["tick"]) - ask_data, bid_data = [], [] - for sorted_data in sorted_liquidity_data: - if sorted_data["tick"] > current_tick: - ask_data.append(sorted_data) - else: - bid_data.append(sorted_data) - return ask_data, bid_data - - def calculate_liquidity_amount( - self, tick: Decimal, liquidity_pair_total: Decimal - ) -> Decimal: - """ - Calculate the liquidity amount based on the liquidity delta and sqrt ratio. - :param tick: Decimal - The sqrt ratio. - :param liquidity_pair_total: Decimal - The liquidity pair total. - :return: Decimal - The liquidity amount. - """ - sqrt_ratio = self.get_sqrt_ratio(tick) - liquidity_delta = liquidity_pair_total / (sqrt_ratio / Decimal(2**128)) - return liquidity_delta / 10**self.token_a_decimal - - def tick_to_price(self, tick: Decimal) -> Decimal: - """ - Convert tick to price. - :param tick: tick value - :return: price by tick - """ - sqrt_ratio = self.get_sqrt_ratio(tick) - # calculate price by formula price = (sqrt_ratio / (2 ** 128)) ** 2 * 10 ** (token_a_decimal - token_b_decimal) - price = ((sqrt_ratio / (Decimal(2) ** 128)) ** 2) * 10 ** ( - self.token_a_decimal - self.token_b_decimal - ) - return price - - -def debug_code() -> None: - """ - This function is used to test the EkuboOrderBook class. - """ - token_a = "0x49d36570d4e46f48e99674bd3fcc84644ddd6b96f7c741b1562b82f9e004dc7" # ETH - token_b = ( - "0x53c91253bc9682c04929ca02ed00b3e423f6710d2ee7e0d5ebb06f3ecf368a8" # USDC - ) - order_book = EkuboOrderBook(token_a, token_b) - order_book.fetch_price_and_liquidity() - print(order_book.get_order_book(), "\n") diff --git a/apps/web_app/order_books/haiko/README.md b/apps/web_app/order_books/haiko/README.md deleted file mode 100644 index 7d0443f7..00000000 --- a/apps/web_app/order_books/haiko/README.md +++ /dev/null @@ -1,23 +0,0 @@ -# Haiko - -This folder contains interactions with Haiko API, -[BlastAPI](https://docs.blastapi.io/blast-documentation/apis-documentation/core-api/starknet) and -[Haiko Smart Contract](https://starkscan.co/contract/0x0038925b0bcf4dce081042ca26a96300d9e181b910328db54a6c89e5451503f5) on Starknet. - -# Endpoints Documentation: -- [Get Haiko Markets](docs/markets.md) - Information about markets for different tokens. -- [Get Supported Tokens](docs/tokens.md) - All supported tokens on Haiko. -- [Get Block Information](docs/block_info.md) - Retrieve information about latest Haiko block. -- [Get USD Prices](docs/prices.md) - Retrieve USD prices for tokens on Haiko. - -# How to run -In this folder: `haiko` run next command -```bash -python main.py -``` - -# How to get report about supported tokens -In this folder: `haiko` run next command and check `reports` folder to see the report. -```bash -python report.py -``` diff --git a/apps/web_app/order_books/haiko/__init__.py b/apps/web_app/order_books/haiko/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/apps/web_app/order_books/haiko/api_connector.py b/apps/web_app/order_books/haiko/api_connector.py deleted file mode 100644 index 34e2a1d3..00000000 --- a/apps/web_app/order_books/haiko/api_connector.py +++ /dev/null @@ -1,148 +0,0 @@ -from web_app.utils.abstractions import AbstractionAPIConnector - - -class HaikoAPIConnector(AbstractionAPIConnector): - API_URL = "https://app.haiko.xyz/api/v1" - - @classmethod - def get_supported_tokens(cls, existing_only: bool = True) -> list[dict]: - """ - Get all tokens supported by Haiko. - :param existing_only: If True, return only tokens that are currently available on Haiko. - :return: List of all tokens supported by Haiko. - The response list structure is as follows: - [ - { - "address": "0x49d36570d4e46f48e99674bd3fcc84644ddd6b96f7c741b1562b82f9e004dc7", - "name": "Ether", - "symbol": "ETH", - "decimals": 18, - "rank": 7, - "coingeckoAddress": "0x049d36570d4e46f48e99674bd3fcc84644ddd6b96f7c741b1562b82f9e004dc7" - } - ] - """ - if not isinstance(existing_only, bool): - raise ValueError("Existing only parameter must be a bool") - endpoint = f"/tokens?network=mainnet&existingOnly={existing_only}" - return cls.send_get_request(endpoint) # type: ignore - - @classmethod - def get_market_depth(cls, market_id: str) -> list[dict]: - """ - Get the market depth for a specific market. - :param market_id: The market ID in hexadecimal. - :return: List of market depth. - The response list structure is as follows: - [ - { - "price": "1.2072265814306946", - "liquidityCumulative": "4231256547876" - }, - ... - ] - """ - endpoint = f"/depth?network=mainnet&id={market_id}" - return cls.send_get_request(endpoint) # type: ignore - - @classmethod - def get_pair_markets(cls, token0: str, token1: str) -> list[dict]: - """ - Get Haiko markets for provided token pair. - :return: List of Haiko markets for a token pair. - The response list structure is as follows: - [ - { - "marketId": "0x581defc9a9b4e77fcb3ec274983e18ea30115727f7647f2eb1d23858292d873", - "baseToken": { - "address": "0x4718f5a0fc34cc1af16a1cdee98ffb20c31f5cd61d6ab07201858f4287c938d", - "name": "Starknet Token", - "symbol": "STRK", - "decimals": 18, - "rank": 10 - }, - "quoteToken": { - "address": "0x53c91253bc9682c04929ca02ed00b3e423f6710d2ee7e0d5ebb06f3ecf368a8", - "name": "USD Coin", - "symbol": "USDC", - "decimals": 6, - "rank": 2 - }, - "width": 200, - "strategy": { - "address": "0x0", - "name": null, - "symbol": null, - "version": null - }, - "swapFeeRate": 100, - "feeController": "0x0", - "controller": "0x0", - "currLimit": -2744284, - "currSqrtPrice": "1.0987386319915646", - "currPrice": "1.2072265814306946", - "tvl": "580.7339", - "feeApy": 0.05661423233103435 - } - ] - """ - endpoint = f"/markets-by-pair?network=mainnet&token0={token0}&token1={token1}" - return cls.send_get_request(endpoint) # type: ignore - - @classmethod - def get_usd_prices(cls, token_a_name, token_b_name) -> dict: - """ - Get USD prices for the provided token pair. - :return: USD prices for the provided token pair. - The response dictionary structure is as follows: - { - "ETH": 3761.71, - "USDC": 0.999003 - } - """ - endpoint = f"/usd-prices?network=mainnet&tokens={token_a_name},{token_b_name}" - return cls.send_get_request(endpoint) # type: ignore - - -class HaikoBlastAPIConnector(AbstractionAPIConnector): - API_URL = "https://starknet-mainnet.blastapi.io" - PROJECT_ID = "a419bd5a-ec9e-40a7-93a4-d16467fb79b3" - - @classmethod - def _post_request_builder(cls, call_method: str, params: dict) -> dict: - """ - Build request body for Blast API POST request from common base. - :param call_method: BlastAPI method to call. - :param params: Parameters for the method. - :return: Response from BlastAPI. - """ - if not isinstance(call_method, str) or not isinstance(params, dict): - raise ValueError("Call method must be a string and params must be a dict.") - body = {"jsonrpc": "2.0", "method": call_method, "params": params, "id": 0} - endpoint = f"/{cls.PROJECT_ID}" - return cls.send_post_request(endpoint, json=body) - - @classmethod - def get_block_info(cls) -> dict: - """ - Get information about the latest block. - :return: Information about the latest block. - The response dictionary structure is as follows: - { - "jsonrpc": "2.0", - "result": { - "status": "ACCEPTED_ON_L2", - "block_hash": "0x18ec1a3931bb5a286f801a950e1153bd427d6d3811591cc01e6f074615a1f76", - "parent_hash": "0x413229e9996b3025feb6b276a33249fb0ff0f92d8aeea284deb35ea4093dea2", - "block_number": 4503, - "new_root": "0xc95a878188acf408e285027bd5e7674a88529b8c65ef6c1999b3569aea8bc8", - "timestamp": 1661246333, - "sequencer_address": "0x5dcd266a80b8a5f29f04d779c6b166b80150c24f2180a75e82427242dab20a9", - "transactions": ["0x6a19b22f4fe4018d4d60ff844770a5459534d0a69f850f3c9cdcf70a132df94", ...], - }, - "id": 0 - } - """ - return cls._post_request_builder( - "starknet_getBlockWithTxHashes", params={"block_id": "latest"} - ) diff --git a/apps/web_app/order_books/haiko/docs/block_info.md b/apps/web_app/order_books/haiko/docs/block_info.md deleted file mode 100644 index 45135af6..00000000 --- a/apps/web_app/order_books/haiko/docs/block_info.md +++ /dev/null @@ -1,29 +0,0 @@ -# Get Block Information - -### Class Method -Use this class method to retrieve information about block using Haiko project id: -`haiko.api_connector.HaikoBlastAPIConnector.get_block_info()` - -### Data Structure -This endpoint retrieves information about the latest Haiko block. The response includes an object containing the following details: block number and timestamp. - -### Return -- Returns dictionary with id, jsonrpc version and block information. - -### Example of Returned Data -```json -{ - "jsonrpc": "2.0", - "result": { - "status": "ACCEPTED_ON_L2", - "block_hash": "0x18ec1a3931bb5a286f801a950e1153bd427d6d3811591cc01e6f074615a1f76", - "parent_hash": "0x413229e9996b3025feb6b276a33249fb0ff0f92d8aeea284deb35ea4093dea2", - "block_number": 4503, - "new_root": "0xc95a878188acf408e285027bd5e7674a88529b8c65ef6c1999b3569aea8bc8", - "timestamp": 1661246333, - "sequencer_address": "0x5dcd266a80b8a5f29f04d779c6b166b80150c24f2180a75e82427242dab20a9", - "transactions": ["0x6a19b22f4fe4018d4d60ff844770a5459534d0a69f850f3c9cdcf70a132df94", "0x5fb5b63f0226ef426c81168d0235269398b63aa145ca6a3c47294caa691cfdc"] - }, - "id": 0 -} -``` diff --git a/apps/web_app/order_books/haiko/docs/depth.md b/apps/web_app/order_books/haiko/docs/depth.md deleted file mode 100644 index 503b4a75..00000000 --- a/apps/web_app/order_books/haiko/docs/depth.md +++ /dev/null @@ -1,31 +0,0 @@ -### Get depth of the market - -### Class Method -Use this class method to retrieve market depth: -`haiko.api_connector.HaikoAPIConnector.get_market_depth(market_id)` - -### Data Structure -This endpoint retrieves the liquidity for each price for a specified market. -The response includes an array of objects, each detailing liquidity information in a pool for a price. -Each object in the array contains: -- **price**: The index of the tick within the pool's range. - - Type: `str` -- **liquidityCumulative**: The current liquidity in a pool. - - Type: `str` - -### Parameters -- **market_id**: The identifier of the market in hexadecimal format. - - Type: `str` - -### Return -- Returns an array of dictionaries, each providing details on the liquidity state for each price. - -### Example of Returned Data -```json -[ - { - "price": "3750.5342", - "liquidityCumulative": "534939583228319557" - } -] -``` diff --git a/apps/web_app/order_books/haiko/docs/markets.md b/apps/web_app/order_books/haiko/docs/markets.md deleted file mode 100644 index b881d651..00000000 --- a/apps/web_app/order_books/haiko/docs/markets.md +++ /dev/null @@ -1,89 +0,0 @@ -# Get Markets - -### Class Method -Use this class method to retrieve detailed information about markets for tokens: -`haiko.api_connector.HaikoAPIConnector.get_token_markets(token0, token1)` - -### Data Structure -This endpoint retrieves a list of dictionaries, each containing details about a market. The structure of each dictionary in the list is as follows(only used and known info): -- **marketId**: The unique identifier of the market in hexadecimal format. - - Type: `str` -- **baseToken**: A dictionary containing details about the base token: - - **address**: The address of the base token in hexadecimal format. - - Type: `str` - - **symbol**: The symbol of the base token. - - Type: `str` - - **decimals**: The number of decimal places used to represent the base token. - - Type: `int` - - **name**: The name of the base token. - - Type: `str` - - **rank**: The rank of the base token on the market. - - Type: `int` -- **quoteToken**: A dictionary containing details about the quote token: - - **address**: The address of the quote token in hexadecimal format. - - Type: `str` - - **symbol**: The symbol of the quote token. - - Type: `str` - - **decimals**: The number of decimal places used to represent the quote token. - - Type: `int` - - **name**: The name of the quote token. - - Type: `str` - - **rank**: The rank of the quote token on the market. - - Type: `int` -- **width** The width of the tick on the market. - - Type: `int` -- **currLimit**: The current tick on the market. - - Type: `int` -- **currPrice**: The current price on the market. - - Type: `str` -- **currSqrtPrice**: The square root of the current price on the market. - - Type: `str` -- **tvl**: The total value locked in the market. - - Type: `str` - -### Parameters -- **token0**: The address of the base token in the hexadecimal format. - - Type: `str` -- **token1**: The address of the quote token in the hexadecimal format. - - Type: `str` - -### Return -- The method returns a list of dictionaries. Each dictionary provides detailed information about a specific market. - -### Example of Returned Data -```json -[ - { - "marketId": "0x581defc9a9b4e77fcb3ec274983e18ea30115727f7647f2eb1d23858292d873", - "baseToken": { - "address": "0x4718f5a0fc34cc1af16a1cdee98ffb20c31f5cd61d6ab07201858f4287c938d", - "name": "Starknet Token", - "symbol": "STRK", - "decimals": 18, - "rank": 10 - }, - "quoteToken": { - "address": "0x53c91253bc9682c04929ca02ed00b3e423f6710d2ee7e0d5ebb06f3ecf368a8", - "name": "USD Coin", - "symbol": "USDC", - "decimals": 6, - "rank": 2 - }, - "width": 200, - "strategy": { - "address": "0x0", - "name": null, - "symbol": null, - "version": null - }, - "swapFeeRate": 100, - "feeController": "0x0", - "controller": "0x0", - "currLimit": -2744284, - "currSqrtPrice": "1.0987386319915646", - "currPrice": "1.2072265814306946", - "tvl": "580.7339", - "feeApy": 0.05661423233103435 - } -] -``` diff --git a/apps/web_app/order_books/haiko/docs/prices.md b/apps/web_app/order_books/haiko/docs/prices.md deleted file mode 100644 index d90acd93..00000000 --- a/apps/web_app/order_books/haiko/docs/prices.md +++ /dev/null @@ -1,29 +0,0 @@ -# Get USD Prices - -### Class Method -Use this class method to retrieve prices of tokens: -`haiko.api_connector.HaikoAPIConnector.get_usd_prices(token0_name, token1_name)` - -### Data Structure -This endpoint retrieves a dictionary containing the prices of tokens. The keys of the dictionary are: -- **token0_name**: The price of the first token. - - Type: `str` -- **token1_name**: The price of the second token. - - Type: `str` - -### Parameters -- **token0_name**: The name of the base token. - - Type: `str` -- **token1_name**: The name of the quote token. - - Type: `str` - -### Return -- The method returns a dictionary containing the prices of the specified tokens. - -### Example of Returned Data -```json -{ - "ETH": "232.54", - "USDC": "0.867987" -} -``` diff --git a/apps/web_app/order_books/haiko/docs/tokens.md b/apps/web_app/order_books/haiko/docs/tokens.md deleted file mode 100644 index 9ebedd29..00000000 --- a/apps/web_app/order_books/haiko/docs/tokens.md +++ /dev/null @@ -1,35 +0,0 @@ -# Get List of Supported Tokens - -### Class Method -Retrieve a list of tokens by using: -`haiko.api_connector.HaikoAPIConnector.get_supported_tokens()` - -### Data Structure -This endpoint provides a list of tokens with detailed information. Each entry in the returned list is a dictionary containing: -- **address**: The address of the token on layer 2, in hexadecimal format. - - Type: `str` -- **name**: The name of the token, such as "Wrapped BTC". - - Type: `str` -- **symbol**: The abbreviated symbol of the token, such as "WBTC". - - Type: `int` -- **decimals**: The number of decimal places the token is divided into. - - Type: `str` -- **rank**: The rank of the token on the market. - - Type: `int` - -### Return -The method returns a list of dictionaries, each containing detailed information about a token. - -### Example of Returned Data -```json -[ - { - "address": "0x49d36570d4e46f48e99674bd3fcc84644ddd6b96f7c741b1562b82f9e004dc7", - "name": "Ether", - "symbol": "ETH", - "decimals": 18, - "rank": 7, - "coingeckoAddress": "0x049d36570d4e46f48e99674bd3fcc84644ddd6b96f7c741b1562b82f9e004dc7" - } -] -``` diff --git a/apps/web_app/order_books/haiko/logger.py b/apps/web_app/order_books/haiko/logger.py deleted file mode 100644 index 2f8bf5d4..00000000 --- a/apps/web_app/order_books/haiko/logger.py +++ /dev/null @@ -1,35 +0,0 @@ -import logging -from datetime import datetime -from pathlib import Path - - -def get_logger(path: str, echo: bool = False): - """ - Configure and get logger for haiko order book - :param path: path to log file - :param echo: write logs in terminal if set to True - :return: Configured logger - """ - path_dir = Path(path) - path_dir.mkdir(parents=True, exist_ok=True) - logger = logging.getLogger("Haiko_Logger") - logger.setLevel(logging.DEBUG) - log_path = path_dir / datetime.now().strftime("order_book_%Y%m%d_%H%M%S.log") - file_handler = logging.FileHandler(log_path) - file_handler.setLevel(logging.DEBUG) - - formatter = logging.Formatter( - "%(asctime)s - %(name)s - %(levelname)s - %(message)s" - ) - file_handler.setFormatter(formatter) - logger.addHandler(file_handler) - - if not echo: - return logger - - console_handler = logging.StreamHandler() - console_handler.setLevel(logging.DEBUG) - console_handler.setFormatter(formatter) - logger.addHandler(console_handler) - - return logger diff --git a/apps/web_app/order_books/haiko/main.py b/apps/web_app/order_books/haiko/main.py deleted file mode 100644 index f66d8616..00000000 --- a/apps/web_app/order_books/haiko/main.py +++ /dev/null @@ -1,272 +0,0 @@ -from decimal import Decimal - -from web_app.order_books.abstractions import OrderBookBase -from web_app.order_books.constants import TOKEN_MAPPING -from web_app.order_books.haiko.api_connector import ( - HaikoAPIConnector, - HaikoBlastAPIConnector, -) -from web_app.order_books.haiko.logger import get_logger - - -class HaikoOrderBook(OrderBookBase): - DEX = "Haiko" - - def __init__(self, token_a, token_b, apply_filtering: bool = False): - """ - Initialize the HaikoOrderBook object. - :param token_a: baseToken hexadecimal address - :param token_b: quoteToken hexadecimal address - :param apply_filtering: bool - If True apply min and max price filtering to the order book data - """ - super().__init__(token_a, token_b) - self.haiko_connector = HaikoAPIConnector() - self.blast_connector = HaikoBlastAPIConnector() - self.apply_filtering = apply_filtering - self.logger = get_logger("./logs", echo=True) - - self.token_a_price = Decimal(0) - self.token_b_price = Decimal(0) - - self._decimals_diff = 10 ** ( - self.token_a_decimal - self.token_b_decimal or self.token_a_decimal - ) - self._check_tokens_supported() - self._set_usd_prices() - - def _get_valid_tokens_addresses(self) -> tuple[str, str]: - """ - Get tokens' addresses without leading zeros. - :return: tuple - The token addresses tuple without leading zeros - """ - if not isinstance(self.token_a, str) or not isinstance(self.token_b, str): - raise ValueError("Token addresses must be strings.") - return hex(int(self.token_a, base=16)), hex(int(self.token_b, base=16)) - - def _set_usd_prices(self) -> None: - """Set USD prices for tokens based on Haiko API.""" - token_a_info = TOKEN_MAPPING.get(self.token_a) - token_b_info = TOKEN_MAPPING.get(self.token_b) - if not token_a_info or not token_b_info: - raise ValueError("Information about tokens isn't available.") - token_a_name = token_a_info.name - token_b_name = token_b_info.name - prices = self.haiko_connector.get_usd_prices(token_a_name, token_b_name) - self.token_a_price = Decimal(prices.get(token_a_name, 0)) - self.token_b_price = Decimal(prices.get(token_b_name, 0)) - if self.token_a_price == 0 or self.token_b_price == 0: - raise RuntimeError("Prices for tokens aren't available.") - - def _check_tokens_supported(self) -> None: - """Check if a pair of tokens is supported by Haiko""" - supported_tokens = self.haiko_connector.get_supported_tokens( - existing_only=False - ) - if isinstance(supported_tokens, dict) and supported_tokens.get("error"): - raise RuntimeError(f"Unexpected error from API: {supported_tokens}") - valid_tokens = self._get_valid_tokens_addresses() - supported_tokens_filtered = [ - token for token in supported_tokens if token["address"] in valid_tokens - ] - if len(supported_tokens_filtered) != 2: - raise ValueError("One of tokens isn't supported by Haiko") - - def set_current_price(self, current_price: Decimal) -> None: - """ - Set the current price based on the current tick. - :param current_price: Decimal - Current market price - """ - self.current_price = current_price - - def sort_asks_bids(self) -> None: - """Sort bids and asks data in correct order.""" - self.asks.sort(key=lambda ask: ask[0]) - self.bids.sort(key=lambda bid: bid[0], reverse=True) - - def fetch_price_and_liquidity(self) -> None: - tokens_markets = self._filter_markets_data( - self.haiko_connector.get_pair_markets(self.token_a, self.token_b) - ) - latest_block_info = self.blast_connector.get_block_info() - if latest_block_info.get("error"): - raise RuntimeError(f"Blast-api returned an error: {latest_block_info}") - - self.block = latest_block_info["result"]["block_number"] - self.timestamp = latest_block_info["result"]["timestamp"] - - if not tokens_markets: - message = "Markets for this pair isn't available for now" - self.logger.critical(f"Pair of tokens: {self.token_a}-{self.token_b}") - self.logger.critical(f"{message}\n") - return - - for market in tokens_markets: - market_id = market["marketId"] - market_depth_list = self.haiko_connector.get_market_depth(market_id) - if not market_depth_list: - self.logger.info(f"Market depth for market {market_id} is empty.") - continue - self._calculate_order_book(market_depth_list, Decimal(market["currPrice"])) - - self.sort_asks_bids() - self.current_price = max(tokens_markets, key=lambda x: Decimal(x["tvl"]))[ - "currPrice" - ] - - def _calculate_order_book( - self, market_ticks_liquidity: list, current_price: Decimal - ) -> None: - self.set_current_price(current_price) - price_range = self.calculate_price_range() - asks, bids = [], [] - for tick_info in market_ticks_liquidity: - tick_info["price"] = Decimal(tick_info["price"]) - tick_info["liquidityCumulative"] = Decimal(tick_info["liquidityCumulative"]) - if tick_info["price"] >= current_price: - asks.append(tick_info) - else: - bids.append(tick_info) - - if not asks or not bids: - return - - bids.sort(key=lambda x: x["price"], reverse=True) - asks.sort(key=lambda x: x["price"]) - self.add_bids(bids, price_range) - self.add_asks(asks, bids[0]["liquidityCumulative"], price_range) - - def add_asks( - self, market_asks: list[dict], pool_liquidity: Decimal, price_range: tuple - ) -> None: - """ - Add `asks` to the order book. - :param market_asks: list of dictionaries with price and liquidityCumulative - :param pool_liquidity: current liquidity in a pool - :param price_range: tuple of Decimal - minimal and maximal acceptable prices - """ - if not market_asks: - return - local_asks = [] - x = self._get_token_amount( - pool_liquidity, - self.current_price.sqrt(), - market_asks[0]["price"].sqrt(), - ) - local_asks.append((self.current_price, x)) - for index, ask in enumerate(market_asks): - if index == 0: - continue - current_price = Decimal(market_asks[index - 1]["price"]) - x = self._get_token_amount( - Decimal(market_asks[index - 1]["liquidityCumulative"]), - current_price.sqrt(), - Decimal(market_asks[index]["price"]).sqrt(), - ) - local_asks.append((current_price, x)) - if self.apply_filtering: - self.asks.extend( - [ask for ask in local_asks if price_range[0] < ask[0] < price_range[1]] - ) - return - self.asks.extend(local_asks) - - def add_bids(self, market_bids: list[dict], price_range: tuple) -> None: - """ - Add `bids` to the order book. - :param market_bids: list of dictionaries with price and liquidityCumulative - :param price_range: tuple of Decimal - minimal and maximal acceptable prices - """ - if not market_bids: - return - local_bids = [] - prev_price = Decimal(market_bids[0]["price"]) - y = self._get_token_amount( - Decimal(market_bids[0]["liquidityCumulative"]), - self.current_price.sqrt(), - prev_price.sqrt(), - is_ask=False, - ) - local_bids.append((prev_price, y)) - for index, bid in enumerate(market_bids[::-1]): - if index == 0: - continue - current_price = Decimal(market_bids[index - 1]["price"]) - y = self._get_token_amount( - Decimal(market_bids[index]["liquidityCumulative"]), - current_price.sqrt(), - Decimal(market_bids[index]["price"]).sqrt(), - is_ask=False, - ) - local_bids.append((current_price, y)) - if self.apply_filtering: - self.bids.extend( - [bid for bid in local_bids if price_range[0] < bid[0] < price_range[1]] - ) - return - self.bids.extend(local_bids) - - def _get_token_amount( - self, - current_liq: Decimal, - current_sqrt: Decimal, - next_sqrt: Decimal, - is_ask: bool = True, - ) -> Decimal: - """ - Calculate token amount based on liquidity data and current data processed(asks/bids). - :param current_liq: Decimal - Current price liquidity - :param current_sqrt: Decimal - Current square root of a price - :param next_sqrt: Decimal - Next square root of a price - :param is_ask: bool - True if an ask data - :return: Decimal - token amount - """ - if is_ask and (current_sqrt == 0 or next_sqrt == 0): - raise ValueError("Square root of prices for asks can't be zero.") - if not is_ask and next_sqrt == 0: - return abs(current_liq / current_sqrt) / self._decimals_diff - amount = abs(current_liq / next_sqrt - current_liq / current_sqrt) - return amount / self._decimals_diff - - def calculate_liquidity_amount(self, tick, liquidity_pair_total) -> Decimal: - sqrt_ratio = self.get_sqrt_ratio(tick) - liquidity_delta = liquidity_pair_total / (sqrt_ratio / Decimal(2**128)) - return liquidity_delta / 10**self.token_a_decimal - - def tick_to_price(self, tick: Decimal) -> Decimal: - return Decimal("1.00001") ** tick * ( - 10 ** (self.token_a_decimal - self.token_b_decimal) - ) - - def _filter_markets_data(self, all_markets_data: list) -> list: - """ - Filter markets for actual token pair. - :param all_markets_data: all supported markets provided bt Haiko - :return: list of markets data for actual token pair - """ - token_a_valid, token_b_valid = self._get_valid_tokens_addresses() - return list( - filter( - lambda market: market["baseToken"]["address"] == token_a_valid - and market["quoteToken"]["address"] == token_b_valid, - all_markets_data, - ) - ) - - -if __name__ == "__main__": - # token_0 = "0x4718f5a0fc34cc1af16a1cdee98ffb20c31f5cd61d6ab07201858f4287c938d" # STRK - # token_1 = "0x49d36570d4e46f48e99674bd3fcc84644ddd6b96f7c741b1562b82f9e004dc7" # ETH - # token_0 = "0x49d36570d4e46f48e99674bd3fcc84644ddd6b96f7c741b1562b82f9e004dc7" # ETH - # token_1 = "0x53c91253bc9682c04929ca02ed00b3e423f6710d2ee7e0d5ebb06f3ecf368a8" # USDC - token_0 = ( - "0x042b8f0484674ca266ac5d08e4ac6a3fe65bd3129795def2dca5c34ecc5f96d2" # wstETH - ) - token_1 = "0x49d36570d4e46f48e99674bd3fcc84644ddd6b96f7c741b1562b82f9e004dc7" # ETH - # token_0 = "0x4718f5a0fc34cc1af16a1cdee98ffb20c31f5cd61d6ab07201858f4287c938d" # STRK - # token_1 = "0x53c91253bc9682c04929ca02ed00b3e423f6710d2ee7e0d5ebb06f3ecf368a8" # USDC - # token_0 = "0x07e2c010c0b381f347926d5a203da0335ef17aefee75a89292ef2b0f94924864" # wstETH - # token_1 = "0x49d36570d4e46f48e99674bd3fcc84644ddd6b96f7c741b1562b82f9e004dc7" - # token_1 = "0x068f5c6a61780768455de69077e07e89787839bf8166decfbf92b645209c0fb8" - order_book = HaikoOrderBook(token_0, token_1) - order_book.fetch_price_and_liquidity() - serialized_order_book = order_book.serialize() diff --git a/apps/web_app/order_books/haiko/report.py b/apps/web_app/order_books/haiko/report.py deleted file mode 100644 index dc76adcb..00000000 --- a/apps/web_app/order_books/haiko/report.py +++ /dev/null @@ -1,114 +0,0 @@ -import asyncio -import json -import logging -from datetime import datetime -from pathlib import Path - -from web_app.order_books.constants import TOKEN_MAPPING -from web_app.order_books.haiko.main import HaikoOrderBook - - -def filter_logs(logs_path: Path) -> None: - """ - Remove empty log files. - :param logs_path: Path to the directory with log files. - """ - for file in logs_path.iterdir(): - if file.stat().st_size == 0: - file.unlink() - - -def serialize_asks_bids(order_book: dict) -> None: - """ - Convert asks and bids to serializable format. - :param order_book: Order book data. - """ - order_book["asks"] = [[float(ask[0]), float(ask[1])] for ask in order_book["asks"]] - order_book["bids"] = [[float(bid[0]), float(bid[1])] for bid in order_book["bids"]] - - -def get_report() -> dict: - """ - Get report for all token pairs. - :return: Report with order books data. - The report structure is as follows: - { - "empty_pairs": [STRK-ETH, ...], - "ETH-USDC": { - "is_empty": False, - "order_book": { - "asks": [[price1, amount1], [price2, amount2], ...], - "bids": [[price1, amount1], [price2, amount2], ...] - } - }, - ... - } - """ - report = {"empty_pairs": []} - all_tokens = set(TOKEN_MAPPING.keys()) - - for base_token in TOKEN_MAPPING: - current_tokens = all_tokens - {base_token} - for quote_token in current_tokens: - try: - order_book = HaikoOrderBook(base_token, quote_token) - except ValueError: - logging.log( - logging.ERROR, f"Pair of tokens: {base_token}-{quote_token}" - ) - logging.log(logging.ERROR, "One of the tokens isn't supported by Haiko") - continue - except RuntimeError as e: - logging.log( - logging.ERROR, f"Pair of tokens: {base_token}-{quote_token}" - ) - logging.log(logging.ERROR, e) - continue - - try: - base_token_name = TOKEN_MAPPING[base_token].name - quote_token_name = TOKEN_MAPPING[quote_token].name - order_book.fetch_price_and_liquidity() - token_pair = f"{base_token_name}-{quote_token_name}" - report[token_pair] = {"is_empty": False, "order_book": {}} - entry = order_book.get_order_book() - serialize_asks_bids(entry) - report[token_pair]["order_book"] = entry - if not order_book.bids and not order_book.asks: - report[token_pair]["is_empty"] = True - report["empty_pairs"].append(token_pair) - except KeyError: - order_book.logger.error(f"Pair of tokens: {base_token}-{quote_token}") - order_book.logger.error( - "One of the tokens doesn't present in tokens mapping" - ) - except RuntimeError as e: - order_book.logger.error(f"Pair of tokens: {base_token}-{quote_token}") - order_book.logger.error(e) - except Exception as e: - order_book.logger.error(f"Pair of tokens: {base_token}-{quote_token}") - order_book.logger.error(f"Unexpected error: {e}") - - return report - - -def write_report(report: dict, path: str | Path) -> None: - """ - Write report to a json file. - :param report: Report data. - :param path: Path to the file. - """ - try: - with open(path, mode="w", encoding="UTF-8") as file: - json.dump(report, file, indent=4) - except IOError as e: - print(f"Error writing report: {e}") - - -if __name__ == "__main__": - report_data = get_report() - reports_dir = Path("./reports") - reports_dir.mkdir(parents=True, exist_ok=True) - report_path = reports_dir / datetime.now().strftime("report_%Y%m%d_%H%M%S.json") - filter_logs(reports_dir) - write_report(report_data, report_path) diff --git a/apps/web_app/utils/helpers.py b/apps/web_app/utils/helpers.py deleted file mode 100644 index a94e95ec..00000000 --- a/apps/web_app/utils/helpers.py +++ /dev/null @@ -1,76 +0,0 @@ -import logging -import os -from decimal import Decimal -from typing import Iterator, Union - -import google.cloud.storage -import pandas as pd -from utils.exceptions import TokenValidationError -from utils.settings import TOKEN_SETTINGS - - -class TokenValues: - """A class that holds all token values""" - - def __init__( - self, - values: dict[str, Union[bool, Decimal]] | None = None, - init_value: Decimal = Decimal("0"), - ) -> None: - if values: - self._validate_token_values(values) - self.values: dict[str, Decimal] = values - else: - self.values: dict[str, Decimal] = { - token: init_value for token in TOKEN_SETTINGS - } - - @staticmethod - def _validate_token_values(token_values: dict[str, Union[bool, Decimal]]) -> None: - """ - Validate's token_values keys - :param token_values: dict[str, Union[bool, Decimal]] - :return: None - """ - if set(token_values.keys()) != set(TOKEN_SETTINGS.keys()): - raise TokenValidationError( - "Token values keys do not match with TOKEN_SETTINGS keys" - ) - - -MAX_ROUNDING_ERRORS: TokenValues = TokenValues( - values={ - "ETH": Decimal("0.5e13"), - "wBTC": Decimal("1e2"), - "USDC": Decimal("1e4"), - "DAI": Decimal("1e16"), - "USDT": Decimal("1e4"), - "wstETH": Decimal("0.5e13"), - "LORDS": Decimal("0.5e13"), - "STRK": Decimal("0.5e13"), - }, -) - - -class Portfolio(TokenValues): - """A class that describes holdings of tokens.""" - - MAX_ROUNDING_ERRORS: TokenValues = MAX_ROUNDING_ERRORS - - def __init__(self) -> None: - super().__init__(init_value=Decimal("0")) - - -def get_symbol(address: str) -> str: - """ - Returns the symbol of a given address. - :param address: the address of the symbol - :return: str - """ - address_int = int(address, base=16) - - for symbol, settings in TOKEN_SETTINGS.items(): - if int(settings.address, base=16) == address_int: - return symbol - - raise KeyError(f"Address = {address} does not exist in the symbol table.") diff --git a/apps/web_app/utils/settings.py b/apps/web_app/utils/settings.py deleted file mode 100644 index 4ca849aa..00000000 --- a/apps/web_app/utils/settings.py +++ /dev/null @@ -1,53 +0,0 @@ -from dataclasses import dataclass -from decimal import Decimal - - -@dataclass -class TokenSettings: - symbol: str - decimal_factor: Decimal - address: str - - -TOKEN_SETTINGS: dict[str, TokenSettings] = { - "ETH": TokenSettings( - symbol="ETH", - decimal_factor=Decimal("1e18"), - address="0x049d36570d4e46f48e99674bd3fcc84644ddd6b96f7c741b1562b82f9e004dc7", - ), - "wBTC": TokenSettings( - symbol="wBTC", - decimal_factor=Decimal("1e8"), - address="0x03fe2b97c1fd336e750087d68b9b867997fd64a2661ff3ca5a7c771641e8e7ac", - ), - "USDC": TokenSettings( - symbol="USDC", - decimal_factor=Decimal("1e6"), - address="0x053c91253bc9682c04929ca02ed00b3e423f6710d2ee7e0d5ebb06f3ecf368a8", - ), - "DAI": TokenSettings( - symbol="DAI", - decimal_factor=Decimal("1e18"), - address="0x00da114221cb83fa859dbdb4c44beeaa0bb37c7537ad5ae66fe5e0efd20e6eb3", - ), - "USDT": TokenSettings( - symbol="USDT", - decimal_factor=Decimal("1e6"), - address="0x068f5c6a61780768455de69077e07e89787839bf8166decfbf92b645209c0fb8", - ), - "wstETH": TokenSettings( - symbol="wstETH", - decimal_factor=Decimal("1e18"), - address="0x042b8f0484674ca266ac5d08e4ac6a3fe65bd3129795def2dca5c34ecc5f96d2", - ), - "LORDS": TokenSettings( - symbol="LORDS", - decimal_factor=Decimal("1e18"), - address="0x0124aeb495b947201f5fac96fd1138e326ad86195b98df6dec9009158a533b49", - ), - "STRK": TokenSettings( - symbol="STRK", - decimal_factor=Decimal("1e18"), - address="0x4718f5a0fc34cc1af16a1cdee98ffb20c31f5cd61d6ab07201858f4287c938d", - ), -} diff --git a/apps/web_app/utils/state.py b/apps/web_app/utils/state.py deleted file mode 100644 index 4c1249f8..00000000 --- a/apps/web_app/utils/state.py +++ /dev/null @@ -1,153 +0,0 @@ -from abc import ABC -from collections import defaultdict -from dataclasses import dataclass -from decimal import Decimal - -from shared.types import Portfolio, TokenValues -from .settings import TOKEN_SETTINGS as BASE_TOKEN_SETTINGS -from .settings import TokenSettings as BaseTokenSettings - - -@dataclass -class SpecificTokenSettings: - collateral_factor: Decimal - debt_factor: Decimal - - -@dataclass -class TokenSettings(SpecificTokenSettings, BaseTokenSettings): - pass - - -LOAN_ENTITY_SPECIFIC_TOKEN_SETTINGS: dict[str, SpecificTokenSettings] = { - "ETH": SpecificTokenSettings( - collateral_factor=Decimal("1"), debt_factor=Decimal("1") - ), - "wBTC": SpecificTokenSettings( - collateral_factor=Decimal("1"), debt_factor=Decimal("1") - ), - "USDC": SpecificTokenSettings( - collateral_factor=Decimal("1"), debt_factor=Decimal("1") - ), - "DAI": SpecificTokenSettings( - collateral_factor=Decimal("1"), debt_factor=Decimal("1") - ), - "USDT": SpecificTokenSettings( - collateral_factor=Decimal("1"), debt_factor=Decimal("1") - ), - "wstETH": SpecificTokenSettings( - collateral_factor=Decimal("1"), debt_factor=Decimal("1") - ), - "LORDS": SpecificTokenSettings( - collateral_factor=Decimal("1"), debt_factor=Decimal("1") - ), - "STRK": SpecificTokenSettings( - collateral_factor=Decimal("1"), debt_factor=Decimal("1") - ), -} -TOKEN_SETTINGS: dict[str, TokenSettings] = { - token: TokenSettings( - symbol=BASE_TOKEN_SETTINGS[token].symbol, - decimal_factor=BASE_TOKEN_SETTINGS[token].decimal_factor, - address=BASE_TOKEN_SETTINGS[token].address, - collateral_factor=LOAN_ENTITY_SPECIFIC_TOKEN_SETTINGS[token].collateral_factor, - debt_factor=LOAN_ENTITY_SPECIFIC_TOKEN_SETTINGS[token].debt_factor, - ) - for token in BASE_TOKEN_SETTINGS -} - - -class InterestRateModels(TokenValues): - """ - A class that describes the state of the interest rate indices which help transform face amounts into raw amounts. - Raw amount is the amount that would have been accumulated into the face amount if it were deposited at genesis. - """ - - def __init__(self) -> None: - super().__init__(init_value=Decimal("1")) - - -class LoanEntity(ABC): - """ - A class that describes and entity which can hold collateral, borrow debt and be liquidable. For example, on - Starknet, such an entity is the user in case of zkLend, Nostra Alpha and Nostra Mainnet, or an individual loan in - case od Hashstack V0 and Hashstack V1. - """ - - TOKEN_SETTINGS: dict[str, TokenSettings] = TOKEN_SETTINGS - - def __init__(self) -> None: - self.collateral: Portfolio = Portfolio() - self.debt: Portfolio = Portfolio() - - def compute_collateral_usd( - self, - risk_adjusted: bool, - collateral_interest_rate_models: InterestRateModels, - prices: TokenValues, - ) -> Decimal: - """ - Compute's collateral usd of interest - :param risk_adjusted: bool - :param collateral_interest_rate_models: InterestRateModels - :param prices: TokenValues - :return: Decimal - """ - return sum( - token_amount - / self.TOKEN_SETTINGS[token].decimal_factor - * ( - self.TOKEN_SETTINGS[token].collateral_factor - if risk_adjusted - else Decimal("1") - ) - * collateral_interest_rate_models.values[token] - * prices.values[token] - for token, token_amount in self.collateral.values.items() - ) - - def compute_debt_usd( - self, - risk_adjusted: bool, - debt_interest_rate_models: InterestRateModels, - prices: TokenValues, - ) -> Decimal: - """ - Compute's debt usd of interest - :param risk_adjusted: bool - :param debt_interest_rate_models: InterestRateModels - :param prices: TokenValues - :return: Decimal - """ - return sum( - token_amount - / self.TOKEN_SETTINGS[token].decimal_factor - / ( - self.TOKEN_SETTINGS[token].debt_factor - if risk_adjusted - else Decimal("1") - ) - * debt_interest_rate_models.values[token] - * prices.values[token] - for token, token_amount in self.debt.values.items() - ) - - -class State(ABC): - """ - A class that describes the state of all loan entities of the given lending protocol. - """ - - EVENTS_METHODS_MAPPING: dict[str, str] = {} - - def __init__( - self, - loan_entity_class: LoanEntity, - verbose_user: str | None = None, - ) -> None: - self.loan_entity_class: LoanEntity = loan_entity_class - self.verbose_user: str | None = verbose_user - self.loan_entities: defaultdict = defaultdict(self.loan_entity_class) - self.collateral_interest_rate_models: InterestRateModels = InterestRateModels() - self.debt_interest_rate_models: InterestRateModels = InterestRateModels() - self.last_block_number: int = 0 diff --git a/apps/web_app/utils/zklend.py b/apps/web_app/utils/zklend.py deleted file mode 100644 index 7f992fc8..00000000 --- a/apps/web_app/utils/zklend.py +++ /dev/null @@ -1,138 +0,0 @@ -from dataclasses import dataclass -from decimal import Decimal - -from .helpers import Portfolio, TokenValues -from .settings import TOKEN_SETTINGS as BASE_TOKEN_SETTINGS -from .settings import TokenSettings as BaseTokenSettings -from .state import InterestRateModels, LoanEntity, State -from shared.constants import ProtocolIDs - -# IT GIVES `ModuleNotFoundError` THAT'S WHY I COMMENTED OUT IT -# from data_handler.handlers.loan_states.zklend.fetch_zklend_specific_token_settings import ZKLEND_SPECIFIC_TOKEN_SETTINGS - -ADDRESS: str = "0x04c0a5193d58f74fbace4b74dcf65481e734ed1714121bdc571da345540efa05" - - -@dataclass -class ZkLendSpecificTokenSettings: - collateral_factor: Decimal - debt_factor: Decimal - liquidation_bonus: Decimal - protocol_token_address: str - - -@dataclass -class TokenSettings(ZkLendSpecificTokenSettings, BaseTokenSettings): - pass - - -# IT GIVES `ModuleNotFoundError` THAT'S WHY I COMMENTED OUT IT -# TOKEN_SETTINGS: dict[str, TokenSettings] = { -# token: TokenSettings( -# symbol=BASE_TOKEN_SETTINGS[token].symbol, -# decimal_factor=BASE_TOKEN_SETTINGS[token].decimal_factor, -# address=BASE_TOKEN_SETTINGS[token].address, -# collateral_factor=ZKLEND_SPECIFIC_TOKEN_SETTINGS[token].collateral_factor, -# debt_factor=ZKLEND_SPECIFIC_TOKEN_SETTINGS[token].debt_factor, -# liquidation_bonus=ZKLEND_SPECIFIC_TOKEN_SETTINGS[token].liquidation_bonus, -# protocol_token_address=ZKLEND_SPECIFIC_TOKEN_SETTINGS[ -# token -# ].protocol_token_address, -# ) -# for token in BASE_TOKEN_SETTINGS -# } - -# Keys are values of the "key_name" column in the database, values are the respective method names. -EVENTS_METHODS_MAPPING: dict[str, str] = { - "AccumulatorsSync": "process_accumulators_sync_event", - "zklend::market::Market::AccumulatorsSync": "process_accumulators_sync_event", - "Deposit": "process_deposit_event", - "zklend::market::Market::Deposit": "process_deposit_event", - "CollateralEnabled": "process_collateral_enabled_event", - "zklend::market::Market::CollateralEnabled": "process_collateral_enabled_event", - "CollateralDisabled": "process_collateral_disabled_event", - "zklend::market::Market::CollateralDisabled": "process_collateral_disabled_event", - "Withdrawal": "process_withdrawal_event", - "zklend::market::Market::Withdrawal": "process_withdrawal_event", - "Borrowing": "process_borrowing_event", - "zklend::market::Market::Borrowing": "process_borrowing_event", - "Repayment": "process_repayment_event", - "zklend::market::Market::Repayment": "process_repayment_event", - "Liquidation": "process_liquidation_event", - "zklend::market::Market::Liquidation": "process_liquidation_event", -} - - -class ZkLendLoanEntity(LoanEntity): - """ - A class that describes the zkLend loan entity. On top of the abstract `LoanEntity`, it implements the `deposit` and - `collateral_enabled` attributes in order to help with accounting for the changes in collateral. This is because - under zkLend, collateral is the amount deposited that is specificaly flagged with `collateral_enabled` set to True - for the given token. To properly account for the changes in collateral, we must hold the information about the - given token's deposits being enabled as collateral or not and the amount of the deposits. We keep all balances in raw - amounts. - """ - - TOKEN_SETTINGS: dict[str, TokenSettings] = ... - - def __init__(self) -> None: - super().__init__() - self.deposit: Portfolio = Portfolio() - self.collateral_enabled: TokenValues = TokenValues(init_value=False) - - def compute_health_factor( - self, - standardized: bool, - collateral_interest_rate_models: InterestRateModels | None = None, - debt_interest_rate_models: InterestRateModels | None = None, - prices: TokenValues | None = TokenValues(), - risk_adjusted_collateral_usd: Decimal | None = None, - debt_usd: Decimal | None = None, - ) -> Decimal: - """ - Compute's health ratio factor for zkLend loans. - :param standardized: InterestRateModels | None = None - :param collateral_interest_rate_models: InterestRateModels | None = None - :param debt_interest_rate_models: InterestRateModels | None = None - :param prices: TokenValues | None = None - :param risk_adjusted_collateral_usd: Decimal | None = None - :param debt_usd: Decimal | None = None - :return: Decimal - """ - if risk_adjusted_collateral_usd is None: - risk_adjusted_collateral_usd = self.compute_collateral_usd( - collateral_interest_rate_models=collateral_interest_rate_models, - prices=prices, - risk_adjusted=True, - ) - - if debt_usd is None: - debt_usd = self.compute_debt_usd( - debt_interest_rate_models=debt_interest_rate_models, - prices=prices, - risk_adjusted=False, - ) - - if debt_usd == Decimal("0"): - return Decimal("Inf") - - return risk_adjusted_collateral_usd / debt_usd - - -class ZkLendState(State): - """ - A class that describes the state of all zkLend loan entities. It implements methods for correct processing of every - relevant event. - """ - - PROTOCOL_NAME: str = ProtocolIDs.ZKLEND.value - EVENTS_METHODS_MAPPING: dict[str, str] = EVENTS_METHODS_MAPPING - - def __init__( - self, - verbose_user: str | None = None, - ) -> None: - super().__init__( - loan_entity_class=ZkLendLoanEntity, - verbose_user=verbose_user, - ) From 5033f31167dab4093ab468bef7f5111422a63d32 Mon Sep 17 00:00:00 2001 From: djeck1432 Date: Fri, 29 Nov 2024 08:31:15 +0100 Subject: [PATCH 08/10] adjust pipeline --- .github/workflows/ci.yml | 2 +- .github/workflows/pylint.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 33c3193e..ccb1aeac 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,6 +1,6 @@ name: CI Workflow -on: [push, pull_request] +on: [push] jobs: diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index 598f8a6b..4beaddbe 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -1,6 +1,6 @@ name: Pylint Check -on: [push, pull_request] +on: [push] jobs: lint: From ea5c06e090bf84e4b4bb12568dbf3258bdf73893 Mon Sep 17 00:00:00 2001 From: Joewizy Date: Fri, 29 Nov 2024 12:24:02 +0100 Subject: [PATCH 09/10] resolved-dashboard-modifications --- apps/dashboard_app/charts/main.py | 2 +- apps/dashboard_app/charts/main_chart_figure.py | 2 +- apps/dashboard_app/helpers/ekubo.py | 18 ++++++++++-------- apps/dashboard_app/helpers/protocol_stats.py | 4 ++-- apps/dashboard_app/helpers/tools.py | 4 ++-- 5 files changed, 16 insertions(+), 14 deletions(-) diff --git a/apps/dashboard_app/charts/main.py b/apps/dashboard_app/charts/main.py index 7d47a62b..143cf835 100644 --- a/apps/dashboard_app/charts/main.py +++ b/apps/dashboard_app/charts/main.py @@ -29,7 +29,7 @@ class Dashboard: # "Nostra Mainnet", ] - def __init__(self, ): + def __init__(self, zklend_state): """ Initialize the dashboard. :param zklend_state: ZkLendState diff --git a/apps/dashboard_app/charts/main_chart_figure.py b/apps/dashboard_app/charts/main_chart_figure.py index e3df8f79..c0fbe9ad 100644 --- a/apps/dashboard_app/charts/main_chart_figure.py +++ b/apps/dashboard_app/charts/main_chart_figure.py @@ -121,7 +121,7 @@ def get_main_chart_figure( customdata = get_custom_data(data) # Add bars for each protocol and the total liquidable debt - for col in color_map_protocol.items(): + for col in color_map_protocol.keys(): try: figure.add_trace( plotly.graph_objs.Bar( diff --git a/apps/dashboard_app/helpers/ekubo.py b/apps/dashboard_app/helpers/ekubo.py index cf1803cf..834ef6b0 100644 --- a/apps/dashboard_app/helpers/ekubo.py +++ b/apps/dashboard_app/helpers/ekubo.py @@ -90,13 +90,14 @@ def fetch_liquidity(self, bids: bool = True) -> dict[str, str | list[float]]: :return: dict[str, str | list[float]] """ - params = self.params_for_bids if bids else self.params_for_asks + params = self.params_for_bids if not bids: + params = self.params_for_asks logging.warning( "Using collateral token as base token and debt token as quote token." ) - response = requests.get(self.URL, params=params, timeout=10) + response = requests.get(self.URL, params=params) if response.ok: liquidity = response.json() @@ -107,12 +108,13 @@ def fetch_liquidity(self, bids: bool = True) -> dict[str, str | list[float]]: data["prices"], data["quantities"] = zip(*liquidity[data["type"]]) except ValueError: time.sleep(300 if bids else 5) - return self.fetch_liquidity(bids=True) - return data - - time.sleep(300 if bids else 5) - return self.fetch_liquidity( - bids= not bids, + self.fetch_liquidity(bids=True) + else: + return data + else: + time.sleep(300 if bids else 5) + self.fetch_liquidity( + bids=False if bids else True, ) def _get_available_liquidity( diff --git a/apps/dashboard_app/helpers/protocol_stats.py b/apps/dashboard_app/helpers/protocol_stats.py index 867f2dbe..b2c92aa3 100644 --- a/apps/dashboard_app/helpers/protocol_stats.py +++ b/apps/dashboard_app/helpers/protocol_stats.py @@ -135,7 +135,7 @@ def get_collateral_stats( for state in states: protocol = get_protocol(state=state) token_collaterals = defaultdict(float) - for token in TOKEN_SETTINGS.items(): + for token in TOKEN_SETTINGS(): # TODO: save zkLend amounts under token_addresses? if protocol == "zkLend": token_addresses = [ @@ -193,7 +193,7 @@ def get_debt_stats( for state in states: protocol = get_protocol(state=state) token_debts = defaultdict(float) - for token in TOKEN_SETTINGS.items(): + for token in TOKEN_SETTINGS: # TODO: save zkLend amounts under token_addresses? if protocol == "zkLend": token_addresses = [ diff --git a/apps/dashboard_app/helpers/tools.py b/apps/dashboard_app/helpers/tools.py index 5b46ab65..376bd707 100644 --- a/apps/dashboard_app/helpers/tools.py +++ b/apps/dashboard_app/helpers/tools.py @@ -132,12 +132,12 @@ def get_prices(token_decimals: dict[str, int]) -> dict[str, float]: return prices -def add_leading_zeros(address: str) -> str: +def add_leading_zeros(hash: str) -> str: """ Converts e.g. `0x436d8d078de345c11493bd91512eae60cd2713e05bcaa0bb9f0cba90358c6e` to `0x00436d8d078de345c11493bd91512eae60cd2713e05bcaa0bb9f0cba90358c6e`. """ - return "0x" + address[2:].zfill(64) + return "0x" + hash[2:].zfill(64) def get_addresses( From 761a0bd629f4803b8631c2956f8ab7ef409ca96c Mon Sep 17 00:00:00 2001 From: Joewizy Date: Sat, 30 Nov 2024 05:25:43 +0100 Subject: [PATCH 10/10] fixed-code-modification --- apps/dashboard_app/helpers/loans_table.py | 11 --------- apps/dashboard_app/helpers/protocol_stats.py | 4 ++-- apps/data_handler/README.md | 24 -------------------- 3 files changed, 2 insertions(+), 37 deletions(-) diff --git a/apps/dashboard_app/helpers/loans_table.py b/apps/dashboard_app/helpers/loans_table.py index 12f4c88d..db16de8b 100644 --- a/apps/dashboard_app/helpers/loans_table.py +++ b/apps/dashboard_app/helpers/loans_table.py @@ -15,17 +15,6 @@ def get_protocol(state: State) -> str: Takes a parameter of State which gets the loan entities and returns the string. """ - # TODO: Improve the inference. - if isinstance(state, ZkLendState): - return "zkLend" - if isinstance(state, NostraAlphaState) and not isinstance( - state, NostraMainnetState - ): - return "Nostra Alpha" - if isinstance(state, NostraMainnetState): - return "Nostra Mainnet" - raise ValueError - return state.get_protocol_name diff --git a/apps/dashboard_app/helpers/protocol_stats.py b/apps/dashboard_app/helpers/protocol_stats.py index b2c92aa3..40ccb878 100644 --- a/apps/dashboard_app/helpers/protocol_stats.py +++ b/apps/dashboard_app/helpers/protocol_stats.py @@ -75,7 +75,7 @@ def get_supply_stats( for state in states: protocol = get_protocol(state=state) token_supplies = {} - for token in TOKEN_SETTINGS.items(): + for token in TOKEN_SETTINGS: ( addresses, selector, @@ -135,7 +135,7 @@ def get_collateral_stats( for state in states: protocol = get_protocol(state=state) token_collaterals = defaultdict(float) - for token in TOKEN_SETTINGS(): + for token in TOKEN_SETTINGS: # TODO: save zkLend amounts under token_addresses? if protocol == "zkLend": token_addresses = [ diff --git a/apps/data_handler/README.md b/apps/data_handler/README.md index 3b4dab2e..ae2f047e 100644 --- a/apps/data_handler/README.md +++ b/apps/data_handler/README.md @@ -96,30 +96,6 @@ docker-compose exec backend bash ``` -## Running Migration Command: -1. Go to root folder `derisk-research` - -### Prepare the Environment -2. Run up db in docker: -``` -docker-compose -f devops/dev/docker-compose.db.yaml up -d --remove-orphans -``` -3. Navigate to the `data_hander` directory: -``` -cd apps/data_handler -``` -4. Install all dependencies: -``` -poetry install -``` -5. Go back to apps folder: -``` -cd .. -``` -5. Run migration command: -``` -alembic -c data_handler/alembic.ini revision --autogenerate -m "your migration message here" -======= ## How to run migration command: 1. Set up `.env.dev` into `derisk-research/apps/data_handler` 2. Go back to `derisk-research/apps` directory