diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 94c65727e..1ade669b7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -69,7 +69,9 @@ jobs: - name: Run all checks run: pdm run make all env: - NODE_API_KEY: ${{ secrets.ALCHEMY_KEY }} + ALCHEMY_API_KEY: ${{ secrets.ALCHEMY_API_KEY }} + ETHERSCAN_API_KEY: ${{ secrets.ETHERSCAN_API_KEY }} + ONFINALITY_API_KEY: ${{ secrets.ONFINALITY_API_KEY }} - name: Publish stable image uses: docker/build-push-action@v5 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8551d517f..71cb3f1c6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -49,4 +49,6 @@ jobs: - name: Run tests run: pdm run make test env: - NODE_API_KEY: ${{ secrets.ALCHEMY_KEY }} + ALCHEMY_API_KEY: ${{ secrets.ALCHEMY_API_KEY }} + ETHERSCAN_API_KEY: ${{ secrets.ETHERSCAN_API_KEY }} + ONFINALITY_API_KEY: ${{ secrets.ONFINALITY_API_KEY }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 90fe92a00..c47b352f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ The format is based on [Keep a Changelog], and this project adheres to [Semantic Releases prior to 7.0 has been removed from this file to declutter search results; see the [archived copy](https://github.com/dipdup-io/dipdup/blob/8.0.0b5/CHANGELOG.md) for the full list. -## [8.2.0rc1] - ????-??-?? +## [8.2.0rc1] - 2024-12-23 ### Added diff --git a/docs/8.examples/_demos_table.md b/docs/8.examples/_demos_table.md index 69907f164..0f157bae8 100644 --- a/docs/8.examples/_demos_table.md +++ b/docs/8.examples/_demos_table.md @@ -1,22 +1,22 @@ | name | network | description | source | |-|-|-|-| -| demo_blank | | Empty config for a fresh start | [link](https://github.com/dipdup-io/dipdup/tree/8.1.3/src/demo_blank) | -| demo_substrate_events | | Substrate balance transfers | [link](https://github.com/dipdup-io/dipdup/tree/8.1.3/src/demo_substrate_events) | -| demo_evm_events | EVM | ERC-20 token transfers (from event logs) | [link](https://github.com/dipdup-io/dipdup/tree/8.1.3/src/demo_evm_events) | -| demo_evm_transactions | EVM | ERC-20 token transfers (from transactions) | [link](https://github.com/dipdup-io/dipdup/tree/8.1.3/src/demo_evm_transactions) | -| demo_evm_uniswap | EVM | Uniswap V3 pools, positions, etc. (advanced, uses TimescaleDB) | [link](https://github.com/dipdup-io/dipdup/tree/8.1.3/src/demo_evm_uniswap) | -| demo_starknet_events | Starknet | ERC-20 token transfers (from events) | [link](https://github.com/dipdup-io/dipdup/tree/8.1.3/src/demo_starknet_events) | -| demo_tezos_auction | Tezos | NFT marketplace (TzColors) | [link](https://github.com/dipdup-io/dipdup/tree/8.1.3/src/demo_tezos_auction) | -| demo_tezos_dao | Tezos | DAO registry (Homebase DAO) | [link](https://github.com/dipdup-io/dipdup/tree/8.1.3/src/demo_tezos_dao) | -| demo_tezos_dex | Tezos | DEX balances and liquidity (Quipuswap) | [link](https://github.com/dipdup-io/dipdup/tree/8.1.3/src/demo_tezos_dex) | -| demo_tezos_domains | Tezos | Domain name service (Tezos Domains) | [link](https://github.com/dipdup-io/dipdup/tree/8.1.3/src/demo_tezos_domains) | -| demo_tezos_etherlink | Tezos | Etherlink smart rollup transactions | [link](https://github.com/dipdup-io/dipdup/tree/8.1.3/src/demo_tezos_etherlink) | -| demo_tezos_events | Tezos | Processing contract events | [link](https://github.com/dipdup-io/dipdup/tree/8.1.3/src/demo_tezos_events) | -| demo_tezos_factories | Tezos | Example of spawning indexes in runtime | [link](https://github.com/dipdup-io/dipdup/tree/8.1.3/src/demo_tezos_factories) | -| demo_tezos_head | Tezos | Processing head block metadata (realtime only) | [link](https://github.com/dipdup-io/dipdup/tree/8.1.3/src/demo_tezos_head) | -| demo_tezos_nft_marketplace | Tezos | NFT marketplace (hic at nunc) | [link](https://github.com/dipdup-io/dipdup/tree/8.1.3/src/demo_tezos_nft_marketplace) | -| demo_tezos_raw | Tezos | Process raw operations without filtering and typed payloads | [link](https://github.com/dipdup-io/dipdup/tree/8.1.3/src/demo_tezos_raw) | -| demo_tezos_token | Tezos | FA1.2 token contract operations | [link](https://github.com/dipdup-io/dipdup/tree/8.1.3/src/demo_tezos_token) | -| demo_tezos_token_balances | Tezos | FA1.2 token balances | [link](https://github.com/dipdup-io/dipdup/tree/8.1.3/src/demo_tezos_token_balances) | -| demo_tezos_token_transfers | Tezos | FA1.2 token transfers | [link](https://github.com/dipdup-io/dipdup/tree/8.1.3/src/demo_tezos_token_transfers) | +| demo_blank | | Empty config for a fresh start | [link](https://github.com/dipdup-io/dipdup/tree/8.2.0rc1/src/demo_blank) | +| demo_substrate_events | | Substrate balance transfers | [link](https://github.com/dipdup-io/dipdup/tree/8.2.0rc1/src/demo_substrate_events) | +| demo_evm_events | EVM | ERC-20 token transfers (from event logs) | [link](https://github.com/dipdup-io/dipdup/tree/8.2.0rc1/src/demo_evm_events) | +| demo_evm_transactions | EVM | ERC-20 token transfers (from transactions) | [link](https://github.com/dipdup-io/dipdup/tree/8.2.0rc1/src/demo_evm_transactions) | +| demo_evm_uniswap | EVM | Uniswap V3 pools, positions, etc. (advanced, uses TimescaleDB) | [link](https://github.com/dipdup-io/dipdup/tree/8.2.0rc1/src/demo_evm_uniswap) | +| demo_starknet_events | Starknet | ERC-20 token transfers (from events) | [link](https://github.com/dipdup-io/dipdup/tree/8.2.0rc1/src/demo_starknet_events) | +| demo_tezos_auction | Tezos | NFT marketplace (TzColors) | [link](https://github.com/dipdup-io/dipdup/tree/8.2.0rc1/src/demo_tezos_auction) | +| demo_tezos_dao | Tezos | DAO registry (Homebase DAO) | [link](https://github.com/dipdup-io/dipdup/tree/8.2.0rc1/src/demo_tezos_dao) | +| demo_tezos_dex | Tezos | DEX balances and liquidity (Quipuswap) | [link](https://github.com/dipdup-io/dipdup/tree/8.2.0rc1/src/demo_tezos_dex) | +| demo_tezos_domains | Tezos | Domain name service (Tezos Domains) | [link](https://github.com/dipdup-io/dipdup/tree/8.2.0rc1/src/demo_tezos_domains) | +| demo_tezos_etherlink | Tezos | Etherlink smart rollup transactions | [link](https://github.com/dipdup-io/dipdup/tree/8.2.0rc1/src/demo_tezos_etherlink) | +| demo_tezos_events | Tezos | Processing contract events | [link](https://github.com/dipdup-io/dipdup/tree/8.2.0rc1/src/demo_tezos_events) | +| demo_tezos_factories | Tezos | Example of spawning indexes in runtime | [link](https://github.com/dipdup-io/dipdup/tree/8.2.0rc1/src/demo_tezos_factories) | +| demo_tezos_head | Tezos | Processing head block metadata (realtime only) | [link](https://github.com/dipdup-io/dipdup/tree/8.2.0rc1/src/demo_tezos_head) | +| demo_tezos_nft_marketplace | Tezos | NFT marketplace (hic at nunc) | [link](https://github.com/dipdup-io/dipdup/tree/8.2.0rc1/src/demo_tezos_nft_marketplace) | +| demo_tezos_raw | Tezos | Process raw operations without filtering and typed payloads | [link](https://github.com/dipdup-io/dipdup/tree/8.2.0rc1/src/demo_tezos_raw) | +| demo_tezos_token | Tezos | FA1.2 token contract operations | [link](https://github.com/dipdup-io/dipdup/tree/8.2.0rc1/src/demo_tezos_token) | +| demo_tezos_token_balances | Tezos | FA1.2 token balances | [link](https://github.com/dipdup-io/dipdup/tree/8.2.0rc1/src/demo_tezos_token_balances) | +| demo_tezos_token_transfers | Tezos | FA1.2 token transfers | [link](https://github.com/dipdup-io/dipdup/tree/8.2.0rc1/src/demo_tezos_token_transfers) | diff --git a/docs/9.release-notes/1.v8.2.md b/docs/9.release-notes/1.v8.2.md index 0019e60ee..8e8b5bdf3 100644 --- a/docs/9.release-notes/1.v8.2.md +++ b/docs/9.release-notes/1.v8.2.md @@ -1,5 +1,5 @@ --- -title: "8.2" +title: "8.2 (rc)" description: DipDup 8.2 release notes --- @@ -7,5 +7,15 @@ description: DipDup 8.2 release notes # Release Notes: 8.2 +Today we are glad to present you DipDup 8.2. This release is all about one thing: indexing Substrate networks. + +If you're new to DipDup and want action, follow the [Substrate Quickstart Guide](../0.quickstart-substrate.md)! + +If you're familiar with DipDup, the process is very similar to indexing EVM and Starknet networks. Run the `dipdup new` command to initialize the project interactively, update the with your URLs/variables/addresses (in this case, pallets) to index, run `dipdup init`, and you're good to go. + +We support three sources of Substrate data. Squid Network provides all kinds of historical data in a very efficient format. For the blocks yet to be processed by Squid Network you can use Substrate node RPC. Indexing with node RPC is much slower, but allows to process blocks as soon as they are announced. Finally, an optional Subscan API connector is used to speed up the project initialization. + +This release is a pretty big milestone for us, but we decided to make it a minor one, since we follow the semantic versioning, and 8.2 doesn't introduce any changes that break backward compatibility. So, you can extend your existing projects with Substrate data right after the update. A if you're still on 7.x branch, it's a pretty good time for us to remind you to upgrade! :) + {{ #include 9.release-notes/_8.2_changelog.md }} {{ #include 9.release-notes/_footer.md }} diff --git a/pdm.lock b/pdm.lock index 756396142..d9e392495 100644 --- a/pdm.lock +++ b/pdm.lock @@ -912,7 +912,7 @@ files = [ [[package]] name = "jinja2" -version = "3.1.4" +version = "3.1.5" requires_python = ">=3.7" summary = "A very fast and expressive template engine." groups = ["default", "docs", "perf"] @@ -920,8 +920,8 @@ dependencies = [ "MarkupSafe>=2.0", ] files = [ - {file = "jinja2-3.1.4-py3-none-any.whl", hash = "sha256:bc5dd2abb727a5319567b7a813e6a2e7318c39f4f487cfe6c89c6f9c7d25197d"}, - {file = "jinja2-3.1.4.tar.gz", hash = "sha256:4a3aee7acbbe7303aede8e9648d13b8bf88a429282aa6122a993f0ac800cb369"}, + {file = "jinja2-3.1.5-py3-none-any.whl", hash = "sha256:aba0f4dc9ed8013c424088f68a5c226f7d6097ed89b246d7749c2ec4175c6adb"}, + {file = "jinja2-3.1.5.tar.gz", hash = "sha256:8fefff8dc3034e27bb80d67c671eb8a9bc424c0ef4c0826edbff304cceff43bb"}, ] [[package]] @@ -1993,9 +1993,9 @@ dependencies = [ "asgiref<4.0.0,>=3.4.1", "bip-utils<3.0.0,>=2.9.3", "crypto-cpp-py==1.4.4", - "eth-keyfile<0.9.0,>=0.8.1", + "eth-keyfile<1.0.0,>=0.8.1", "lark<2.0.0,>=1.1.5", - "ledgerwallet<0.6.0,>=0.5.0", + "ledgerwallet<1.0.0,>=0.5.0", "marshmallow-dataclass<8.8.0", "marshmallow-oneofschema==3.1.1", "marshmallow<4.0.0,>=3.15.0", diff --git a/pyproject.toml b/pyproject.toml index 7e2af3079..d5a4ce9c3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "dipdup" description = "Modular framework for creating selective indexers and featureful backends for dapps" -version = "8.1.3" +version = "8.2.0rc1" license = { text = "MIT" } authors = [ { name = "Lev Gorodetskii", email = "dipdup@drsr.io" }, diff --git a/requirements.txt b/requirements.txt index 46821d46c..75e3c8c8d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -45,7 +45,7 @@ idna==3.10 inflect==5.6.2 iso8601==2.1.0 isort==5.13.2 -jinja2==3.1.4 +jinja2==3.1.5 lark==1.2.2 lru-dict==1.3.0 markupsafe==3.0.2 diff --git a/src/demo_blank/pyproject.toml b/src/demo_blank/pyproject.toml index 16dd679eb..5f253c420 100644 --- a/src/demo_blank/pyproject.toml +++ b/src/demo_blank/pyproject.toml @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 [project] name = "demo_blank" version = "0.0.1" diff --git a/src/demo_evm_events/pyproject.toml b/src/demo_evm_events/pyproject.toml index f0826f9c1..cd97e8a05 100644 --- a/src/demo_evm_events/pyproject.toml +++ b/src/demo_evm_events/pyproject.toml @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 [project] name = "demo_evm_events" version = "0.0.1" diff --git a/src/demo_evm_events/types/eth_usdt/evm_events/transfer.py b/src/demo_evm_events/types/eth_usdt/evm_events/transfer.py index 27f55f337..536d2bcde 100644 --- a/src/demo_evm_events/types/eth_usdt/evm_events/transfer.py +++ b/src/demo_evm_events/types/eth_usdt/evm_events/transfer.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_evm_transactions/pyproject.toml b/src/demo_evm_transactions/pyproject.toml index 8c732222b..bec1e8112 100644 --- a/src/demo_evm_transactions/pyproject.toml +++ b/src/demo_evm_transactions/pyproject.toml @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 [project] name = "demo_evm_transactions" version = "0.0.1" diff --git a/src/demo_evm_transactions/types/eth_usdt/evm_transactions/transfer.py b/src/demo_evm_transactions/types/eth_usdt/evm_transactions/transfer.py index 8778c1929..0c16d0206 100644 --- a/src/demo_evm_transactions/types/eth_usdt/evm_transactions/transfer.py +++ b/src/demo_evm_transactions/types/eth_usdt/evm_transactions/transfer.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_evm_uniswap/pyproject.toml b/src/demo_evm_uniswap/pyproject.toml index 76a1f3bad..7997563da 100644 --- a/src/demo_evm_uniswap/pyproject.toml +++ b/src/demo_evm_uniswap/pyproject.toml @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 [project] name = "demo_evm_uniswap" version = "0.0.1" diff --git a/src/demo_evm_uniswap/types/factory/evm_events/pool_created.py b/src/demo_evm_uniswap/types/factory/evm_events/pool_created.py index 18eb7d9f5..954ebe533 100644 --- a/src/demo_evm_uniswap/types/factory/evm_events/pool_created.py +++ b/src/demo_evm_uniswap/types/factory/evm_events/pool_created.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_evm_uniswap/types/pool/evm_events/burn.py b/src/demo_evm_uniswap/types/pool/evm_events/burn.py index b961055eb..4b2529722 100644 --- a/src/demo_evm_uniswap/types/pool/evm_events/burn.py +++ b/src/demo_evm_uniswap/types/pool/evm_events/burn.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_evm_uniswap/types/pool/evm_events/collect.py b/src/demo_evm_uniswap/types/pool/evm_events/collect.py index 0c8e51cd8..fe3f2f21d 100644 --- a/src/demo_evm_uniswap/types/pool/evm_events/collect.py +++ b/src/demo_evm_uniswap/types/pool/evm_events/collect.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_evm_uniswap/types/pool/evm_events/flash.py b/src/demo_evm_uniswap/types/pool/evm_events/flash.py index 742d1b2ee..52a9b7fcb 100644 --- a/src/demo_evm_uniswap/types/pool/evm_events/flash.py +++ b/src/demo_evm_uniswap/types/pool/evm_events/flash.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_evm_uniswap/types/pool/evm_events/initialize.py b/src/demo_evm_uniswap/types/pool/evm_events/initialize.py index 2d40ddf4b..b2a29d098 100644 --- a/src/demo_evm_uniswap/types/pool/evm_events/initialize.py +++ b/src/demo_evm_uniswap/types/pool/evm_events/initialize.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_evm_uniswap/types/pool/evm_events/mint.py b/src/demo_evm_uniswap/types/pool/evm_events/mint.py index f5aa263e1..918dcfd15 100644 --- a/src/demo_evm_uniswap/types/pool/evm_events/mint.py +++ b/src/demo_evm_uniswap/types/pool/evm_events/mint.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_evm_uniswap/types/pool/evm_events/swap.py b/src/demo_evm_uniswap/types/pool/evm_events/swap.py index 1dafea49f..f4f77ee77 100644 --- a/src/demo_evm_uniswap/types/pool/evm_events/swap.py +++ b/src/demo_evm_uniswap/types/pool/evm_events/swap.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_evm_uniswap/types/position_manager/evm_events/collect.py b/src/demo_evm_uniswap/types/position_manager/evm_events/collect.py index f3abdfc40..8d5b206a8 100644 --- a/src/demo_evm_uniswap/types/position_manager/evm_events/collect.py +++ b/src/demo_evm_uniswap/types/position_manager/evm_events/collect.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_evm_uniswap/types/position_manager/evm_events/decrease_liquidity.py b/src/demo_evm_uniswap/types/position_manager/evm_events/decrease_liquidity.py index 916e654e7..6537a51ef 100644 --- a/src/demo_evm_uniswap/types/position_manager/evm_events/decrease_liquidity.py +++ b/src/demo_evm_uniswap/types/position_manager/evm_events/decrease_liquidity.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_evm_uniswap/types/position_manager/evm_events/increase_liquidity.py b/src/demo_evm_uniswap/types/position_manager/evm_events/increase_liquidity.py index 072fbd2d0..5f2be1a35 100644 --- a/src/demo_evm_uniswap/types/position_manager/evm_events/increase_liquidity.py +++ b/src/demo_evm_uniswap/types/position_manager/evm_events/increase_liquidity.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_evm_uniswap/types/position_manager/evm_events/transfer.py b/src/demo_evm_uniswap/types/position_manager/evm_events/transfer.py index 0fae46a8f..0053497da 100644 --- a/src/demo_evm_uniswap/types/position_manager/evm_events/transfer.py +++ b/src/demo_evm_uniswap/types/position_manager/evm_events/transfer.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_starknet_events/pyproject.toml b/src/demo_starknet_events/pyproject.toml index 7fd02e9c1..962cbbcc7 100644 --- a/src/demo_starknet_events/pyproject.toml +++ b/src/demo_starknet_events/pyproject.toml @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 [project] name = "demo_starknet_events" version = "0.0.1" diff --git a/src/demo_starknet_events/types/stark_usdt/starknet_events/transfer.py b/src/demo_starknet_events/types/stark_usdt/starknet_events/transfer.py index ab51f65fa..a212e1854 100644 --- a/src/demo_starknet_events/types/stark_usdt/starknet_events/transfer.py +++ b/src/demo_starknet_events/types/stark_usdt/starknet_events/transfer.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_substrate_events/pyproject.toml b/src/demo_substrate_events/pyproject.toml index 69e8fd163..392f797cd 100644 --- a/src/demo_substrate_events/pyproject.toml +++ b/src/demo_substrate_events/pyproject.toml @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.2 +# generated by DipDup 8.2.0rc1 [project] name = "demo_substrate_events" version = "0.0.1" diff --git a/src/demo_substrate_events/types/assethub/substrate_events/assets_transferred/v601.py b/src/demo_substrate_events/types/assethub/substrate_events/assets_transferred/v601.py index afcbb6380..a838ee10d 100644 --- a/src/demo_substrate_events/types/assethub/substrate_events/assets_transferred/v601.py +++ b/src/demo_substrate_events/types/assethub/substrate_events/assets_transferred/v601.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.2 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_substrate_events/types/assethub/substrate_events/assets_transferred/v700.py b/src/demo_substrate_events/types/assethub/substrate_events/assets_transferred/v700.py index 7f84fa9a8..3b0274190 100644 --- a/src/demo_substrate_events/types/assethub/substrate_events/assets_transferred/v700.py +++ b/src/demo_substrate_events/types/assethub/substrate_events/assets_transferred/v700.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.2 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_tezos_auction/pyproject.toml b/src/demo_tezos_auction/pyproject.toml index 296715a9c..2ccb25ffe 100644 --- a/src/demo_tezos_auction/pyproject.toml +++ b/src/demo_tezos_auction/pyproject.toml @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 [project] name = "demo_tezos_auction" version = "0.0.1" diff --git a/src/demo_tezos_auction/types/tzcolors_auction/tezos_parameters/bid.py b/src/demo_tezos_auction/types/tzcolors_auction/tezos_parameters/bid.py index 75b3884d4..70f9aa1fd 100644 --- a/src/demo_tezos_auction/types/tzcolors_auction/tezos_parameters/bid.py +++ b/src/demo_tezos_auction/types/tzcolors_auction/tezos_parameters/bid.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_tezos_auction/types/tzcolors_auction/tezos_parameters/create_auction.py b/src/demo_tezos_auction/types/tzcolors_auction/tezos_parameters/create_auction.py index ea5cf76c2..01f5bffa2 100644 --- a/src/demo_tezos_auction/types/tzcolors_auction/tezos_parameters/create_auction.py +++ b/src/demo_tezos_auction/types/tzcolors_auction/tezos_parameters/create_auction.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_tezos_auction/types/tzcolors_auction/tezos_parameters/withdraw.py b/src/demo_tezos_auction/types/tzcolors_auction/tezos_parameters/withdraw.py index eca48aecb..2e9d12db3 100644 --- a/src/demo_tezos_auction/types/tzcolors_auction/tezos_parameters/withdraw.py +++ b/src/demo_tezos_auction/types/tzcolors_auction/tezos_parameters/withdraw.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_tezos_auction/types/tzcolors_auction/tezos_storage.py b/src/demo_tezos_auction/types/tzcolors_auction/tezos_storage.py index ea63ed3dc..ebbc2e863 100644 --- a/src/demo_tezos_auction/types/tzcolors_auction/tezos_storage.py +++ b/src/demo_tezos_auction/types/tzcolors_auction/tezos_storage.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_tezos_dao/pyproject.toml b/src/demo_tezos_dao/pyproject.toml index ba690f993..517664eac 100644 --- a/src/demo_tezos_dao/pyproject.toml +++ b/src/demo_tezos_dao/pyproject.toml @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 [project] name = "demo_tezos_dao" version = "0.0.1" diff --git a/src/demo_tezos_dao/types/registry/tezos_parameters/propose.py b/src/demo_tezos_dao/types/registry/tezos_parameters/propose.py index 30a8a8f8f..47c147e1f 100644 --- a/src/demo_tezos_dao/types/registry/tezos_parameters/propose.py +++ b/src/demo_tezos_dao/types/registry/tezos_parameters/propose.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_tezos_dao/types/registry/tezos_storage.py b/src/demo_tezos_dao/types/registry/tezos_storage.py index 0ca288a2f..be32259bc 100644 --- a/src/demo_tezos_dao/types/registry/tezos_storage.py +++ b/src/demo_tezos_dao/types/registry/tezos_storage.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_tezos_dex/pyproject.toml b/src/demo_tezos_dex/pyproject.toml index 39ffebe93..ce4abaf1b 100644 --- a/src/demo_tezos_dex/pyproject.toml +++ b/src/demo_tezos_dex/pyproject.toml @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 [project] name = "demo_tezos_dex" version = "0.0.1" diff --git a/src/demo_tezos_dex/types/fa12_token/tezos_parameters/transfer.py b/src/demo_tezos_dex/types/fa12_token/tezos_parameters/transfer.py index d701723ce..8634af799 100644 --- a/src/demo_tezos_dex/types/fa12_token/tezos_parameters/transfer.py +++ b/src/demo_tezos_dex/types/fa12_token/tezos_parameters/transfer.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_tezos_dex/types/fa12_token/tezos_storage.py b/src/demo_tezos_dex/types/fa12_token/tezos_storage.py index 34f7a25d0..6c1e59210 100644 --- a/src/demo_tezos_dex/types/fa12_token/tezos_storage.py +++ b/src/demo_tezos_dex/types/fa12_token/tezos_storage.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_tezos_dex/types/fa2_token/tezos_parameters/transfer.py b/src/demo_tezos_dex/types/fa2_token/tezos_parameters/transfer.py index 7676b58ac..8acb67ae1 100644 --- a/src/demo_tezos_dex/types/fa2_token/tezos_parameters/transfer.py +++ b/src/demo_tezos_dex/types/fa2_token/tezos_parameters/transfer.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_tezos_dex/types/fa2_token/tezos_storage.py b/src/demo_tezos_dex/types/fa2_token/tezos_storage.py index 8648ce20e..de680e0fc 100644 --- a/src/demo_tezos_dex/types/fa2_token/tezos_storage.py +++ b/src/demo_tezos_dex/types/fa2_token/tezos_storage.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_tezos_dex/types/quipu_fa12/tezos_parameters/divest_liquidity.py b/src/demo_tezos_dex/types/quipu_fa12/tezos_parameters/divest_liquidity.py index 07d4713d6..97ba78bfa 100644 --- a/src/demo_tezos_dex/types/quipu_fa12/tezos_parameters/divest_liquidity.py +++ b/src/demo_tezos_dex/types/quipu_fa12/tezos_parameters/divest_liquidity.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_tezos_dex/types/quipu_fa12/tezos_parameters/invest_liquidity.py b/src/demo_tezos_dex/types/quipu_fa12/tezos_parameters/invest_liquidity.py index 09ecc33eb..76236ff2b 100644 --- a/src/demo_tezos_dex/types/quipu_fa12/tezos_parameters/invest_liquidity.py +++ b/src/demo_tezos_dex/types/quipu_fa12/tezos_parameters/invest_liquidity.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_tezos_dex/types/quipu_fa12/tezos_parameters/tez_to_token_payment.py b/src/demo_tezos_dex/types/quipu_fa12/tezos_parameters/tez_to_token_payment.py index 29905dd76..e990590d3 100644 --- a/src/demo_tezos_dex/types/quipu_fa12/tezos_parameters/tez_to_token_payment.py +++ b/src/demo_tezos_dex/types/quipu_fa12/tezos_parameters/tez_to_token_payment.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_tezos_dex/types/quipu_fa12/tezos_parameters/token_to_tez_payment.py b/src/demo_tezos_dex/types/quipu_fa12/tezos_parameters/token_to_tez_payment.py index 11c0b843d..5b1b8ef29 100644 --- a/src/demo_tezos_dex/types/quipu_fa12/tezos_parameters/token_to_tez_payment.py +++ b/src/demo_tezos_dex/types/quipu_fa12/tezos_parameters/token_to_tez_payment.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_tezos_dex/types/quipu_fa12/tezos_parameters/transfer.py b/src/demo_tezos_dex/types/quipu_fa12/tezos_parameters/transfer.py index d701723ce..8634af799 100644 --- a/src/demo_tezos_dex/types/quipu_fa12/tezos_parameters/transfer.py +++ b/src/demo_tezos_dex/types/quipu_fa12/tezos_parameters/transfer.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_tezos_dex/types/quipu_fa12/tezos_parameters/withdraw_profit.py b/src/demo_tezos_dex/types/quipu_fa12/tezos_parameters/withdraw_profit.py index e60194566..80a03a163 100644 --- a/src/demo_tezos_dex/types/quipu_fa12/tezos_parameters/withdraw_profit.py +++ b/src/demo_tezos_dex/types/quipu_fa12/tezos_parameters/withdraw_profit.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_tezos_dex/types/quipu_fa12/tezos_storage.py b/src/demo_tezos_dex/types/quipu_fa12/tezos_storage.py index bae488c4e..c32a6fd06 100644 --- a/src/demo_tezos_dex/types/quipu_fa12/tezos_storage.py +++ b/src/demo_tezos_dex/types/quipu_fa12/tezos_storage.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_tezos_dex/types/quipu_fa2/tezos_parameters/divest_liquidity.py b/src/demo_tezos_dex/types/quipu_fa2/tezos_parameters/divest_liquidity.py index 07d4713d6..97ba78bfa 100644 --- a/src/demo_tezos_dex/types/quipu_fa2/tezos_parameters/divest_liquidity.py +++ b/src/demo_tezos_dex/types/quipu_fa2/tezos_parameters/divest_liquidity.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_tezos_dex/types/quipu_fa2/tezos_parameters/invest_liquidity.py b/src/demo_tezos_dex/types/quipu_fa2/tezos_parameters/invest_liquidity.py index 09ecc33eb..76236ff2b 100644 --- a/src/demo_tezos_dex/types/quipu_fa2/tezos_parameters/invest_liquidity.py +++ b/src/demo_tezos_dex/types/quipu_fa2/tezos_parameters/invest_liquidity.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_tezos_dex/types/quipu_fa2/tezos_parameters/tez_to_token_payment.py b/src/demo_tezos_dex/types/quipu_fa2/tezos_parameters/tez_to_token_payment.py index 29905dd76..e990590d3 100644 --- a/src/demo_tezos_dex/types/quipu_fa2/tezos_parameters/tez_to_token_payment.py +++ b/src/demo_tezos_dex/types/quipu_fa2/tezos_parameters/tez_to_token_payment.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_tezos_dex/types/quipu_fa2/tezos_parameters/token_to_tez_payment.py b/src/demo_tezos_dex/types/quipu_fa2/tezos_parameters/token_to_tez_payment.py index 11c0b843d..5b1b8ef29 100644 --- a/src/demo_tezos_dex/types/quipu_fa2/tezos_parameters/token_to_tez_payment.py +++ b/src/demo_tezos_dex/types/quipu_fa2/tezos_parameters/token_to_tez_payment.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_tezos_dex/types/quipu_fa2/tezos_parameters/transfer.py b/src/demo_tezos_dex/types/quipu_fa2/tezos_parameters/transfer.py index 7676b58ac..8acb67ae1 100644 --- a/src/demo_tezos_dex/types/quipu_fa2/tezos_parameters/transfer.py +++ b/src/demo_tezos_dex/types/quipu_fa2/tezos_parameters/transfer.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_tezos_dex/types/quipu_fa2/tezos_parameters/withdraw_profit.py b/src/demo_tezos_dex/types/quipu_fa2/tezos_parameters/withdraw_profit.py index e60194566..80a03a163 100644 --- a/src/demo_tezos_dex/types/quipu_fa2/tezos_parameters/withdraw_profit.py +++ b/src/demo_tezos_dex/types/quipu_fa2/tezos_parameters/withdraw_profit.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_tezos_dex/types/quipu_fa2/tezos_storage.py b/src/demo_tezos_dex/types/quipu_fa2/tezos_storage.py index a2cc56931..39936486b 100644 --- a/src/demo_tezos_dex/types/quipu_fa2/tezos_storage.py +++ b/src/demo_tezos_dex/types/quipu_fa2/tezos_storage.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_tezos_domains/pyproject.toml b/src/demo_tezos_domains/pyproject.toml index fa3aa33ac..8cbaf89b8 100644 --- a/src/demo_tezos_domains/pyproject.toml +++ b/src/demo_tezos_domains/pyproject.toml @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 [project] name = "demo_tezos_domains" version = "0.0.1" diff --git a/src/demo_tezos_domains/types/name_registry/tezos_big_maps/store_expiry_map_key.py b/src/demo_tezos_domains/types/name_registry/tezos_big_maps/store_expiry_map_key.py index c79058799..25857f84c 100644 --- a/src/demo_tezos_domains/types/name_registry/tezos_big_maps/store_expiry_map_key.py +++ b/src/demo_tezos_domains/types/name_registry/tezos_big_maps/store_expiry_map_key.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_tezos_domains/types/name_registry/tezos_big_maps/store_expiry_map_value.py b/src/demo_tezos_domains/types/name_registry/tezos_big_maps/store_expiry_map_value.py index b321cbdb7..159e48fc4 100644 --- a/src/demo_tezos_domains/types/name_registry/tezos_big_maps/store_expiry_map_value.py +++ b/src/demo_tezos_domains/types/name_registry/tezos_big_maps/store_expiry_map_value.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_tezos_domains/types/name_registry/tezos_big_maps/store_records_key.py b/src/demo_tezos_domains/types/name_registry/tezos_big_maps/store_records_key.py index b8deaec5e..03859ab06 100644 --- a/src/demo_tezos_domains/types/name_registry/tezos_big_maps/store_records_key.py +++ b/src/demo_tezos_domains/types/name_registry/tezos_big_maps/store_records_key.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_tezos_domains/types/name_registry/tezos_big_maps/store_records_value.py b/src/demo_tezos_domains/types/name_registry/tezos_big_maps/store_records_value.py index 7ad4e4d6c..5fef466c6 100644 --- a/src/demo_tezos_domains/types/name_registry/tezos_big_maps/store_records_value.py +++ b/src/demo_tezos_domains/types/name_registry/tezos_big_maps/store_records_value.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_tezos_etherlink/pyproject.toml b/src/demo_tezos_etherlink/pyproject.toml index 00ba24a35..a808bea0e 100644 --- a/src/demo_tezos_etherlink/pyproject.toml +++ b/src/demo_tezos_etherlink/pyproject.toml @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 [project] name = "demo_tezos_etherlink" version = "0.0.1" diff --git a/src/demo_tezos_etherlink/types/rollup/tezos_parameters/default.py b/src/demo_tezos_etherlink/types/rollup/tezos_parameters/default.py index 3192a7969..bc111d229 100644 --- a/src/demo_tezos_etherlink/types/rollup/tezos_parameters/default.py +++ b/src/demo_tezos_etherlink/types/rollup/tezos_parameters/default.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_tezos_etherlink/types/rollup/tezos_storage.py b/src/demo_tezos_etherlink/types/rollup/tezos_storage.py index 9cb8f6d00..90614df5f 100644 --- a/src/demo_tezos_etherlink/types/rollup/tezos_storage.py +++ b/src/demo_tezos_etherlink/types/rollup/tezos_storage.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_tezos_etherlink/types/ticket_helper/tezos_parameters/default.py b/src/demo_tezos_etherlink/types/ticket_helper/tezos_parameters/default.py index 0eae30b9f..39c365391 100644 --- a/src/demo_tezos_etherlink/types/ticket_helper/tezos_parameters/default.py +++ b/src/demo_tezos_etherlink/types/ticket_helper/tezos_parameters/default.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_tezos_etherlink/types/ticket_helper/tezos_storage.py b/src/demo_tezos_etherlink/types/ticket_helper/tezos_storage.py index 79c9e4ec4..08dd88575 100644 --- a/src/demo_tezos_etherlink/types/ticket_helper/tezos_storage.py +++ b/src/demo_tezos_etherlink/types/ticket_helper/tezos_storage.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_tezos_etherlink/types/ticketer/tezos_parameters/deposit.py b/src/demo_tezos_etherlink/types/ticketer/tezos_parameters/deposit.py index 692d2a3b9..5f71ecbe6 100644 --- a/src/demo_tezos_etherlink/types/ticketer/tezos_parameters/deposit.py +++ b/src/demo_tezos_etherlink/types/ticketer/tezos_parameters/deposit.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_tezos_etherlink/types/ticketer/tezos_parameters/withdraw.py b/src/demo_tezos_etherlink/types/ticketer/tezos_parameters/withdraw.py index ddd95ae0c..f443426e6 100644 --- a/src/demo_tezos_etherlink/types/ticketer/tezos_parameters/withdraw.py +++ b/src/demo_tezos_etherlink/types/ticketer/tezos_parameters/withdraw.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_tezos_etherlink/types/ticketer/tezos_storage.py b/src/demo_tezos_etherlink/types/ticketer/tezos_storage.py index c40b65e68..dde0a9ec7 100644 --- a/src/demo_tezos_etherlink/types/ticketer/tezos_storage.py +++ b/src/demo_tezos_etherlink/types/ticketer/tezos_storage.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_tezos_events/pyproject.toml b/src/demo_tezos_events/pyproject.toml index 63ff4f9e3..8c59a7473 100644 --- a/src/demo_tezos_events/pyproject.toml +++ b/src/demo_tezos_events/pyproject.toml @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 [project] name = "demo_tezos_events" version = "0.0.1" diff --git a/src/demo_tezos_events/types/events_contract/tezos_events/move.py b/src/demo_tezos_events/types/events_contract/tezos_events/move.py index 055c540ed..142694422 100644 --- a/src/demo_tezos_events/types/events_contract/tezos_events/move.py +++ b/src/demo_tezos_events/types/events_contract/tezos_events/move.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_tezos_events/types/events_contract/tezos_events/roll.py b/src/demo_tezos_events/types/events_contract/tezos_events/roll.py index a36ec627a..b4aa08e64 100644 --- a/src/demo_tezos_events/types/events_contract/tezos_events/roll.py +++ b/src/demo_tezos_events/types/events_contract/tezos_events/roll.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_tezos_factories/pyproject.toml b/src/demo_tezos_factories/pyproject.toml index 9167e733e..a26769bcc 100644 --- a/src/demo_tezos_factories/pyproject.toml +++ b/src/demo_tezos_factories/pyproject.toml @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 [project] name = "demo_tezos_factories" version = "0.0.1" diff --git a/src/demo_tezos_factories/types/factory/tezos_storage.py b/src/demo_tezos_factories/types/factory/tezos_storage.py index c2292857c..06127272a 100644 --- a/src/demo_tezos_factories/types/factory/tezos_storage.py +++ b/src/demo_tezos_factories/types/factory/tezos_storage.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_tezos_factories/types/token/tezos_parameters/transfer.py b/src/demo_tezos_factories/types/token/tezos_parameters/transfer.py index 7676b58ac..8acb67ae1 100644 --- a/src/demo_tezos_factories/types/token/tezos_parameters/transfer.py +++ b/src/demo_tezos_factories/types/token/tezos_parameters/transfer.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_tezos_factories/types/token/tezos_storage.py b/src/demo_tezos_factories/types/token/tezos_storage.py index c440341d1..1e02ea6cb 100644 --- a/src/demo_tezos_factories/types/token/tezos_storage.py +++ b/src/demo_tezos_factories/types/token/tezos_storage.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_tezos_head/pyproject.toml b/src/demo_tezos_head/pyproject.toml index 2c19dc7b4..710847c25 100644 --- a/src/demo_tezos_head/pyproject.toml +++ b/src/demo_tezos_head/pyproject.toml @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 [project] name = "demo_tezos_head" version = "0.0.1" diff --git a/src/demo_tezos_nft_marketplace/pyproject.toml b/src/demo_tezos_nft_marketplace/pyproject.toml index e42ad38f1..7056d204b 100644 --- a/src/demo_tezos_nft_marketplace/pyproject.toml +++ b/src/demo_tezos_nft_marketplace/pyproject.toml @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 [project] name = "demo_tezos_nft_marketplace" version = "0.0.1" diff --git a/src/demo_tezos_nft_marketplace/types/hen_minter/tezos_parameters/cancel_swap.py b/src/demo_tezos_nft_marketplace/types/hen_minter/tezos_parameters/cancel_swap.py index 42f2723d1..f4e4f265d 100644 --- a/src/demo_tezos_nft_marketplace/types/hen_minter/tezos_parameters/cancel_swap.py +++ b/src/demo_tezos_nft_marketplace/types/hen_minter/tezos_parameters/cancel_swap.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_tezos_nft_marketplace/types/hen_minter/tezos_parameters/collect.py b/src/demo_tezos_nft_marketplace/types/hen_minter/tezos_parameters/collect.py index 8ee628aa7..41ff33ae6 100644 --- a/src/demo_tezos_nft_marketplace/types/hen_minter/tezos_parameters/collect.py +++ b/src/demo_tezos_nft_marketplace/types/hen_minter/tezos_parameters/collect.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_tezos_nft_marketplace/types/hen_minter/tezos_parameters/mint_objkt.py b/src/demo_tezos_nft_marketplace/types/hen_minter/tezos_parameters/mint_objkt.py index 3c6ecf325..d1116c952 100644 --- a/src/demo_tezos_nft_marketplace/types/hen_minter/tezos_parameters/mint_objkt.py +++ b/src/demo_tezos_nft_marketplace/types/hen_minter/tezos_parameters/mint_objkt.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_tezos_nft_marketplace/types/hen_minter/tezos_parameters/swap.py b/src/demo_tezos_nft_marketplace/types/hen_minter/tezos_parameters/swap.py index a8fc3fd4b..64dc0e29e 100644 --- a/src/demo_tezos_nft_marketplace/types/hen_minter/tezos_parameters/swap.py +++ b/src/demo_tezos_nft_marketplace/types/hen_minter/tezos_parameters/swap.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_tezos_nft_marketplace/types/hen_minter/tezos_storage.py b/src/demo_tezos_nft_marketplace/types/hen_minter/tezos_storage.py index 90cf90630..8125846e6 100644 --- a/src/demo_tezos_nft_marketplace/types/hen_minter/tezos_storage.py +++ b/src/demo_tezos_nft_marketplace/types/hen_minter/tezos_storage.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_tezos_nft_marketplace/types/hen_objkts/tezos_parameters/mint.py b/src/demo_tezos_nft_marketplace/types/hen_objkts/tezos_parameters/mint.py index 0e0e443d0..e2d96f181 100644 --- a/src/demo_tezos_nft_marketplace/types/hen_objkts/tezos_parameters/mint.py +++ b/src/demo_tezos_nft_marketplace/types/hen_objkts/tezos_parameters/mint.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_tezos_nft_marketplace/types/hen_objkts/tezos_storage.py b/src/demo_tezos_nft_marketplace/types/hen_objkts/tezos_storage.py index 5502240e5..6e996918e 100644 --- a/src/demo_tezos_nft_marketplace/types/hen_objkts/tezos_storage.py +++ b/src/demo_tezos_nft_marketplace/types/hen_objkts/tezos_storage.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_tezos_raw/pyproject.toml b/src/demo_tezos_raw/pyproject.toml index 38e926caa..a676e574b 100644 --- a/src/demo_tezos_raw/pyproject.toml +++ b/src/demo_tezos_raw/pyproject.toml @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 [project] name = "demo_tezos_raw" version = "0.0.1" diff --git a/src/demo_tezos_token/pyproject.toml b/src/demo_tezos_token/pyproject.toml index 1ad0d4d49..19d427ac6 100644 --- a/src/demo_tezos_token/pyproject.toml +++ b/src/demo_tezos_token/pyproject.toml @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 [project] name = "demo_tezos_token" version = "0.0.1" diff --git a/src/demo_tezos_token/types/tzbtc/tezos_parameters/mint.py b/src/demo_tezos_token/types/tzbtc/tezos_parameters/mint.py index 1fdadfa0c..fdcc8a437 100644 --- a/src/demo_tezos_token/types/tzbtc/tezos_parameters/mint.py +++ b/src/demo_tezos_token/types/tzbtc/tezos_parameters/mint.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_tezos_token/types/tzbtc/tezos_parameters/transfer.py b/src/demo_tezos_token/types/tzbtc/tezos_parameters/transfer.py index d701723ce..8634af799 100644 --- a/src/demo_tezos_token/types/tzbtc/tezos_parameters/transfer.py +++ b/src/demo_tezos_token/types/tzbtc/tezos_parameters/transfer.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_tezos_token/types/tzbtc/tezos_storage.py b/src/demo_tezos_token/types/tzbtc/tezos_storage.py index 6c6493bd4..70057a90a 100644 --- a/src/demo_tezos_token/types/tzbtc/tezos_storage.py +++ b/src/demo_tezos_token/types/tzbtc/tezos_storage.py @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 from __future__ import annotations diff --git a/src/demo_tezos_token_balances/pyproject.toml b/src/demo_tezos_token_balances/pyproject.toml index b3d8ec8ff..d5aeee79f 100644 --- a/src/demo_tezos_token_balances/pyproject.toml +++ b/src/demo_tezos_token_balances/pyproject.toml @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 [project] name = "demo_tezos_token_balances" version = "0.0.1" diff --git a/src/demo_tezos_token_transfers/pyproject.toml b/src/demo_tezos_token_transfers/pyproject.toml index 2be3d9386..23ced8c40 100644 --- a/src/demo_tezos_token_transfers/pyproject.toml +++ b/src/demo_tezos_token_transfers/pyproject.toml @@ -1,4 +1,4 @@ -# generated by DipDup 8.1.3 +# generated by DipDup 8.2.0rc1 [project] name = "demo_tezos_token_transfers" version = "0.0.1" diff --git a/src/dipdup/config/abi_etherscan.py b/src/dipdup/config/abi_etherscan.py index 8e423501f..02028bc1c 100644 --- a/src/dipdup/config/abi_etherscan.py +++ b/src/dipdup/config/abi_etherscan.py @@ -1,2 +1,4 @@ # NOTE: Alias, remove in 9.0 -from dipdup.config.evm_etherscan import EvmEtherscanDatasourceConfig as EvmEtherscanDatasourceConfig +from dipdup.config.evm_etherscan import EvmEtherscanDatasourceConfig as AbiEtherscanDatasourceConfig + +__all__ = ('AbiEtherscanDatasourceConfig',) diff --git a/src/dipdup/config/evm_etherscan.py b/src/dipdup/config/evm_etherscan.py index 834c72e62..2ff8ba722 100644 --- a/src/dipdup/config/evm_etherscan.py +++ b/src/dipdup/config/evm_etherscan.py @@ -1,5 +1,6 @@ from __future__ import annotations +import logging from typing import Literal from pydantic import ConfigDict @@ -8,6 +9,8 @@ from dipdup.config import DatasourceConfig from dipdup.config import HttpConfig +_logger = logging.getLogger(__name__) + @dataclass(config=ConfigDict(extra='forbid', defer_build=True), kw_only=True) class EvmEtherscanDatasourceConfig(DatasourceConfig): @@ -25,3 +28,9 @@ class EvmEtherscanDatasourceConfig(DatasourceConfig): api_key: str | None = None http: HttpConfig | None = None + + def __post_init__(self) -> None: + if self.kind == 'abi.etherscan': + _logger.warning( + '`abi.etherscan` datasource has been renamed to `evm.etherscan`. Please, update your config.' + ) diff --git a/src/dipdup/datasources/__init__.py b/src/dipdup/datasources/__init__.py index aa8acd06a..6cda44f4a 100644 --- a/src/dipdup/datasources/__init__.py +++ b/src/dipdup/datasources/__init__.py @@ -8,6 +8,7 @@ from typing import TypeVar from uuid import uuid4 +from pysignalr.exceptions import ConnectionError from pysignalr.messages import CompletionMessage from dipdup.config import DatasourceConfig @@ -160,6 +161,22 @@ def _get_ws_client(self) -> WebsocketTransport: return self._ws_client + async def _ws_loop(self) -> None: + self._logger.info('Establishing realtime connection') + client = self._get_ws_client() + retry_sleep = self._http_config.retry_sleep + + for _ in range(1, self._http_config.retry_count + 1): + try: + await client.run() + except ConnectionError as e: + self._logger.error('Websocket connection error: %s', e) + await self.emit_disconnected() + await asyncio.sleep(retry_sleep) + retry_sleep *= self._http_config.retry_multiplier + + raise DatasourceError('Websocket connection failed', self.name) + # FIXME: Not necessary a WS datasource class JsonRpcDatasource(WebsocketDatasource[DatasourceConfigT]): diff --git a/src/dipdup/datasources/_aiosubstrate.py b/src/dipdup/datasources/_aiosubstrate.py new file mode 100644 index 000000000..3fbaf49e5 --- /dev/null +++ b/src/dipdup/datasources/_aiosubstrate.py @@ -0,0 +1,38 @@ +from collections.abc import Callable +from typing import Any + +from aiosubstrate import SubstrateInterface + +from dipdup.datasources import JsonRpcDatasource + + +class SubstrateInterfaceProxy(SubstrateInterface): + def __init__(self, datasource: JsonRpcDatasource[Any]) -> None: + super().__init__(datasource.url) # type: ignore[no-untyped-call] + self._datasource = datasource + + async def http_request( + self, + method: str, + params: list[Any], + ) -> Any: + return await self._datasource._jsonrpc_request( + method=method, + params=params, + raw=True, + ws=False, + ) + + async def websocket_request( + self, + method: str, + params: Any, + result_handler: Callable[..., Any] | None = None, + ) -> Any: + assert not result_handler + return await self._datasource._jsonrpc_request( + method=method, + params=params, + raw=True, + ws=True, + ) diff --git a/src/dipdup/datasources/evm_node.py b/src/dipdup/datasources/evm_node.py index 064b8f2c8..0e61d8290 100644 --- a/src/dipdup/datasources/evm_node.py +++ b/src/dipdup/datasources/evm_node.py @@ -10,9 +10,6 @@ from typing import TYPE_CHECKING from typing import Any -import pysignalr -import pysignalr.exceptions - from dipdup.config import HttpConfig from dipdup.config.evm_node import EvmNodeDatasourceConfig from dipdup.datasources import JsonRpcDatasource @@ -164,22 +161,6 @@ async def _emitter_loop(self) -> None: del self._level_data[head.hash] - async def _ws_loop(self) -> None: - self._logger.info('Establishing realtime connection') - client = self._get_ws_client() - retry_sleep = self._http_config.retry_sleep - - for _ in range(1, self._http_config.retry_count + 1): - try: - await client.run() - except pysignalr.exceptions.ConnectionError as e: - self._logger.error('Websocket connection error: %s', e) - await self.emit_disconnected() - await asyncio.sleep(retry_sleep) - retry_sleep *= self._http_config.retry_multiplier - - raise DatasourceError('Websocket connection failed', self.name) - @property def ws_available(self) -> bool: return self._config.ws_url is not None diff --git a/src/dipdup/datasources/substrate_node.py b/src/dipdup/datasources/substrate_node.py index 2421641e6..35ba53053 100644 --- a/src/dipdup/datasources/substrate_node.py +++ b/src/dipdup/datasources/substrate_node.py @@ -4,15 +4,16 @@ from asyncio import Queue from collections.abc import Awaitable from collections.abc import Callable +from contextlib import suppress from copy import copy from dataclasses import dataclass from dataclasses import field -from functools import partial +from functools import cached_property from pathlib import Path +from typing import TYPE_CHECKING from typing import Any import orjson -import pysignalr.exceptions from dipdup.config import HttpConfig from dipdup.config.substrate_node import SubstrateNodeDatasourceConfig @@ -27,7 +28,9 @@ from dipdup.models.substrate_node import SubstrateNodeSubscription from dipdup.pysignalr import Message from dipdup.pysignalr import WebsocketMessage -from dipdup.utils import Watchdog + +if TYPE_CHECKING: + from aiosubstrate.base import SubstrateInterface _logger = logging.getLogger(__name__) @@ -92,24 +95,16 @@ def save_file(self) -> None: class SubstrateNodeDatasource(JsonRpcDatasource[SubstrateNodeDatasourceConfig]): _default_http_config = HttpConfig( - batch_size=20, + batch_size=10, ) def __init__(self, config: SubstrateNodeDatasourceConfig) -> None: - from aiosubstrate.base import SubstrateInterface - - # NOTE: Use our aiohttp session and limiters - SubstrateInterface.http_request = partial(self._jsonrpc_request, raw=True) # type: ignore[method-assign] - super().__init__(config) self._pending_subscription: SubstrateNodeSubscription | None = None self._subscription_ids: dict[str, SubstrateNodeSubscription] = {} - self._interface = SubstrateInterface(config.url) # type: ignore[no-untyped-call] self._emitter_queue: Queue[SubscriptionMessage] = Queue() - self._watchdog: Watchdog = Watchdog(self._http_config.connection_timeout) - self._on_head_callbacks: set[HeadCallback] = set() self._on_event_callbacks: set[EventCallback] = set() @@ -118,7 +113,6 @@ async def run(self) -> None: await asyncio.gather( self._ws_loop(), self._emitter_loop(), - # self._watchdog.run(), ) else: while True: @@ -134,22 +128,11 @@ async def initialize(self) -> None: await self._interface.init_props() # type: ignore[no-untyped-call] self._interface.reload_type_registry() - async def _ws_loop(self) -> None: - # TODO: probably add to inheritance WebsocketSubscriptionDatasource, and move this method there - self._logger.info('Establishing realtime connection') - client = self._get_ws_client() - retry_sleep = self._http_config.retry_sleep + @cached_property + def _interface(self) -> 'SubstrateInterface': + from dipdup.datasources._aiosubstrate import SubstrateInterfaceProxy - for _ in range(1, self._http_config.retry_count + 1): - try: - await client.run() - except pysignalr.exceptions.ConnectionError as e: - self._logger.error('Websocket connection error: %s', e) - await self.emit_disconnected() - await asyncio.sleep(retry_sleep) - retry_sleep *= self._http_config.retry_multiplier - - raise DatasourceError('Websocket connection failed', self.name) + return SubstrateInterfaceProxy(self) @property def ws_available(self) -> bool: @@ -193,8 +176,6 @@ async def _on_message(self, message: Message) -> None: # NOTE: Save subscription id if self._pending_subscription: self._subscription_ids[data['result']] = self._pending_subscription - self._requests[data['id']] = (self._requests[data['id']][0], data) - self._requests[data['id']][0].set() # NOTE: Possibly unreliable logic from evm_node, and possibly too time consuming for message handling level = await self.get_head_level() @@ -202,8 +183,10 @@ async def _on_message(self, message: Message) -> None: # NOTE: Set None to identify possible subscriptions conflicts self._pending_subscription = None - else: - raise Exception + + self._requests[data['id']] = (self._requests[data['id']][0], data) + self._requests[data['id']][0].set() + elif 'method' in data and data['method'].startswith('chain_'): subscription_id = data['params']['subscription'] if subscription_id not in self._subscription_ids: @@ -223,11 +206,10 @@ async def get_block_hash(self, height: int) -> str: async def get_block_header(self, hash: str) -> _BlockHeader: response = await self._jsonrpc_request('chain_getHeader', [hash]) - # FIXME: missing fields - return { + return { # type: ignore[typeddict-item] + **response, 'hash': hash, 'number': int(response['number'], 16), - 'prev_root': response['parentHash'], } async def get_metadata_header(self, height: int) -> MetadataHeader: @@ -247,7 +229,12 @@ async def get_full_block(self, hash: str) -> dict[str, Any]: return await self._jsonrpc_request('chain_getBlock', [hash]) # type: ignore[no-any-return] async def get_events(self, block_hash: str) -> tuple[_SubstrateNodeEventResponse, ...]: - events = await self._interface.get_events(block_hash) + # FIXME: aiosubstrate bug, fix asap + while True: + with suppress(AttributeError): + events = await self._interface.get_events(block_hash) + break + await asyncio.sleep(0.1) result: list[_SubstrateNodeEventResponse] = [] for raw_event in events: @@ -336,7 +323,7 @@ async def _emitter_loop(self) -> None: self._logger.info('New head: %s', level) await self.emit_head(level_data.head) - # NOTE: subscribing to finalized head, no rollback required + # NOTE: Subscribing to finalized head, no rollback handling required if level_data.fetch_events: block_hash = await self.get_block_hash(level) diff --git a/src/dipdup/datasources/tezos_tzkt.py b/src/dipdup/datasources/tezos_tzkt.py index 3dd9b1bc1..73df76f53 100644 --- a/src/dipdup/datasources/tezos_tzkt.py +++ b/src/dipdup/datasources/tezos_tzkt.py @@ -1,4 +1,3 @@ -import asyncio import logging from asyncio import Event from collections import defaultdict @@ -13,10 +12,8 @@ from functools import partial from typing import Any from typing import NamedTuple -from typing import NoReturn from typing import cast -import pysignalr.exceptions from pysignalr.client import SignalRClient from pysignalr.messages import CompletionMessage @@ -28,7 +25,7 @@ from dipdup.config.tezos_tzkt import TZKT_API_URLS from dipdup.config.tezos_tzkt import TezosTzktDatasourceConfig from dipdup.datasources import Datasource -from dipdup.datasources import IndexDatasource +from dipdup.datasources import WebsocketDatasource from dipdup.exceptions import DatasourceError from dipdup.exceptions import FrameworkException from dipdup.models import Head @@ -46,6 +43,7 @@ from dipdup.models.tezos_tzkt import HeadSubscription from dipdup.models.tezos_tzkt import TezosTzktMessageType from dipdup.models.tezos_tzkt import TezosTzktSubscription +from dipdup.pysignalr import WebsocketTransport from dipdup.utils import split_by_chunks ORIGINATION_REQUEST_LIMIT = 100 @@ -231,7 +229,7 @@ def get_address(self, code_hash: int, type_hash: int) -> str: return self._hashes_to_address[(code_hash, type_hash)] -class TezosTzktDatasource(IndexDatasource[TezosTzktDatasourceConfig]): +class TezosTzktDatasource(WebsocketDatasource[TezosTzktDatasourceConfig]): _default_http_config = HttpConfig( retry_sleep=1, retry_multiplier=1.1, @@ -281,22 +279,8 @@ async def __aenter__(self) -> None: def request_limit(self) -> int: return self._http_config.batch_size - # FIXME: Join retry logic with other index datasources async def run(self) -> None: - self._logger.info('Establishing realtime connection') - signalr_client = self._get_signalr_client() - retry_sleep = self._http_config.retry_sleep - - for _ in range(1, self._http_config.retry_count + 1): - try: - await signalr_client.run() - except pysignalr.exceptions.ConnectionError as e: - self._logger.error('Websocket connection error: %s', e) - await self.emit_disconnected() - await asyncio.sleep(retry_sleep) - retry_sleep *= self._http_config.retry_multiplier - - raise DatasourceError('Websocket connection failed', self.name) + await self._ws_loop() async def initialize(self) -> None: head_block = await self.get_head_block() @@ -1201,6 +1185,9 @@ async def _iter_batches( else: offset += self.request_limit + def _get_ws_client(self) -> WebsocketTransport: + return self._get_signalr_client() # type: ignore[return-value] + def _get_signalr_client(self) -> SignalRClient: """Create SignalR client, register message callbacks""" if self._signalr_client: @@ -1235,23 +1222,7 @@ async def _send( client = self._get_signalr_client() await client.send(method, arguments, on_invocation) # type: ignore[arg-type] - async def _on_connected(self) -> None: - self._logger.info('Realtime connection established') - # NOTE: Subscribing here will block WebSocket loop, don't do it. - await self.emit_connected() - - async def _on_disconnected(self) -> None: - self._logger.info('Realtime connection lost, resetting subscriptions') - self._subscriptions.reset() - # NOTE: Just in case - self._contract_hashes.reset() - await self.emit_disconnected() - - async def _on_error(self, message: CompletionMessage) -> NoReturn: - """Raise exception from WS server's error message""" - raise DatasourceError(datasource=self.name, msg=cast(str, message.error)) - - async def _on_message(self, type_: TezosTzktMessageType, message: list[dict[str, Any]]) -> None: + async def _on_message(self, type_: TezosTzktMessageType, message: list[dict[str, Any]]) -> None: # type: ignore[override] """Parse message received from Websocket, ensure it's correct in the current context and yield data.""" # NOTE: Parse messages and either buffer or yield data for item in message: diff --git a/src/dipdup/indexes/_subsquid.py b/src/dipdup/indexes/_subsquid.py index 9c93883e9..9c9210e3e 100644 --- a/src/dipdup/indexes/_subsquid.py +++ b/src/dipdup/indexes/_subsquid.py @@ -46,6 +46,8 @@ async def _get_node_sync_level( node = node or random.choice(self.node_datasources) node_sync_level = await node.get_head_level() + node._logger.info('current head is %s', node_sync_level) + subsquid_lag = abs(node_sync_level - subsquid_level) subsquid_available = subsquid_level - index_level self._logger.info('Subsquid is %s levels behind; %s available', subsquid_lag, subsquid_available) @@ -64,7 +66,9 @@ async def _synchronize(self, sync_level: int) -> None: return if self.subsquid_datasources: - subsquid_sync_level = await self.subsquid_datasources[0].get_head_level() + datasource = self.subsquid_datasources[0] + subsquid_sync_level = await datasource.get_head_level() + datasource._logger.info('current head is %s', subsquid_sync_level) metrics._sqd_processor_chain_height = subsquid_sync_level else: subsquid_sync_level = 0 @@ -74,15 +78,16 @@ async def _synchronize(self, sync_level: int) -> None: # NOTE: Fetch last blocks from node if there are not enough realtime messages in queue if node_sync_level: sync_level = min(sync_level, node_sync_level) - self._logger.debug('Using node datasource; sync level: %s', sync_level) + self._logger.info('Synchronizing with `node`: %s -> %s', index_level, sync_level) await self._synchronize_node(sync_level) else: sync_level = min(sync_level, subsquid_sync_level) + self._logger.info('Synchronizing with `subsquid`: %s -> %s', index_level, sync_level) await self._synchronize_subsquid(sync_level) if not self.node_datasources and not self._subsquid_started: self._subsquid_started = True - self._logger.info('No `evm.node` datasources available; polling Subsquid') + self._logger.info('No `node` datasources available; polling Subsquid') for datasource in self.subsquid_datasources: await datasource.start() diff --git a/src/dipdup/indexes/substrate_events/fetcher.py b/src/dipdup/indexes/substrate_events/fetcher.py index ba647f034..a56ae3710 100644 --- a/src/dipdup/indexes/substrate_events/fetcher.py +++ b/src/dipdup/indexes/substrate_events/fetcher.py @@ -1,4 +1,8 @@ +import asyncio from collections.abc import AsyncIterator +from collections.abc import Iterable +from contextlib import suppress +from typing import Any from dipdup.datasources.substrate_node import SubstrateNodeDatasource from dipdup.datasources.substrate_subsquid import SubstrateSubsquidDatasource @@ -57,8 +61,115 @@ async def fetch_by_level(self) -> AsyncIterator[tuple[int, tuple[SubstrateEventD yield level, events async def fetch_events(self) -> AsyncIterator[tuple[SubstrateEventData, ...]]: - for level in range(self._first_level, self._last_level): - block_hash = await self.get_random_node().get_block_hash(level) - event_dicts = await self.get_random_node().get_events(block_hash) - block_header = await self.get_random_node().get_block_header(block_hash) - yield tuple(SubstrateEventData.from_node(event_dict, block_header) for event_dict in event_dicts) + node = self.get_random_node() + batch_size = node._http_config.batch_size + queue_limit = 50 + + queues: dict[str, asyncio.Queue[Any]] = { + 'levels': asyncio.Queue(), + 'hashes': asyncio.Queue(), + 'headers': asyncio.Queue(), + 'events': asyncio.Queue(), + } + + for level in range(self._first_level, self._last_level + 1): + await queues['levels'].put(level) + + async def _hashes_loop() -> None: + async def _batch(levels: Iterable[int]) -> None: + block_hashes = await asyncio.gather( + *(node.get_block_hash(level) for level in levels), + ) + for block_hash in block_hashes: + await queues['hashes'].put(block_hash) + if queues['hashes'].qsize() >= queue_limit: + await asyncio.sleep(1) + + batch = [] + while queues['levels'].qsize() > 0: + batch.append(await queues['levels'].get()) + if len(batch) >= batch_size: + await _batch(batch) + batch = [] + + if batch: + await _batch(batch) + + async def _headers_loop() -> None: + async def _batch(hashes: Iterable[str]) -> None: + block_headers = await asyncio.gather( + *(node.get_block_header(hash_) for hash_ in hashes), + ) + for block_header in block_headers: + await queues['headers'].put(block_header) + if queues['headers'].qsize() >= queue_limit: + await asyncio.sleep(1) + + batch = [] + while True: + block_hash = await queues['hashes'].get() + batch.append(block_hash) + if len(batch) >= batch_size: + await _batch(batch) + batch = [] + + if queues['levels'].qsize() == 0 and queues['hashes'].qsize() == 0: + break + + if batch: + await _batch(batch) + + async def _events_loop() -> None: + async def _batch(headers: Iterable[dict[str, Any]]) -> None: + block_events = await asyncio.gather( + *(node.get_events(header['hash']) for header in headers), + ) + for header, events in zip(headers, block_events, strict=True): + await queues['events'].put((header, events)) + + batch = [] + while True: + block_header = await queues['headers'].get() + batch.append(block_header) + if len(batch) >= batch_size: + await _batch(batch) + batch = [] + + if queues['levels'].qsize() == 0 and queues['hashes'].qsize() == 0 and queues['headers'].qsize() == 0: + break + + if batch: + await _batch(batch) + + async def _log_loop() -> None: + while True: + await asyncio.sleep(1) + self._logger.debug( + 'queues: levels=%d hashes=%d headers=%d events=%d', + queues['levels'].qsize(), + queues['hashes'].qsize(), + queues['headers'].qsize(), + queues['events'].qsize(), + ) + + tasks = ( + asyncio.create_task(_hashes_loop()), + asyncio.create_task(_headers_loop()), + asyncio.create_task(_events_loop()), + asyncio.create_task(_log_loop()), + ) + + while True: + if sum(queues[q].qsize() for q in queues) == 0 and all(t.done() for t in tasks[:-1]): + break + + for t in tasks: + if t.done() or t.cancelled(): + await t + + with suppress(asyncio.TimeoutError): + while True: + header, events = await asyncio.wait_for(queues['events'].get(), timeout=1) + yield tuple(SubstrateEventData.from_node(event, header) for event in events) + + tasks[-1].cancel() diff --git a/src/dipdup/models/substrate.py b/src/dipdup/models/substrate.py index 947a892c6..f3608abb2 100644 --- a/src/dipdup/models/substrate.py +++ b/src/dipdup/models/substrate.py @@ -11,13 +11,23 @@ from dipdup.runtimes import SubstrateRuntime -class _BlockHeaderExtra(TypedDict): +class _BlockHeader(TypedDict): + hash: str number: int + parentHash: str + stateRoot: str + extrinsicsRoot: str + digest: dict[str, dict[str, list[str]]] + + +class _BlockHeaderExtra(TypedDict): hash: str + number: int parentHash: str stateRoot: str extrinsicsRoot: str - digest: str + digest: dict[str, dict[str, list[str]]] + specName: str specVersion: int implName: str @@ -35,12 +45,6 @@ class _SubstrateSubsquidEventResponse(TypedDict): header: _BlockHeaderExtra -class _BlockHeader(TypedDict): - hash: str - number: int - prev_root: str - - class _SubstrateNodeEventResponse(TypedDict): name: str index: int @@ -55,7 +59,6 @@ class SubstrateEventData(HasLevel): index: int extrinsic_index: int call_address: list[str] | None - # we receive decoded args from node datasource and encoded from subsquid datasource args: list[Any] | None = None decoded_args: dict[str, Any] | None = None header: _BlockHeader @@ -84,11 +87,7 @@ def from_subsquid(cls, event_dict: _SubstrateSubsquidEventResponse) -> Self: call_address=event_dict['callAddress'], args=event_dict['args'], decoded_args=None, - header={ - 'hash': event_dict['header']['hash'], - 'number': event_dict['header']['number'], - 'prev_root': event_dict['header']['parentHash'], - }, + header=event_dict['header'], header_extra=event_dict['header'], ) @@ -112,20 +111,18 @@ class SubstrateEvent(Generic[PayloadT]): # TODO: Use lazy decoding in other models with typed payload @cached_property def payload(self) -> PayloadT: - # NOTE: from node datasource + # NOTE: We receive decoded args from node datasource and encoded from subsquid datasource if self.data.decoded_args is not None: - return cast(PayloadT, self.data.decoded_args) - - # NOTE: from subsquid datasource - assert self.data.args is not None and self.data.header_extra is not None - return cast( - PayloadT, - self.runtime.decode_event_args( + payload = self.data.decoded_args + elif self.data.args is not None and self.data.header_extra is not None: + payload = self.runtime.decode_event_args( name=self.name, args=self.data.args, spec_version=str(self.data.header_extra['specVersion']), - ), - ) + ) + else: + raise NotImplementedError + return cast(PayloadT, payload) @property def level(self) -> int: diff --git a/src/dipdup/pysignalr.py b/src/dipdup/pysignalr.py index 98267ab1d..8d76e78b0 100644 --- a/src/dipdup/pysignalr.py +++ b/src/dipdup/pysignalr.py @@ -1,53 +1,10 @@ """ -Turning pysignalr back into a basic websocket client usable for JSONRPC nodes, but with state management and reconnection logic. - -Eventually this code will be moved to the upstream library. +Compatibility aliases for the `pysignalr` package. """ -import asyncio -from collections.abc import Iterable -from typing import Any - -import orjson -from pysignalr.messages import HandshakeRequestMessage -from pysignalr.messages import HandshakeResponseMessage +from pysignalr.messages import JSONMessage as WebsocketMessage from pysignalr.messages import Message as Message -from pysignalr.messages import MessageType -from pysignalr.protocol.abstract import Protocol -from pysignalr.transport.websocket import WebsocketTransport as SignalRWebsocketTransport -from websockets.client import WebSocketClientProtocol - -KEEPALIVE_INTERVAL = 5 - - -class WebsocketMessage(Message, type_=MessageType.invocation): - def __init__(self, data: dict[str, Any]) -> None: - self.data = data - - def dump(self) -> dict[str, Any]: - return self.data - - -class WebsocketTransport(SignalRWebsocketTransport): - async def _keepalive(self, conn: WebSocketClientProtocol) -> None: - while True: - await conn.ensure_open() - await asyncio.sleep(KEEPALIVE_INTERVAL) - - async def _handshake(self, conn: WebSocketClientProtocol) -> None: - return - - -class WebsocketProtocol(Protocol): - def __init__(self) -> None: - pass - - def decode(self, raw_message: str | bytes) -> tuple[WebsocketMessage]: - json_message = orjson.loads(raw_message) - return (WebsocketMessage(data=json_message),) - - def encode(self, message: Message | HandshakeRequestMessage) -> str | bytes: - return orjson.dumps(message.dump()) +from pysignalr.protocol.json import BaseJSONProtocol as WebsocketProtocol +from pysignalr.transport.websocket import BaseWebsocketTransport as WebsocketTransport - def decode_handshake(self, raw_message: str | bytes) -> tuple[HandshakeResponseMessage, Iterable[Message]]: - raise NotImplementedError +__all__ = ['Message', 'WebsocketMessage', 'WebsocketProtocol', 'WebsocketTransport'] diff --git a/tests/configs/demo_starknet_events.yml b/tests/configs/demo_starknet_events.yml index e4afe7026..4164d2507 100644 --- a/tests/configs/demo_starknet_events.yml +++ b/tests/configs/demo_starknet_events.yml @@ -4,10 +4,10 @@ package: demo_starknet_events datasources: subsquid: kind: starknet.subsquid - url: https://v2.archive.subsquid.io/network/starknet-mainnet + url: ${SUBSQUID_URL:-https://v2.archive.subsquid.io/network/starknet-mainnet} node: kind: starknet.node - url: https://starknet-mainnet.g.alchemy.com/v2 + url: ${NODE_URL:-https://starknet-mainnet.g.alchemy.com/v2}/${ALCHEMY_API_KEY:-''} contracts: stark_usdt: @@ -25,3 +25,5 @@ indexes: - callback: on_transfer contract: stark_usdt name: Transfer + first_level: 1005479 + last_level: 1005489 \ No newline at end of file diff --git a/tests/configs/demo_substrate_events.yml b/tests/configs/demo_substrate_events.yml new file mode 100644 index 000000000..63d921666 --- /dev/null +++ b/tests/configs/demo_substrate_events.yml @@ -0,0 +1,33 @@ +spec_version: 3.0 +package: demo_substrate_events + +runtimes: + assethub: + kind: substrate + type_registry: statemint + +datasources: + subsquid: + kind: substrate.subsquid + url: https://v2.archive.subsquid.io/network/asset-hub-polkadot + subscan: + kind: substrate.subscan + url: https://assethub-polkadot.api.subscan.io/api + node: + kind: substrate.node + url: https://statemint.api.onfinality.io/rpc?apikey=${ONFINALITY_API_KEY:-''} + ws_url: wss://statemint.api.onfinality.io/ws?apikey=${ONFINALITY_API_KEY:-''} + +indexes: + assethub_transfers: + kind: substrate.events + runtime: assethub + datasources: + - subsquid + - subscan + - node + handlers: + - callback: on_transfer + name: Assets.Transferred + first_level: 7836000 + last_level: 7836100 diff --git a/tests/configs/test_evm.yml b/tests/configs/test_evm.yml index 54cd349d5..678b1401f 100644 --- a/tests/configs/test_evm.yml +++ b/tests/configs/test_evm.yml @@ -8,5 +8,5 @@ datasources: api_key: ${ETHERSCAN_API_KEY:-''} evm_node: kind: evm.node - url: ${NODE_URL:-https://eth-mainnet.g.alchemy.com/v2}/${NODE_API_KEY:-''} - ws_url: ${NODE_WS_URL:-wss://eth-mainnet.g.alchemy.com/v2}/${NODE_API_KEY:-''} \ No newline at end of file + url: ${NODE_URL:-https://eth-mainnet.g.alchemy.com/v2}/${ALCHEMY_API_KEY:-''} + ws_url: ${NODE_WS_URL:-wss://eth-mainnet.g.alchemy.com/v2}/${ALCHEMY_API_KEY:-''} \ No newline at end of file diff --git a/tests/test_demos.py b/tests/test_demos.py index 297a56a5c..a92ce8f37 100644 --- a/tests/test_demos.py +++ b/tests/test_demos.py @@ -148,6 +148,20 @@ async def assert_run_evm_transactions() -> None: assert holders == 22 +async def assert_run_starknet_events() -> None: + import demo_starknet_events.models + + holders = await demo_starknet_events.models.Holder.filter().count() + assert holders == 15 + + +async def assert_run_substrate_events() -> None: + import demo_substrate_events.models + + holders = await demo_substrate_events.models.Holder.filter().count() + assert holders == 11 + + async def assert_run_dao() -> None: import demo_tezos_dao.models @@ -160,18 +174,35 @@ async def assert_run_dao() -> None: test_args = ('config', 'package', 'cmd', 'assert_fn') test_params = ( - ('demo_tezos_token.yml', 'demo_tezos_token', 'run', assert_run_token), - ('demo_tezos_token.yml', 'demo_tezos_token', 'init', None), - ('demo_tezos_nft_marketplace.yml', 'demo_tezos_nft_marketplace', 'run', assert_run_nft_marketplace), - ('demo_tezos_nft_marketplace.yml', 'demo_tezos_nft_marketplace', 'init', None), + # NOTE: Tezos ('demo_tezos_auction.yml', 'demo_tezos_auction', 'run', assert_run_auction), ('demo_tezos_auction.yml', 'demo_tezos_auction', 'init', None), + ('demo_tezos_dao.yml', 'demo_tezos_dao', 'run', assert_run_dao), + ('demo_tezos_dao.yml', 'demo_tezos_dao', 'init', None), + ('demo_tezos_dex.yml', 'demo_tezos_dex', 'run', assert_run_dex), + ('demo_tezos_dex.yml', 'demo_tezos_dex', 'init', None), + ('demo_tezos_domains.yml', 'demo_tezos_domains', 'run', assert_run_domains), + ('demo_tezos_domains.yml', 'demo_tezos_domains', 'init', None), + ('demo_tezos_etherlink.yml', 'demo_tezos_etherlink', 'run', None), + ('demo_tezos_etherlink.yml', 'demo_tezos_etherlink', 'init', None), + ('demo_tezos_events.yml', 'demo_tezos_events', 'run', assert_run_events), + ('demo_tezos_events.yml', 'demo_tezos_events', 'init', None), + ('demo_tezos_factories.yml', 'demo_tezos_factories', 'run', assert_run_factories), + ('demo_tezos_factories.yml', 'demo_tezos_factories', 'init', None), + ('demo_tezos_nft_marketplace.yml', 'demo_tezos_nft_marketplace', 'run', assert_run_nft_marketplace), + ('demo_tezos_nft_marketplace.yml', 'demo_tezos_nft_marketplace', 'init', None), ( 'demo_tezos_token_transfers.yml', 'demo_tezos_token_transfers', 'run', partial(assert_run_token_transfers, 4, '-0.01912431'), ), + ('demo_tezos_raw.yml', 'demo_tezos_raw', 'run', assert_run_raw), + ('demo_tezos_raw.yml', 'demo_tezos_raw', 'init', None), + ('demo_tezos_token.yml', 'demo_tezos_token', 'run', assert_run_token), + ('demo_tezos_token.yml', 'demo_tezos_token', 'init', None), + ('demo_tezos_token_balances.yml', 'demo_tezos_token_balances', 'run', assert_run_balances), + ('demo_tezos_token_balances.yml', 'demo_tezos_token_balances', 'init', None), # TODO: Too many token transfer runs ('demo_tezos_token_transfers.yml', 'demo_tezos_token_transfers', 'init', None), ( @@ -193,30 +224,21 @@ async def assert_run_dao() -> None: # 'run', # partial(assert_run_token_transfers, 2, '-0.02302128'), # ), - ('demo_tezos_token_balances.yml', 'demo_tezos_token_balances', 'run', assert_run_balances), - ('demo_tezos_token_balances.yml', 'demo_tezos_token_balances', 'init', None), - ('demo_tezos_domains.yml', 'demo_tezos_domains', 'run', assert_run_domains), - ('demo_tezos_domains.yml', 'demo_tezos_domains', 'init', None), - ('demo_tezos_dex.yml', 'demo_tezos_dex', 'run', assert_run_dex), - ('demo_tezos_dex.yml', 'demo_tezos_dex', 'init', None), - ('demo_tezos_dao.yml', 'demo_tezos_dao', 'run', assert_run_dao), - ('demo_tezos_dao.yml', 'demo_tezos_dao', 'init', None), - ('demo_tezos_factories.yml', 'demo_tezos_factories', 'run', assert_run_factories), - ('demo_tezos_factories.yml', 'demo_tezos_factories', 'init', None), - ('demo_tezos_events.yml', 'demo_tezos_events', 'run', assert_run_events), - ('demo_tezos_events.yml', 'demo_tezos_events', 'init', None), - ('demo_tezos_raw.yml', 'demo_tezos_raw', 'run', assert_run_raw), - ('demo_tezos_raw.yml', 'demo_tezos_raw', 'init', None), + # NOTE: EVM indexes ('demo_evm_events.yml:test_evm.yml', 'demo_evm_events', 'run', assert_run_evm_events), ('demo_evm_events.yml:test_evm.yml', 'demo_evm_events', 'init', None), ('demo_evm_transactions.yml:test_evm.yml', 'demo_evm_transactions', 'run', assert_run_evm_transactions), ('demo_evm_transactions.yml:test_evm.yml', 'demo_evm_transactions', 'init', None), - ('demo_tezos_etherlink.yml', 'demo_tezos_etherlink', 'run', None), - ('demo_tezos_etherlink.yml', 'demo_tezos_etherlink', 'init', None), - # NOTE: Indexes with `evm.node` as index datasource + # NOTE: EVM indexes with node only ('demo_evm_events_node.yml:test_evm.yml', 'demo_evm_events', 'run', assert_run_evm_events), ('demo_evm_transactions_node.yml:test_evm.yml', 'demo_evm_transactions', 'run', assert_run_evm_transactions), - # NOTE: Smoke tests for small tools. + # NOTE: Starknet indexes + ('demo_starknet_events.yml', 'demo_starknet_events', 'run', assert_run_starknet_events), + ('demo_starknet_events.yml', 'demo_starknet_events', 'init', None), + # NOTE: Substrate indexes + ('demo_substrate_events.yml', 'demo_substrate_events', 'run', assert_run_substrate_events), + ('demo_substrate_events.yml', 'demo_substrate_events', 'init', None), + # NOTE: Smoke tests for small tools ('demo_tezos_dex.yml', 'demo_tezos_dex', ('config', 'env', '--compose', '--internal'), None), ('demo_tezos_dex.yml', 'demo_tezos_dex', ('config', 'export', '--full'), None), ('demo_tezos_dex.yml', 'demo_tezos_dex', ('package', 'tree'), None), @@ -238,8 +260,12 @@ async def test_run_init( config_paths.append(TEST_CONFIGS / path) config_paths.append(TEST_CONFIGS / 'test_sqlite.yaml') - if 'evm' in config and not {'NODE_API_KEY', 'ETHERSCAN_API_KEY'} <= set(os.environ): - pytest.skip('EVM tests require NODE_API_KEY and ETHERSCAN_API_KEY environment variables') + if 'evm' in config and not {'ALCHEMY_API_KEY', 'ETHERSCAN_API_KEY'} <= set(os.environ): + pytest.skip('EVM tests require ALCHEMY_API_KEY and ETHERSCAN_API_KEY environment variables') + if 'starknet' in config and not {'ALCHEMY_API_KEY'} <= set(os.environ): + pytest.skip('Starknet tests require ALCHEMY_API_KEY environment variable') + if 'substrate' in config and not {'ONFINALITY_API_KEY'} <= set(os.environ): + pytest.skip('Substrate tests require ONFINALITY_API_KEY environment variable') async with AsyncExitStack() as stack: tmp_package_path, env = await stack.enter_async_context( diff --git a/tests/test_index/test_tzkt_operations.py b/tests/test_index/test_tzkt_operations.py index fd67d0780..64e47982e 100644 --- a/tests/test_index/test_tzkt_operations.py +++ b/tests/test_index/test_tzkt_operations.py @@ -50,7 +50,6 @@ async def test_ignored_type_filter( assert not hashes -@pytest.mark.skip('FIXME: Pydantic 2 migration mystery') async def test_get_origination_filters( tzkt: TezosTzktDatasource, index_config: TezosOperationsIndexConfig, @@ -112,7 +111,6 @@ async def test_get_origination_filters( assert hashes == set() -@pytest.mark.skip('FIXME: Pydantic 2 migration mystery') async def test_get_transaction_filters(tzkt: TezosTzktDatasource, index_config: TezosOperationsIndexConfig) -> None: index_config.types = (TezosOperationType.transaction,) index_config.contracts[2].code_hash = -680664524