Protocol support for Rust (#2172)

Co-authored-by: Maksim Ryndin <[email protected]>
Co-authored-by: v0-e <[email protected]>

.gitignore

@@ -8,6 +8,8 @@ v4-proto-py/v4_proto
 v4-proto-js/build
 v4-proto-js/node_modules
 v4-proto-js/src
+v4-proto-rs/target
+v4-proto-rs/Cargo.lock
 
 .idea
 .vscode

v4-proto-rs/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+name = "v4-proto-rs"
+version = "0.1.0"
+edition = "2021"
+license = "AGPL-3.0"
+
+[lib]
+doctest = false
+
+[dependencies]
+cosmos-sdk-proto = "0.21.1"
+tonic = { version = "0.11", features = ["tls", "tls-roots", "transport", "channel"] }
+prost = "0.12"
+prost-types = "0.12"
+
+[build-dependencies]
+tonic-buf-build = "0.2.1"
+prost-build = "0.12" # keep the version the same as in tonic-buf-build
+tonic-build = "0.11" # keep the version the same as in tonic-buf-build

v4-proto-rs/README.md

@@ -0,0 +1,62 @@
+# Rust crate for dYdX Chain protobufs
+
+## Usage as a dependency
+
+Cargo.toml
+
+```toml
+[dependencies]
+v4-proto-rs = "0.1"
+```
+
+*Note:* by default, rust stub files are not rebuilt (see Q&A below)
+
+For more idiomatic Rust you can use conversions (`try_into` and `into`) for the following types:
+* `prost_types::Timestamp` -> `std::time::SystemTime`
+* `prost_types::Duration`-> `std::time::Duration`
+
+## Local development
+
+### Prerequisites
+1) [Rust](https://www.rust-lang.org/tools/install)
+2) [Buf](https://github.com/bufbuild/buf?tab=readme-ov-file#installation) - to resolve 3rd-party dependencies for protobuf files
+3) [protoc](https://github.com/protocolbuffers/protobuf#protobuf-compiler-installation) - to compile protobuf files with their 3rd-party dependencies
+4) [cargo deny](https://github.com/EmbarkStudios/cargo-deny) - to check for security/license/sources issues
+
+Then for a code (re-)generation run
+
+```sh
+V4_PROTO_REBUILD=1 cargo build -vv
+```
+
+Before publishing make sure to run (and fix all warnings and errors)
+
+```sh
+cargo fmt
+cargo clippy
+cargo deny check licenses advisories sources
+```
+
+## Q&A
+
+1) Why do we put autogenerated files to the crate (and git) and do not (re-)generate them at compilation?
+
+    For several reasons:
+    * reproducibility of the dependency
+    * to avoid external dependencies for the lib users (`protoc` and `buf` are only needed for code generation)
+
+    But if a user wants to (re-)generate at compilation time, he/she can set an environment variable `V4_PROTO_REBUILD` (to any value).
+
+2) Why do I need a `protoc` for this crate development? I thought `prost-build` crate generates everything natively with Rust?
+
+    The main work (parsing, linking, etc. - have a look https://protobuf.com/docs/descriptors) is done by `protoc`.
+    The result of the `protoc` work is a "file descriptor" (think of it as IR assembly language like LLVM IR) - a binary file. This file descriptor is an input for a language-specific code generator like `prost`. Think of `prost` crate as a compiler target which generates a ISA-specific "assembly" (in our case, Rust) as an output.
+    `prost-build` always used the `protoc` but since version 0.11 of [prost-build](https://github.com/tokio-rs/prost?tab=readme-ov-file#protoc) it requires `protoc` (the protobuf compiler) to be already installed on the system - before the `protoc` could be compiled during the `prost-build` build (https://github.com/tokio-rs/prost/blob/v0.10.4/prost-build/build.rs#L77).
+
+3) Why do we use `tonic-build` crate and not just `prost-build`?
+
+    `prost-build` generates only serialization-deserialization stubs for messages, but we also need a client implementation (generated by `tonic-build`) because packages in other language implementations of `v4-chain` have ones.
+
+4) Why do we need `buf`?
+
+    [Buf](https://buf.build/) is a tool whose primary function is to resolve dependencies in protobuf files. Protobuf specifications can refer to 3rd-party protobuf specifications and use types declared there. Basically, `buf` builds a list of all used protobuf files, downloads them, and allows exporting (=copying) them to a specified directory. The proto files in this repository and downloaded 3rd-party proto files (aka "includes") are an input for the `protoc`.

v4-proto-rs/build.rs

@@ -0,0 +1,28 @@
+use prost_build::Config;
+use std::env;
+use std::path::PathBuf;
+
+fn main() -> Result<(), tonic_buf_build::error::TonicBufBuildError> {
+    if std::env::var("V4_PROTO_REBUILD").is_ok() {
+        let mut config = Config::new();
+        config.out_dir("src");
+        config.include_file("_includes.rs");
+        config.enable_type_names();
+        let mut path = PathBuf::from(env::var("CARGO_MANIFEST_DIR").map_err(|e| {
+            tonic_buf_build::error::TonicBufBuildError {
+                message: format!("Failed to get CARGO_MANIFEST_DIR: {}", e),
+                cause: None,
+            }
+        })?);
+        path.pop();
+        tonic_buf_build::compile_from_buf_workspace_with_config(
+            tonic_build::configure().build_server(false),
+            Some(config),
+            tonic_buf_build::TonicBufConfig {
+                buf_dir: Some(path),
+            },
+        )?;
+    }
+
+    Ok(())
+}

v4-proto-rs/deny.toml

@@ -0,0 +1,56 @@
+[graph]
+targets = [
+    { triple = "x86_64-unknown-linux-gnu" },
+    { triple = "aarch64-unknown-linux-gnu" },
+    { triple = "x86_64-unknown-linux-musl" },
+    { triple = "aarch64-apple-darwin" },
+    { triple = "x86_64-apple-darwin" },
+    { triple = "x86_64-pc-windows-msvc" },
+]
+all-features = false
+no-default-features = false
+
+[output]
+feature-depth = 1
+
+[advisories]
+db-path = "~/.cargo/advisory-db"
+db-urls = ["https://github.com/rustsec/advisory-db"]
+
+[licenses]
+allow = [
+    "MIT",
+    "Apache-2.0",
+    "BSD-3-Clause",
+    "ISC",
+    "Unicode-DFS-2016",
+    "OpenSSL",
+]
+confidence-threshold = 0.8
+exceptions = []
+
+[[licenses.clarify]]
+name = "ring"
+expression = "MIT AND ISC AND OpenSSL"
+license-files = [
+    { path = "LICENSE", hash = 0xbd0eed23 }
+]
+
+[licenses.private]
+ignore = false
+registries = []
+
+[bans]
+multiple-versions = "warn"
+wildcards = "allow"
+highlight = "all"
+
+[sources]
+unknown-registry = "warn"
+unknown-git = "warn"
+allow-registry = ["https://github.com/rust-lang/crates.io-index"]
+
+[sources.allow-org]
+github = []
+gitlab = []
+bitbucket = []

v4-proto-rs/src/_includes.rs

@@ -0,0 +1,110 @@
+// This file is @generated by prost-build.
+pub mod cosmos {
+    pub mod base {
+        pub mod query {
+            pub mod v1beta1 {
+                include!("cosmos.base.query.v1beta1.rs");
+            }
+        }
+        pub mod v1beta1 {
+            include!("cosmos.base.v1beta1.rs");
+        }
+    }
+}
+pub mod cosmos_proto {
+    include!("cosmos_proto.rs");
+}
+pub mod dydxprotocol {
+    pub mod assets {
+        include!("dydxprotocol.assets.rs");
+    }
+    pub mod blocktime {
+        include!("dydxprotocol.blocktime.rs");
+    }
+    pub mod bridge {
+        include!("dydxprotocol.bridge.rs");
+    }
+    pub mod clob {
+        include!("dydxprotocol.clob.rs");
+    }
+    pub mod daemons {
+        pub mod bridge {
+            include!("dydxprotocol.daemons.bridge.rs");
+        }
+        pub mod liquidation {
+            include!("dydxprotocol.daemons.liquidation.rs");
+        }
+        pub mod pricefeed {
+            include!("dydxprotocol.daemons.pricefeed.rs");
+        }
+    }
+    pub mod delaymsg {
+        include!("dydxprotocol.delaymsg.rs");
+    }
+    pub mod epochs {
+        include!("dydxprotocol.epochs.rs");
+    }
+    pub mod feetiers {
+        include!("dydxprotocol.feetiers.rs");
+    }
+    pub mod govplus {
+        include!("dydxprotocol.govplus.rs");
+    }
+    pub mod indexer {
+        pub mod events {
+            include!("dydxprotocol.indexer.events.rs");
+        }
+        pub mod indexer_manager {
+            include!("dydxprotocol.indexer.indexer_manager.rs");
+        }
+        pub mod off_chain_updates {
+            include!("dydxprotocol.indexer.off_chain_updates.rs");
+        }
+        pub mod protocol {
+            pub mod v1 {
+                include!("dydxprotocol.indexer.protocol.v1.rs");
+            }
+        }
+        pub mod redis {
+            include!("dydxprotocol.indexer.redis.rs");
+        }
+        pub mod shared {
+            include!("dydxprotocol.indexer.shared.rs");
+        }
+        pub mod socks {
+            include!("dydxprotocol.indexer.socks.rs");
+        }
+    }
+    pub mod perpetuals {
+        include!("dydxprotocol.perpetuals.rs");
+    }
+    pub mod prices {
+        include!("dydxprotocol.prices.rs");
+    }
+    pub mod ratelimit {
+        include!("dydxprotocol.ratelimit.rs");
+    }
+    pub mod rewards {
+        include!("dydxprotocol.rewards.rs");
+    }
+    pub mod sending {
+        include!("dydxprotocol.sending.rs");
+    }
+    pub mod stats {
+        include!("dydxprotocol.stats.rs");
+    }
+    pub mod subaccounts {
+        include!("dydxprotocol.subaccounts.rs");
+    }
+    pub mod vault {
+        include!("dydxprotocol.vault.rs");
+    }
+    pub mod vest {
+        include!("dydxprotocol.vest.rs");
+    }
+}
+pub mod google {
+    pub mod api {
+        include!("google.api.rs");
+    }
+}

v4-proto-rs/src/cosmos.base.query.v1beta1.rs

@@ -0,0 +1,77 @@
+// This file is @generated by prost-build.
+/// PageRequest is to be embedded in gRPC request messages for efficient
+/// pagination. Ex:
+///
+///   message SomeRequest {
+///           Foo some_parameter = 1;
+///           PageRequest pagination = 2;
+///   }
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct PageRequest {
+    /// key is a value returned in PageResponse.next_key to begin
+    /// querying the next page most efficiently. Only one of offset or key
+    /// should be set.
+    #[prost(bytes = "vec", tag = "1")]
+    pub key: ::prost::alloc::vec::Vec<u8>,
+    /// offset is a numeric offset that can be used when key is unavailable.
+    /// It is less efficient than using key. Only one of offset or key should
+    /// be set.
+    #[prost(uint64, tag = "2")]
+    pub offset: u64,
+    /// limit is the total number of results to be returned in the result page.
+    /// If left empty it will default to a value to be set by each app.
+    #[prost(uint64, tag = "3")]
+    pub limit: u64,
+    /// count_total is set to true  to indicate that the result set should include
+    /// a count of the total number of items available for pagination in UIs.
+    /// count_total is only respected when offset is used. It is ignored when key
+    /// is set.
+    #[prost(bool, tag = "4")]
+    pub count_total: bool,
+    /// reverse is set to true if results are to be returned in the descending order.
+    ///
+    /// Since: cosmos-sdk 0.43
+    #[prost(bool, tag = "5")]
+    pub reverse: bool,
+}
+impl ::prost::Name for PageRequest {
+    const NAME: &'static str = "PageRequest";
+    const PACKAGE: &'static str = "cosmos.base.query.v1beta1";
+    fn full_name() -> ::prost::alloc::string::String {
+        "cosmos.base.query.v1beta1.PageRequest".into()
+    }
+    fn type_url() -> ::prost::alloc::string::String {
+        "/cosmos.base.query.v1beta1.PageRequest".into()
+    }
+}
+/// PageResponse is to be embedded in gRPC response messages where the
+/// corresponding request message has used PageRequest.
+///
+///   message SomeResponse {
+///           repeated Bar results = 1;
+///           PageResponse page = 2;
+///   }
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct PageResponse {
+    /// next_key is the key to be passed to PageRequest.key to
+    /// query the next page most efficiently. It will be empty if
+    /// there are no more results.
+    #[prost(bytes = "vec", tag = "1")]
+    pub next_key: ::prost::alloc::vec::Vec<u8>,
+    /// total is total number of results available if PageRequest.count_total
+    /// was set, its value is undefined otherwise
+    #[prost(uint64, tag = "2")]
+    pub total: u64,
+}
+impl ::prost::Name for PageResponse {
+    const NAME: &'static str = "PageResponse";
+    const PACKAGE: &'static str = "cosmos.base.query.v1beta1";
+    fn full_name() -> ::prost::alloc::string::String {
+        "cosmos.base.query.v1beta1.PageResponse".into()
+    }
+    fn type_url() -> ::prost::alloc::string::String {
+        "/cosmos.base.query.v1beta1.PageResponse".into()
+    }
+}