diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index c610f725..00000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,205 +0,0 @@ -name: Release - -on: - push: - tags: - - "*-fire" - workflow_dispatch: - inputs: - core_version: - description: 'version of nearcore (optional: if not provided, will be inferred from tag)' - required: false - type: string - -env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} - -jobs: - build-indexer: - name: Build Release - runs-on: ubuntu-latest - - permissions: - contents: read - packages: write - - strategy: - matrix: - rust: [stable] - - steps: - - name: Checkout Code - uses: actions/checkout@v2 - - - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: ${{ matrix.rust }} - components: rustfmt - override: true - - - uses: actions/cache@v2 - name: Cache cargo registry - with: - path: | - ~/.cargo/registry - ~/.cargo/git - key: release-cargo-${{ hashFiles('**/Cargo.toml') }} - - - name: Cache LLVM and Clang - uses: actions/cache@v2 - id: cache-llvm - with: - path: | - ./llvm - key: llvm-10 - - - name: Install LLVM and Clang - uses: KyleMayes/install-llvm-action@v1 - with: - version: "10" - cached: ${{ steps.cache-llvm.outputs.cache-hit }} - - - name: Build target - uses: actions-rs/cargo@v1 - env: - CARGO_PROFILE_RELEASE_CODEGEN_UNITS: '1' - CARGO_PROFILE_RELEASE_LTO: 'fat' - CARGO_BUILD_RUSTFLAGS: '-D warnings' - NEAR_RELEASE_BUILD: 'release' - with: - # We cannot use `cross` tool right now. The reason is that we require some - # local libraries, `libclang.so` specifically. The `cross` tool runs a Docker - # container which does not have the library in question. We will need to wait to - # have support of https://github.com/cross-rs/cross/pull/635 to be able to cross - # compile properly. - # use-cross: true - command: build - args: --release - - - name: Upload Build - uses: actions/upload-artifact@v2 - with: - name: linux-x86_64-unknown-linux-gnu - path: ./target/release/near-firehose-indexer - - release: - name: Release - needs: [build-indexer] - runs-on: ubuntu-latest - - permissions: - contents: write - packages: write - - steps: - - name: Set Env - run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - - - name: Core Version Default Value - if: ${{ github.event.inputs.core_version == '' && !startsWith(github.ref, 'refs/tags/') }} - id: default_core_version - shell: bash - run: | - echo CORE_VERSION=latest >> $GITHUB_ENV - - - name: Core Version From Tag - if: ${{ startsWith(github.ref, 'refs/tags/') }} - id: extract_core_version_from_tag - shell: bash - run: | - echo CORE_VERSION=$(echo "${RELEASE_VERSION}" | sed -E 's/-rc\.[0-9]+//' | sed -E 's/-fire//' | sed -E 's/-hotfix[0-9]*//' | sed -E 's/-debug[0-9]*//' | sed -E 's/-patch[0-9]*//') >> $GITHUB_ENV - - - name: Core Version From Input - if: ${{ github.event.inputs.core_version != '' }} - id: extract_core_version_from_input - shell: bash - run: | - echo CORE_VERSION=$(echo "${CORE_VERSION_INPUT}") >> $GITHUB_ENV - env: - CORE_VERSION_INPUT: ${{ github.event.inputs.core_version }} - - - name: Checkout Code - uses: actions/checkout@v2 - - - name: Generate Change Log - id: changelog - if: ${{ startsWith(github.ref, 'refs/tags/') }} - run: | - chmod 755 ./.github/generate_change_log.sh - CHANGELOG=$(./.github/generate_change_log.sh ${{ env.RELEASE_VERSION }}) - - echo "CHANGELOG<> $GITHUB_ENV - echo "$CHANGELOG" >> $GITHUB_ENV - echo "EOF" >> $GITHUB_ENV - - - name: Download All Artifacts - id: download-artifacts - uses: actions/download-artifact@v2 - with: - path: ./binaries - - - name: Sanitize Downloaded Files - run: | - # We downloaded all the artifacts previously uploaded and we put them in - # the 'binaries' folder. In this folder, the layout is: - # - # binaries - # ├── linux-arm64-unknown-linux-gnu - # │ └── near-firehose-indexer - # └── linux-x86_64-unknown-linux-gnu - # └── near-firehose-indexer - # - # The sub-folder name comes from the 'name' field of the 'actions/upload-artifact@v2' - # step. The 'near-firehose-indexer' file name is the filename of the uploaded 'path' field, - # we used './target/release/near-firehose-indexer' in the upload step so the file name here - # is 'near-firehose-indexer'. - - download_path="${{steps.download-artifacts.outputs.download-path}}" - chmod +x "${download_path}/linux-x86_64-unknown-linux-gnu/near-firehose-indexer" - mv "$download_path/linux-x86_64-unknown-linux-gnu/near-firehose-indexer" "$download_path/near-firehose-indexer-x86_64-unknown-linux-gnu" - - - name: Log in to the Container registry - uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Generate docker tags/labels from github build context - id: meta - uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - tags: | - type=ref,event=tag - type=sha,prefix=,enable=true - flavor: | - latest=${{ startsWith(github.ref, 'refs/tags/') }} - - - name: Build and push Docker image - uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc - with: - context: ${{steps.download-artifacts.outputs.download-path}} - file: ./Dockerfile - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - build-args: | - CORE_VERSION=${{ env.CORE_VERSION }} - - - name: Create Release - uses: softprops/action-gh-release@v1 - if: ${{ startsWith(github.ref, 'refs/tags/') }} - with: - name: ${{ env.RELEASE_VERSION }} - tag_name: ${{ env.RELEASE_VERSION }} - draft: false - prerelease: false - body: ${{ env.CHANGELOG }} - token: ${{ secrets.GITHUB_TOKEN }} - fail_on_unmatched_files: true - generate_release_notes: true - files: | - ${{steps.download-artifacts.outputs.download-path}}/* diff --git a/Cargo.lock b/Cargo.lock index 6cb71cf5..4ca1a673 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1710,6 +1710,9 @@ name = "hex" version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +dependencies = [ + "serde", +] [[package]] name = "http" @@ -1921,17 +1924,6 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4345964bb142484797b161f473a503a434de77149dd8c7427788c6e13379388" -[[package]] -name = "lazy-static-include" -version = "3.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e559d5071cbb9743796593a892d96a7cc411d90aad711f57cc7dcbbf3893c10c" -dependencies = [ - "lazy_static", - "manifest-dir-macros", - "syn", -] - [[package]] name = "lazy_static" version = "1.4.0" @@ -2104,27 +2096,6 @@ dependencies = [ "libc", ] -[[package]] -name = "manifest-dir-macros" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f08150cf2bab1fc47c2196f4f41173a27fcd0f684165e5458c0046b53a472e2f" -dependencies = [ - "once_cell", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "matchers" -version = "0.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f099785f7595cc4b4553a174ce30dd7589ef93391ff414dbb67f62392b9e0ce1" -dependencies = [ - "regex-automata", -] - [[package]] name = "matchers" version = "0.1.0" @@ -2459,55 +2430,21 @@ dependencies = [ [[package]] name = "near-firehose-indexer" -version = "1.27.0" +version = "0.0.0" dependencies = [ "actix", - "actix-rt", - "actix-web", - "anyhow", - "awc", - "base64", - "borsh", - "byteorder", - "chrono", "clap", - "dirs", - "easy-ext", - "enum-map", - "futures", "hex", - "hyper", - "hyper-tls", - "indicatif", - "itertools", - "lazy-static-include", "near-crypto", "near-indexer", - "near-rust-allocator-proxy", "near-vm-errors", - "num-rational", - "once_cell", + "openssl", "openssl-probe", - "parking_lot 0.12.1", - "prometheus 0.11.0", - "prost 0.10.4", - "prost-types 0.10.1", - "rand 0.7.3", - "rayon", - "rlimit", - "rustix", - "serde", - "serde_ignored", - "serde_json", - "smart-default", - "strum", - "tempfile", - "thiserror", - "tokio", + "prost 0.11.6", + "prost-types 0.11.6", "tonic-build 0.5.2", "tracing", - "tracing-subscriber 0.2.25", - "xz2", + "tracing-subscriber", ] [[package]] @@ -2571,7 +2508,7 @@ dependencies = [ "serde_json", "tokio", "tracing", - "tracing-subscriber 0.3.15", + "tracing-subscriber", ] [[package]] @@ -2695,7 +2632,7 @@ dependencies = [ "opentelemetry", "opentelemetry-otlp", "opentelemetry-semantic-conventions", - "prometheus 0.13.1", + "prometheus", "strum", "thiserror", "tokio", @@ -2703,7 +2640,7 @@ dependencies = [ "tracing-appender", "tracing-opentelemetry", "tracing-serde", - "tracing-subscriber 0.3.15", + "tracing-subscriber", ] [[package]] @@ -2839,17 +2776,6 @@ dependencies = [ "syn", ] -[[package]] -name = "near-rust-allocator-proxy" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be44da452581a4f2e7870d86886f50605853943ded9b6a7975495914645cdca4" -dependencies = [ - "backtrace", - "nix 0.23.1", - "tracing", -] - [[package]] name = "near-stable-hasher" version = "0.0.0" @@ -2935,7 +2861,7 @@ dependencies = [ "near-vm-errors", "ripemd", "serde", - "sha2 0.9.9", + "sha2 0.10.2", "sha3", "zeropool-bn", ] @@ -3042,19 +2968,6 @@ dependencies = [ "void", ] -[[package]] -name = "nix" -version = "0.23.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f866317acbd3a240710c63f065ffb1e4fd466259045ccb504130b7f668f35c6" -dependencies = [ - "bitflags", - "cc", - "cfg-if 1.0.0", - "libc", - "memoffset", -] - [[package]] name = "node-runtime" version = "0.0.0" @@ -3420,17 +3333,6 @@ dependencies = [ "parking_lot_core 0.7.2", ] -[[package]] -name = "parking_lot" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" -dependencies = [ - "instant", - "lock_api 0.4.7", - "parking_lot_core 0.8.5", -] - [[package]] name = "parking_lot" version = "0.12.1" @@ -3455,20 +3357,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "parking_lot_core" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216" -dependencies = [ - "cfg-if 1.0.0", - "instant", - "libc", - "redox_syscall 0.2.16", - "smallvec", - "winapi", -] - [[package]] name = "parking_lot_core" version = "0.9.3" @@ -3616,21 +3504,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "prometheus" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8425533e7122f0c3cc7a37e6244b16ad3a2cc32ae7ac6276e2a75da0d9c200d" -dependencies = [ - "cfg-if 1.0.0", - "fnv", - "lazy_static", - "parking_lot 0.11.2", - "protobuf 2.27.1", - "regex", - "thiserror", -] - [[package]] name = "prometheus" version = "0.13.1" @@ -3668,12 +3541,12 @@ dependencies = [ [[package]] name = "prost" -version = "0.10.4" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71adf41db68aa0daaefc69bb30bcd68ded9b9abaad5d1fbb6304c4fb390e083e" +checksum = "21dc42e00223fc37204bd4aa177e69420c604ca4a183209a8f9de30c6d934698" dependencies = [ "bytes", - "prost-derive 0.10.1", + "prost-derive 0.11.6", ] [[package]] @@ -3742,9 +3615,9 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.10.1" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b670f45da57fb8542ebdbb6105a925fe571b67f9e7ed9f47a06a84e72b4e7cc" +checksum = "8bda8c0881ea9f722eb9629376db3d0b903b462477c1aafcb0566610ac28ac5d" dependencies = [ "anyhow", "itertools", @@ -3775,12 +3648,12 @@ dependencies = [ [[package]] name = "prost-types" -version = "0.10.1" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d0a014229361011dc8e69c8a1ec6c2e8d0f2af7c91e3ea3f5b2170298461e68" +checksum = "a5e0526209433e96d83d750dd81a99118edbc55739e7e61a46764fd2ad537788" dependencies = [ "bytes", - "prost 0.10.4", + "prost 0.11.6", ] [[package]] @@ -5105,7 +4978,7 @@ checksum = "09d48f71a791638519505cefafe162606f706c25592e4bde4d97600c0195312e" dependencies = [ "crossbeam-channel", "time 0.3.13", - "tracing-subscriber 0.3.15", + "tracing-subscriber", ] [[package]] @@ -5161,7 +5034,7 @@ dependencies = [ "tracing", "tracing-core", "tracing-log", - "tracing-subscriber 0.3.15", + "tracing-subscriber", ] [[package]] @@ -5174,28 +5047,6 @@ dependencies = [ "tracing-core", ] -[[package]] -name = "tracing-subscriber" -version = "0.2.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e0d2eaa99c3c2e41547cfa109e910a68ea03823cccad4a0525dcbc9b01e8c71" -dependencies = [ - "ansi_term", - "chrono", - "lazy_static", - "matchers 0.0.1", - "regex", - "serde", - "serde_json", - "sharded-slab", - "smallvec", - "thread_local", - "tracing", - "tracing-core", - "tracing-log", - "tracing-serde", -] - [[package]] name = "tracing-subscriber" version = "0.3.15" @@ -5203,7 +5054,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60db860322da191b40952ad9affe65ea23e7dd6a5c442c2c42865810c6ab8e6b" dependencies = [ "ansi_term", - "matchers 0.1.0", + "matchers", "once_cell", "regex", "sharded-slab", @@ -5506,7 +5357,7 @@ dependencies = [ "indexmap", "lazy_static", "libc", - "nix 0.15.0", + "nix", "page_size", "parking_lot 0.10.2", "rustc_version 0.2.3", @@ -5547,7 +5398,7 @@ dependencies = [ "dynasmrt", "lazy_static", "libc", - "nix 0.15.0", + "nix", "serde", "serde_derive", "smallvec", diff --git a/Cargo.toml b/Cargo.toml index 255a4bb1..3c1a1347 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,59 +3,22 @@ members = ["firehose-pb"] [package] name = "near-firehose-indexer" -version = "1.27.0" +version = "0.0.0" authors = ["StreamingFast Developers "] edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +actix = "0.13.0" clap = { version = "3.1.6", features = ["derive"] } -hex = "0.4.3" -base64 = "0.13.0" -prost = "0.10.4" -prost-types = "0.10.1" +hex = { version = "0.4.2", features = ["serde"] } +openssl = { version = "0.10", features = ["vendored"] } openssl-probe = "0.1.4" -tracing-subscriber = "0.2.4" - -parking_lot = "0.12.1" -anyhow = "1.0.51" -awc = "3.0.0" -actix = "0.13.0" -actix-web = "4.0.1" -actix-rt = "2" -byteorder = "1.2" -easy-ext = "0.2" -chrono = { version = "0.4.4", features = ["serde"] } -enum-map = "=2.1.0" -futures = "0.3" -hyper = { version = "0.14", features = ["full"] } -hyper-tls = "0.5.0" -once_cell = "1.13.1" -prometheus = "0.11" -rand = "0.7" -rayon = "1.5" -serde = { version = "1", features = ["derive"] } -serde_json = "1" -serde_ignored = "0.1" -dirs = "3" -borsh = "0.9" -thiserror = "1.0" -tokio = { version = "1.1", features = ["fs"] } -tracing = "0.1.13" -smart-default = "0.6" -num-rational = { version = "0.3", features = ["serde"] } -near-rust-allocator-proxy = { version = "0.4", optional = true } -lazy-static-include = "3" -tempfile = "3" -indicatif = "0.15.0" -xz2 = "0.1.6" -rlimit = "0.7" -strum = "0.24" -rustix = "0.33.7" - -# Added to get rid of a dependencies error even if we do not use it -itertools = "0.10.3" +prost = "0.11.6" +prost-types = "0.11.6" +tracing = { version = "0.1.36", features = ["std"] } +tracing-subscriber = { version = "0.3.15", features = ["env-filter", "fmt", "registry", "std"] } near-indexer = { git = "https://github.com/near/nearcore", rev="1.30.1" } near-crypto = { git = "https://github.com/near/nearcore", rev="1.30.1" } diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 14b97fa7..00000000 --- a/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -ARG CORE_VERSION=latest - -FROM docker.io/nearprotocol/nearcore:$CORE_VERSION as nearcore -FROM ubuntu:22.04 - -RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ - apt-get -y install -y \ - ca-certificates libssl-dev vim htop iotop sysstat wget \ - dstat strace lsof curl jq tzdata && \ - rm -rf /var/cache/apt /var/lib/apt/lists/* - -RUN rm /etc/localtime && ln -snf /usr/share/zoneinfo/America/Montreal /etc/localtime && dpkg-reconfigure -f noninteractive tzdata - -# s5cmd is a CLI tool to manipulate S3 store (Needed to sync NEAR Foundation backup(s)) -RUN mkdir /tmp/s5cmd && \ - cd /tmp/s5cmd && \ - wget -O s5cmd.tar.gz https://github.com/peak/s5cmd/releases/download/v2.0.0/s5cmd_2.0.0_Linux-64bit.tar.gz && \ - tar -xzvf s5cmd.tar.gz && \ - cp s5cmd /usr/bin/ && \ - cd / && \ - rm -rf /tmp/s5cmd - - -COPY near-firehose-indexer-x86_64-unknown-linux-gnu /app/near-firehose-indexer -RUN chmod +x /app/near-firehose-indexer -COPY --from=nearcore /usr/local/bin/neard /app/neard - -ENV PATH "$PATH:/app" \ No newline at end of file