Everything has been moved to be handled by firehose-core (https://g…

…ithub.com/streamingfast/firehose-core)

.github/workflows/docker.yml

@@ -27,7 +27,7 @@ jobs:
 
     strategy:
       matrix:
-        go-version: [1.21.x]
+        go-version: [1.22.x]
 
     outputs:
       tags: ${{ steps.meta.outputs.tags }}

.github/workflows/main.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.21.x]
+        go-version: [1.22.x]
         os: [ubuntu-latest, macos-latest]
     steps:
       - name: Set up Go
@@ -41,5 +41,8 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-go-
 
+      - name: Run Tests
+        run: ./bin/test.sh
+
       - name: Build
         run: go build ./...

.gitignore

@@ -6,5 +6,4 @@
 /build
 /dist
 /nearsf
-/firehose-data
-*.spkg
+/firehose-data*.spkg

CHANGELOG.md

@@ -4,6 +4,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). See [MAINTAINERS.md](./MAINTAINERS.md)
 for instructions to keep up to date.
 
+## Unreleased
+
+* Update to use `firehose-core`
+
 ## [1.1.14](https://github.com/streamingfast/firehose-near/releases/tag/v1.1.14)
 
 * Fixed `reader block stats` to print properly time of importing block.
@@ -203,6 +207,59 @@ The app `substreams-tier1` and `substreams-tier2` should be upgraded concurrentl
 
 #### Backend Changes
 
+### Highlights
+
+This release brings various renames to fully align with all Firehose <Chain> out there. The repository is now using `firehose-core` which should make easier to follow up with latest Firehose/Substreams feature(s).
+
+This brings in a few breaking changes to align the flags across all chains.
+
+### Breaking Changes
+
+* Removed support for `archive-node` app, if you were using this, please use a standard NEAR Archive node to do the same job.
+
+* Flag `common-block-index-sizes` has been renamed to `common-index-block-sizes`.
+
+* String variable `{sf-data-dir}` which interpolates at runtime to Firehose data directory is now `{data-dir}`. If any of your parameter value has `{sf-data-dir}` in its value, change it to `{data-dir}`.
+
+  > **Note** This is an important change, forgetting to change it will change expected locations of data leading to errors or wrong data.
+
+* The default value for `config-file` changed from `sf.yaml` to `firehose.yaml`. If you didn't had this flag defined and wish to keep the old default, define `config-file: sf.yaml`.
+
+* The default value for `data-dir` changed from `sf-data` to `firehose-data`. If you didn't had this flag defined before, you should either move `sf-data` to `firehose-data` or define `data-dir: sf-data`.
+
+  > **Note** This is an important change, forgetting to change it will change expected locations of data leading to errors or wrong data.
+
+* The flag `verbose` has been renamed to `log-verbosity`.
+
+* The default value for `common-blocks-cache-dir` changed from `{sf-data-dir}/blocks-cache` to `file://{data-dir}/storage/blocks-cache`. If you didn't had this flag defined and you had `common-blocks-cache-enabled: true`, you should define `common-blocks-cache-dir: {data-dir}/blocks-cache`.
+
+* The default value for `common-live-blocks-addr` changed from `:15011` to `:10014`. If you didn't had this flag defined and wish to keep the old default, define `common-live-blocks-addr: 15011` and ensure you also modify `relayer-grpc-listen-addr: :15011` (see next entry for details).
+
+* The default value for `relayer-grpc-listen-addr` changed from `:15011` to `:10014`. If you didn't had this flag defined and wish to keep the old default, define `relayer-grpc-listen-addr: 15011` and ensure you also modify `common-live-blocks-addr: :15011` (see previous entry for details).
+
+* The default value for `relayer-source` changed from `:15010` to `:10010`. If you didn't had this flag defined and wish to keep the old default, define `relayer-source: 15010` and ensure you also modify `reader-node-grpc-listen-addr: :15010` (see next entry for details).
+
+* The default value for `reader-node-grpc-listen-addr` changed from `:15010` to `:10010`. If you didn't had this flag defined and wish to keep the old default, define `reader-node-grpc-listen-addr: :15010` and ensure you also modify `relayer-source: :15010` (see previous entry for details).
+
+* The default value for `reader-node-manager-api-addr` changed from `:15009` to `:10011`. If you didn't had this flag defined and wish to keep the old default, define `reader-node-manager-api-addr: :15010`.
+
+* The `reader-node-arguments` is not populated anymore with default `--home={node-data-dir} <extra-args> run` which means you must now specify those manually. The variables `{data-dir}`, `{node-data-dir}` and `{hostname}` are interpolated respectively to Firehose absolute `data-dir` value, to Firehose absolute `reader-node-data-dir` value and to current hostname. To upgrade, if you had no `reader-node-arguments` defined, you must now define `reader-node-arguments: --home="{node-data-dir}" run`, if you had a `+` in your `reader-node-arguments: +--some-flag`, you must now define it like `reader-node-arguments: --home="{node-data-dir}" --some-flag run`.
+
+  > **Note** This is an important change, forgetting to change it will change expected locations of data leading to errors or wrong data.
+
+* The `reader-node-boot-nodes` flag has been removed entirely, if you have boot nodes to specify, specify them in `reader-node-arguments` using `--boot-nodes=...` instead.
+
+* Removed unused flags `reader-node-merge-and-store-directly`, `reader-node-merge-threshold-block-age` and `reader-node-wait-upload-complete-on-shutdown`.
+
+* The flag `receipt-index-builder-index-size` has been renamed to `index-builder-index-size`.
+
+* The flag `receipt-index-builder-start-block` has been renamed to `index-builder-start-block`.
+
+* The flag `receipt-index-builder-stop-block` has been renamed to `index-builder-stop-block`.
+
+* The default value for `firehose-grpc-listen-addr` changed from `:15042` to `:10015`. If you didn't had this flag defined and wish to keep the old default, define `firehose-grpc-listen-addr: :15010`.
+
+* The default value for `merger-grpc-listen-addr` changed from `:15012` to `:10012`. If you didn't had this flag defined and wish to keep the old default, define `merger-grpc-listen-addr: :15012`.
 * Update firehose-core to v0.1.1:
   - added missing `--substreams-tier2-request-stats` request debugging flag
   - added missing firehose rate limiting options flags, `--firehose-rate-limit-bucket-size` and `--firehose-rate-limit-bucket-fill-rate` to manage concurrent connection attempts to Firehose.

cmd/firenear/main.go

@@ -4,30 +4,25 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 	firecore "github.com/streamingfast/firehose-core"
+	fhCmd "github.com/streamingfast/firehose-core/cmd"
+	"github.com/streamingfast/firehose-core/firehose/info"
+	"github.com/streamingfast/firehose-core/node-manager/mindreader"
 	"github.com/streamingfast/firehose-near/codec"
 	pbnear "github.com/streamingfast/firehose-near/pb/sf/near/type/v1"
 	"github.com/streamingfast/firehose-near/transform"
 	"github.com/streamingfast/logging"
-	"github.com/streamingfast/node-manager/mindreader"
-	pbbstream "github.com/streamingfast/pbgo/sf/bstream/v1"
 	"go.uber.org/zap"
 	"google.golang.org/protobuf/reflect/protoreflect"
 )
 
-func init() {
-	firecore.UnsafePayloadKind = pbbstream.Protocol_NEAR
-}
-
 func main() {
-	firecore.Main(&firecore.Chain[*pbnear.Block]{
+	chain := &firecore.Chain[*pbnear.Block]{
 		ShortName:            "near",
 		LongName:             "NEAR",
 		ExecutableName:       "near-firehose-indexer",
 		FullyQualifiedModule: "github.com/streamingfast/firehose-near",
 		Version:              version,
-
-		Protocol:        "NEA",
-		ProtocolVersion: 1,
+		DefaultBlockType:     "sf.near.type.v1.Block",
 
 		BlockFactory: func() firecore.Block { return new(pbnear.Block) },
 
@@ -41,8 +36,8 @@ func main() {
 		},
 
 		ConsoleReaderFactory: func(lines chan string, blockEncoder firecore.BlockEncoder, logger *zap.Logger, tracer logging.Tracer) (mindreader.ConsolerReader, error) {
-			// FIXME: This was hardcoded also in the previouse firehose-near version, Firehose will break if this is not available
-			return codec.NewConsoleReader(lines, blockEncoder, "http://localhost:3030")
+			// FIXME: This was hardcoded also in the previous firehose-near version, Firehose will break if this is not available
+			return codec.NewConsoleReader(lines, firecore.NewBlockEncoder(), "http://localhost:3030")
 		},
 
 		RegisterExtraStartFlags: func(flags *pflag.FlagSet) {
@@ -55,23 +50,25 @@ func main() {
 		ReaderNodeBootstrapperFactory: newReaderNodeBootstrapper,
 
 		Tools: &firecore.ToolsConfig[*pbnear.Block]{
-			BlockPrinter: printBlock,
+			MergedBlockUpgrader: blockUpgrader,
 
 			RegisterExtraCmd: func(chain *firecore.Chain[*pbnear.Block], toolsCmd *cobra.Command, zlog *zap.Logger, tracer logging.Tracer) error {
 				toolsCmd.AddCommand(newToolsGenerateNodeKeyCmd(chain))
-				toolsCmd.AddCommand(newToolsBackfillCmd(zlog))
-
 				return nil
 			},
 
 			TransformFlags: &firecore.TransformFlags{
 				Register: func(flags *pflag.FlagSet) {
 					flags.String("receipt-account-filters", "", "Comma-separated accounts to use as filter/index. If it contains a colon (:), it will be interpreted as <prefix>:<suffix> (each of which can be empty, ex: 'hello:' or ':world')")
 				},
-				Parse: parseReceiptAccountFilters,
+				Parse: receiptAccountFiltersParser,
 			},
 		},
-	})
+
+		InfoResponseFiller: info.DefaultInfoResponseFiller,
+	}
+
+	fhCmd.Main(chain)
 }
 
 // Version value, injected via go build `ldflags` at build time, **must** not be removed or inlined

cmd/firenear/reader_node_bootstraper.go → cmd/firenear/reader_node_bootstrapper.go

@@ -13,20 +13,37 @@ import (
 	"github.com/spf13/viper"
 	"github.com/streamingfast/dstore"
 	firecore "github.com/streamingfast/firehose-core"
-	"github.com/streamingfast/node-manager/operator"
+	"github.com/streamingfast/firehose-core/node-manager/operator"
 	"go.uber.org/zap"
 )
 
-func newReaderNodeBootstrapper(ctx context.Context, logger *zap.Logger, cmd *cobra.Command, resolvedNodeArguments []string, resolver firecore.ReaderNodeArgumentResolver) (operator.Bootstrapper, error) {
-	nodeDataDir := resolver("{node-data-dir}")
-
+func newReaderNodeBootstrapper(
+	ctx context.Context,
+	logger *zap.Logger,
+	cmd *cobra.Command,
+	resolvedNodeArguments []string,
+	resolver firecore.ReaderNodeArgumentResolver,
+) (operator.Bootstrapper, error) {
 	hostname, _ := os.Hostname()
 
+	logger.Info("bootstrapping reader node",
+		zap.String("hostname", hostname),
+		zap.String("config_file", viper.GetString("reader-node-config-file")),
+	)
+
 	configFile := replaceNodeRole(viper.GetString("reader-node-config-file"), hostname)
 	genesisFile := replaceNodeRole(viper.GetString("reader-node-genesis-file"), hostname)
 	nodeKeyFile := replaceHostname(viper.GetString("reader-node-key-file"), hostname)
+	nodeDataDir := replaceHostname(viper.GetString("reader-node-data-dir"), hostname)
 	overwriteNodeFiles := viper.GetBool("reader-node-overwrite-node-files")
 
+	logger.Info("final node configuration",
+		zap.String("config_file", configFile),
+		zap.String("genesis_file", genesisFile),
+		zap.String("node_key_file", nodeKeyFile),
+		zap.String("node_data_dir", nodeDataDir),
+	)
+
 	return &bootstrapper{
 		configFile:  configFile,
 		genesisFile: genesisFile,

cmd/firenear/tools.go

@@ -5,17 +5,35 @@ import (
 	"io"
 	"strings"
 
+	"google.golang.org/protobuf/types/known/timestamppb"
+
 	"github.com/spf13/cobra"
-	"github.com/streamingfast/bstream"
-	"github.com/streamingfast/cli/sflags"
+
+	pbbstream "github.com/streamingfast/bstream/pb/sf/bstream/v1"
+	firecore "github.com/streamingfast/firehose-core"
 	pbtransform "github.com/streamingfast/firehose-near/pb/sf/near/transform/v1"
 	pbnear "github.com/streamingfast/firehose-near/pb/sf/near/type/v1"
 	"go.uber.org/zap"
 	"google.golang.org/protobuf/types/known/anypb"
 )
 
-func printBlock(blk *bstream.Block, alsoPrintTransactions bool, out io.Writer) error {
-	block := blk.ToProtocol().(*pbnear.Block)
+func encodeBlock(blk firecore.Block) (*pbbstream.Block, error) {
+	block := blk.(*pbnear.Block)
+
+	enc := &pbbstream.Block{
+		Number:    block.Num(),
+		Id:        block.ID(),
+		LibNum:    block.LIBNum(),
+		ParentNum: block.GetFirehoseBlockParentNumber(),
+		ParentId:  block.PreviousID(),
+		Timestamp: timestamppb.New(block.GetFirehoseBlockTime()),
+	}
+
+	return enc, nil
+}
+
+func printBlock(blk firecore.Block, alsoPrintTransactions bool, out io.Writer) error {
+	block := blk.(*pbnear.Block)
 
 	transactionCount := 0
 	for _, shard := range block.Shards {
@@ -58,8 +76,27 @@ func printBlock(blk *bstream.Block, alsoPrintTransactions bool, out io.Writer) e
 	return nil
 }
 
-func parseReceiptAccountFilters(cmd *cobra.Command, logger *zap.Logger) ([]*anypb.Any, error) {
-	in := sflags.MustGetString(cmd, "receipt-account-filters")
+func receiptAccountFiltersParser(cmd *cobra.Command, logger *zap.Logger) ([]*anypb.Any, error) {
+	filterStrs, err := cmd.Flags().GetStringSlice("receipt-account-filters")
+	if err != nil {
+		return nil, fmt.Errorf("unable to get receipt-account-filters flag: %w", err)
+	}
+
+	var filters []*anypb.Any
+	for _, filterStr := range filterStrs {
+		filter, err := parseReceiptAccountFilters(filterStr)
+		if err != nil {
+			return nil, fmt.Errorf("unable to parse receipt account filters: %w", err)
+		}
+		if filter != nil {
+			filters = append(filters, filter)
+		}
+	}
+
+	return filters, nil
+}
+
+func parseReceiptAccountFilters(in string) (*anypb.Any, error) {
 	if in == "" {
 		return nil, nil
 	}
@@ -83,9 +120,5 @@ func parseReceiptAccountFilters(cmd *cobra.Command, logger *zap.Logger) ([]*anyp
 		PrefixAndSuffixPairs: pairs,
 	}
 
-	any, err := anypb.New(filters)
-	if err != nil {
-		return nil, err
-	}
-	return []*anypb.Any{any}, nil
+	return anypb.New(filters)
 }