rework well-known chains and info endpoint discovery mechanism

CHANGELOG.md

@@ -11,11 +11,12 @@ If you were at `firehose-core` version `1.0.0` and are bumping to `1.1.0`, you s
 ## Unreleased
 
 * Add `sf.firehose.v2.EndpointInfo/Info` service on Firehose and Substreams endpoints. This involves the following new flags:
-  - `advertise-chain-name` Canonical name of the chain, from the list here: https://thegraph.com/docs/en/developing/supported-networks/ (required)
+  - `advertise-chain-name` Canonical name of the chain according to https://thegraph.com/docs/en/developing/supported-networks/ (required, unless it is in the "well-known" list)
   - `advertise-chain-aliases` Alternate names for that chain (optional)
-  - `advertise-block-features` Only required for ethereum blocks, automatically discovered if run from `firehose-ethereum` program
-  - `advertise-block-id-encoding` Required, one of [BLOCK_ID_ENCODING_BASE58, BLOCK_ID_ENCODING_BASE64, BLOCK_ID_ENCODING_HEX, BLOCK_ID_ENCODING_0X_HEX]
+  - `advertise-block-features` List of features describing the blocks (optional)
+  - `advertise-block-id-encoding` Encoding format of the block ID [BLOCK_ID_ENCODING_BASE58, BLOCK_ID_ENCODING_BASE64, BLOCK_ID_ENCODING_BASE64URL, BLOCK_ID_ENCODING_HEX, BLOCK_ID_ENCODING_0X_HEX] (required, unless the block type is in the "well-known" list)
 
+* Add a well-known list of chains (hard-coded in `wellknown/chains.go` to help automatically determine the 'advertise' flag values)
 * The new info endpoint adds a mandatory fetching of the first streamable block on startup, with a failure if no block can be fetched after 3 minutes and you are running `firehose` or `substreams-tier1` service.
 
 * Substreams: revert module hash calculation from `v1.5.5`, when using a non-zero firstStreamableBlock. Hashes will now be the same even if the chain's first streamable block affects the initialBlock of a module.

cmd/apps/start.go

@@ -96,6 +96,7 @@ func start[B firecore.Block](cmd *cobra.Command, dataDir string, args []string,
 		sflags.MustGetStringSlice(cmd, "advertise-block-features"),
 		bstream.GetProtocolFirstStreamableBlock,
 		chain.InfoResponseFiller,
+		rootLog,
 	)
 
 	launch := launcher.NewLauncher(rootLog, dataDirAbs, infoServer)

firehose/info/endpoint_info.go

@@ -19,6 +19,7 @@ type InfoServer struct {
 	response       *pbfirehose.InfoResponse
 	ready          chan struct{}
 	once           sync.Once
+	logger         *zap.Logger
 }
 
 func (s *InfoServer) Info(ctx context.Context, request *pbfirehose.InfoRequest) (*pbfirehose.InfoResponse, error) {
@@ -37,6 +38,7 @@ func NewInfoServer(
 	blockFeatures []string,
 	firstStreamableBlock uint64,
 	responseFiller func(block *pbbstream.Block, resp *pbfirehose.InfoResponse) error,
+	logger *zap.Logger,
 ) *InfoServer {
 
 	resp := &pbfirehose.InfoResponse{
@@ -51,6 +53,7 @@ func NewInfoServer(
 		responseFiller: responseFiller,
 		response:       resp,
 		ready:          make(chan struct{}),
+		logger:         logger,
 	}
 }
 
@@ -148,6 +151,21 @@ func (s *InfoServer) init(ctx context.Context, fhub *hub.ForkableHub, mergedBloc
 		}
 	}()
 
+	go func() {
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-time.After(5 * time.Second):
+				logger.Warn("waiting to read the first_streamable_block before starting firehose/substreams endpoints",
+					zap.Uint64("first_streamable_block", s.response.FirstStreamableBlockNum),
+					zap.Stringer("merged_blocks_store", mergedBlocksStore.BaseURL()), // , zap.String("one_block_store", oneBlockStore.String())
+					zap.Stringer("one_block_store", oneBlockStore.BaseURL()),         // , zap.String("one_block_store", oneBlockStore.String())
+				)
+			}
+		}
+	}()
+
 	select {
 	case blk := <-ch:
 		if err := s.responseFiller(blk, s.response); err != nil {

firehose/info/info_filler.go

@@ -4,57 +4,39 @@ import (
 	"fmt"
 
 	pbbstream "github.com/streamingfast/bstream/pb/sf/bstream/v1"
+	wellknown "github.com/streamingfast/firehose-core/well-known"
 	pbfirehose "github.com/streamingfast/pbgo/sf/firehose/v2"
 )
 
-var DefaultInfoResponseFiller = func(block *pbbstream.Block, resp *pbfirehose.InfoResponse) error {
-	resp.FirstStreamableBlockId = block.Id
-
-	switch block.Payload.TypeUrl {
-	case "type.googleapis.com/sf.antelope.type.v1.Block":
-		return fillInfoResponseForAntelope(block, resp)
-
-	case "type.googleapis.com/sf.ethereum.type.v2.Block":
-		return fillInfoResponseForEthereum(block, resp)
-
-	case "type.googleapis.com/sf.cosmos.type.v1.Block":
-		return fillInfoResponseForCosmos(block, resp)
-
-	case "type.googleapis.com/sf.solana.type.v1.Block":
-		return fillInfoResponseForSolana(block, resp)
+var DefaultInfoResponseFiller = func(firstStreamableBlock *pbbstream.Block, resp *pbfirehose.InfoResponse) error {
+	resp.FirstStreamableBlockId = firstStreamableBlock.Id
+
+	if resp.ChainName != "" {
+		if chain := wellknown.WellKnownProtocols.ChainByName(resp.ChainName); chain != nil {
+			if firstStreamableBlock.Number == chain.GenesisBlockNumber && chain.GenesisBlockID != firstStreamableBlock.Id { // we don't check if the firstStreamableBlock is something other than our well-known genesis block
+				return fmt.Errorf("chain name defined in flag: %q inconsistent with the genesis block ID %q (expected: %q)", resp.ChainName, ox(firstStreamableBlock.Id), ox(chain.GenesisBlockID))
+			}
+			resp.ChainName = chain.Name
+			resp.ChainNameAliases = chain.Aliases
+		} else if chain := wellknown.WellKnownProtocols.ChainByGenesisBlock(firstStreamableBlock.Number, firstStreamableBlock.Id); chain != nil {
+			return fmt.Errorf("chain name defined in flag: %q inconsistent with the one discovered from genesis block %q", resp.ChainName, chain.Name)
+		}
+	} else {
+		if chain := wellknown.WellKnownProtocols.ChainByGenesisBlock(firstStreamableBlock.Number, firstStreamableBlock.Id); chain != nil {
+			resp.ChainName = chain.Name
+			resp.ChainNameAliases = chain.Aliases
+		}
 	}
 
-	return nil
-}
-
-// this is a simple helper, a full implementation would live in github.com/streamingfast/firehose-ethereum
-func fillInfoResponseForEthereum(block *pbbstream.Block, resp *pbfirehose.InfoResponse) error {
-	resp.BlockIdEncoding = pbfirehose.InfoResponse_BLOCK_ID_ENCODING_HEX
-	var seenBlockType bool
-	for _, feature := range resp.BlockFeatures {
-		if feature == "extended" || feature == "base" || feature == "hybrid" {
-			seenBlockType = true
+	for _, protocol := range wellknown.WellKnownProtocols {
+		if protocol.BlockType == firstStreamableBlock.Payload.TypeUrl {
+			resp.BlockIdEncoding = protocol.BytesEncoding
 			break
 		}
 	}
-	if !seenBlockType {
-		return fmt.Errorf("invalid block features, missing 'base', 'extended' or 'hybrid'")
-	}
-	return nil
-}
-
-// this is a simple helper, a full implementation would live in github.com/pinax-network/firehose-antelope
-func fillInfoResponseForAntelope(block *pbbstream.Block, resp *pbfirehose.InfoResponse) error {
-	resp.BlockIdEncoding = pbfirehose.InfoResponse_BLOCK_ID_ENCODING_HEX
 	return nil
 }
 
-func fillInfoResponseForCosmos(block *pbbstream.Block, resp *pbfirehose.InfoResponse) error {
-	resp.BlockIdEncoding = pbfirehose.InfoResponse_BLOCK_ID_ENCODING_HEX
-	return nil
-}
-
-func fillInfoResponseForSolana(block *pbbstream.Block, resp *pbfirehose.InfoResponse) error {
-	resp.BlockIdEncoding = pbfirehose.InfoResponse_BLOCK_ID_ENCODING_BASE58
-	return nil
+func ox(s string) string {
+	return "0x" + s
 }

go.mod

@@ -30,7 +30,7 @@ require (
 	github.com/streamingfast/jsonpb v0.0.0-20210811021341-3670f0aa02d0
 	github.com/streamingfast/logging v0.0.0-20230608130331-f22c91403091
 	github.com/streamingfast/payment-gateway v0.0.0-20240426151444-581e930c76e2
-	github.com/streamingfast/pbgo v0.0.6-0.20240821201153-468db4096ff0
+	github.com/streamingfast/pbgo v0.0.6-0.20240823134334-812f6a16c5cb
 	github.com/streamingfast/snapshotter v0.0.0-20230316190750-5bcadfde44d0
 	github.com/streamingfast/substreams v1.9.4-0.20240812210000-635f7bcba6cf
 	github.com/stretchr/testify v1.8.4

go.sum

@@ -572,8 +572,8 @@ github.com/streamingfast/overseer v0.2.1-0.20210326144022-ee491780e3ef h1:9IVFHR
 github.com/streamingfast/overseer v0.2.1-0.20210326144022-ee491780e3ef/go.mod h1:cq8CvbZ3ioFmGrHokSAJalS0lC+pVXLKhITScItUGXY=
 github.com/streamingfast/payment-gateway v0.0.0-20240426151444-581e930c76e2 h1:bliib3pAObbM+6cKYQFa8axbCY/x6RczQZrOxdM7OZA=
 github.com/streamingfast/payment-gateway v0.0.0-20240426151444-581e930c76e2/go.mod h1:DsnLrpKZ3DIDL6FmYVuxbC44fXvQdY7aCdSLMpbqZ8Q=
-github.com/streamingfast/pbgo v0.0.6-0.20240821201153-468db4096ff0 h1:kfzU2GvvWt+RQtrHvfqv7zdRA4xv/3AusccIGG3+roM=
-github.com/streamingfast/pbgo v0.0.6-0.20240821201153-468db4096ff0/go.mod h1:eDQjKBYg9BWE2BTaV3UZeLZ5xw05+ywA9RCFTmM1w5Y=
+github.com/streamingfast/pbgo v0.0.6-0.20240823134334-812f6a16c5cb h1:Xqt4ned9ELmQMKcg7cFbm56MKG2gBjnE1M+2HObOs6w=
+github.com/streamingfast/pbgo v0.0.6-0.20240823134334-812f6a16c5cb/go.mod h1:eDQjKBYg9BWE2BTaV3UZeLZ5xw05+ywA9RCFTmM1w5Y=
 github.com/streamingfast/protoreflect v0.0.0-20231205191344-4b629d20ce8d h1:33VIARqUqBUKXJcuQoOS1rVSms54tgxhhNCmrLptpLg=
 github.com/streamingfast/protoreflect v0.0.0-20231205191344-4b629d20ce8d/go.mod h1:aBJivEdekmFWYSQ29EE/fN9IanJWJXbtjy3ky0XD/jE=
 github.com/streamingfast/sf-tracing v0.0.0-20240430173521-888827872b90 h1:94HllkX4ttYVilo8ZJv05b5z8JiMmqBvv4+Jdgk/+2A=

proto/generator/generator.go

@@ -20,25 +20,13 @@ import (
 	connect "connectrpc.com/connect"
 	"github.com/iancoleman/strcase"
 	"github.com/streamingfast/cli"
+	wellknown "github.com/streamingfast/firehose-core/well-known"
 	"google.golang.org/protobuf/proto"
 )
 
 //go:embed *.gotmpl
 var templates embed.FS
 
-var wellKnownProtoRepos = []string{
-	"buf.build/streamingfast/firehose-ethereum",
-	"buf.build/streamingfast/firehose-near",
-	"buf.build/streamingfast/firehose-solana",
-	"buf.build/streamingfast/firehose-bitcoin",
-	"buf.build/pinax/firehose-antelope",
-	"buf.build/pinax/firehose-arweave",
-	"buf.build/pinax/firehose-beacon",
-	"buf.build/streamingfast/firehose-starknet",
-	"buf.build/streamingfast/firehose-cosmos",
-	"buf.build/streamingfast/firehose-gear",
-}
-
 func main() {
 	cli.Ensure(len(os.Args) == 3, "go run ./generator <output_file> <package_name>")
 
@@ -58,7 +46,8 @@ func main() {
 
 	var protofiles []ProtoFile
 
-	for _, wellKnownProtoRepo := range wellKnownProtoRepos {
+	for _, protocol := range wellknown.WellKnownProtocols {
+		wellKnownProtoRepo := protocol.BufBuildURL
 		request := connect.NewRequest(&reflectv1beta1.GetFileDescriptorSetRequest{
 			Module: wellKnownProtoRepo,
 		})