diff --git a/CHANGELOG.md b/CHANGELOG.md index 196cb37..fc13de8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,11 +11,12 @@ If you were at `firehose-core` version `1.0.0` and are bumping to `1.1.0`, you s ## Unreleased * Add `sf.firehose.v2.EndpointInfo/Info` service on Firehose and Substreams endpoints. This involves the following new flags: - - `advertise-chain-name` Canonical name of the chain, from the list here: https://thegraph.com/docs/en/developing/supported-networks/ (required) + - `advertise-chain-name` Canonical name of the chain according to https://thegraph.com/docs/en/developing/supported-networks/ (required, unless it is in the "well-known" list) - `advertise-chain-aliases` Alternate names for that chain (optional) - - `advertise-block-features` Only required for ethereum blocks, automatically discovered if run from `firehose-ethereum` program - - `advertise-block-id-encoding` Required, one of [BLOCK_ID_ENCODING_BASE58, BLOCK_ID_ENCODING_BASE64, BLOCK_ID_ENCODING_HEX, BLOCK_ID_ENCODING_0X_HEX] + - `advertise-block-features` List of features describing the blocks (optional) + - `advertise-block-id-encoding` Encoding format of the block ID [BLOCK_ID_ENCODING_BASE58, BLOCK_ID_ENCODING_BASE64, BLOCK_ID_ENCODING_BASE64URL, BLOCK_ID_ENCODING_HEX, BLOCK_ID_ENCODING_0X_HEX] (required, unless the block type is in the "well-known" list) +* Add a well-known list of chains (hard-coded in `wellknown/chains.go` to help automatically determine the 'advertise' flag values) * The new info endpoint adds a mandatory fetching of the first streamable block on startup, with a failure if no block can be fetched after 3 minutes and you are running `firehose` or `substreams-tier1` service. * Substreams: revert module hash calculation from `v1.5.5`, when using a non-zero firstStreamableBlock. Hashes will now be the same even if the chain's first streamable block affects the initialBlock of a module. diff --git a/cmd/apps/start.go b/cmd/apps/start.go index 2a3d551..9af3fd9 100644 --- a/cmd/apps/start.go +++ b/cmd/apps/start.go @@ -96,6 +96,7 @@ func start[B firecore.Block](cmd *cobra.Command, dataDir string, args []string, sflags.MustGetStringSlice(cmd, "advertise-block-features"), bstream.GetProtocolFirstStreamableBlock, chain.InfoResponseFiller, + rootLog, ) launch := launcher.NewLauncher(rootLog, dataDirAbs, infoServer) diff --git a/firehose/info/endpoint_info.go b/firehose/info/endpoint_info.go index d1c76eb..7b09feb 100644 --- a/firehose/info/endpoint_info.go +++ b/firehose/info/endpoint_info.go @@ -19,6 +19,7 @@ type InfoServer struct { response *pbfirehose.InfoResponse ready chan struct{} once sync.Once + logger *zap.Logger } func (s *InfoServer) Info(ctx context.Context, request *pbfirehose.InfoRequest) (*pbfirehose.InfoResponse, error) { @@ -37,6 +38,7 @@ func NewInfoServer( blockFeatures []string, firstStreamableBlock uint64, responseFiller func(block *pbbstream.Block, resp *pbfirehose.InfoResponse) error, + logger *zap.Logger, ) *InfoServer { resp := &pbfirehose.InfoResponse{ @@ -51,6 +53,7 @@ func NewInfoServer( responseFiller: responseFiller, response: resp, ready: make(chan struct{}), + logger: logger, } } @@ -148,6 +151,21 @@ func (s *InfoServer) init(ctx context.Context, fhub *hub.ForkableHub, mergedBloc } }() + go func() { + for { + select { + case <-ctx.Done(): + return + case <-time.After(5 * time.Second): + logger.Warn("waiting to read the first_streamable_block before starting firehose/substreams endpoints", + zap.Uint64("first_streamable_block", s.response.FirstStreamableBlockNum), + zap.Stringer("merged_blocks_store", mergedBlocksStore.BaseURL()), // , zap.String("one_block_store", oneBlockStore.String()) + zap.Stringer("one_block_store", oneBlockStore.BaseURL()), // , zap.String("one_block_store", oneBlockStore.String()) + ) + } + } + }() + select { case blk := <-ch: if err := s.responseFiller(blk, s.response); err != nil { diff --git a/firehose/info/info_filler.go b/firehose/info/info_filler.go index 674853f..73932f0 100644 --- a/firehose/info/info_filler.go +++ b/firehose/info/info_filler.go @@ -4,57 +4,39 @@ import ( "fmt" pbbstream "github.com/streamingfast/bstream/pb/sf/bstream/v1" + wellknown "github.com/streamingfast/firehose-core/well-known" pbfirehose "github.com/streamingfast/pbgo/sf/firehose/v2" ) -var DefaultInfoResponseFiller = func(block *pbbstream.Block, resp *pbfirehose.InfoResponse) error { - resp.FirstStreamableBlockId = block.Id - - switch block.Payload.TypeUrl { - case "type.googleapis.com/sf.antelope.type.v1.Block": - return fillInfoResponseForAntelope(block, resp) - - case "type.googleapis.com/sf.ethereum.type.v2.Block": - return fillInfoResponseForEthereum(block, resp) - - case "type.googleapis.com/sf.cosmos.type.v1.Block": - return fillInfoResponseForCosmos(block, resp) - - case "type.googleapis.com/sf.solana.type.v1.Block": - return fillInfoResponseForSolana(block, resp) +var DefaultInfoResponseFiller = func(firstStreamableBlock *pbbstream.Block, resp *pbfirehose.InfoResponse) error { + resp.FirstStreamableBlockId = firstStreamableBlock.Id + + if resp.ChainName != "" { + if chain := wellknown.WellKnownProtocols.ChainByName(resp.ChainName); chain != nil { + if firstStreamableBlock.Number == chain.GenesisBlockNumber && chain.GenesisBlockID != firstStreamableBlock.Id { // we don't check if the firstStreamableBlock is something other than our well-known genesis block + return fmt.Errorf("chain name defined in flag: %q inconsistent with the genesis block ID %q (expected: %q)", resp.ChainName, ox(firstStreamableBlock.Id), ox(chain.GenesisBlockID)) + } + resp.ChainName = chain.Name + resp.ChainNameAliases = chain.Aliases + } else if chain := wellknown.WellKnownProtocols.ChainByGenesisBlock(firstStreamableBlock.Number, firstStreamableBlock.Id); chain != nil { + return fmt.Errorf("chain name defined in flag: %q inconsistent with the one discovered from genesis block %q", resp.ChainName, chain.Name) + } + } else { + if chain := wellknown.WellKnownProtocols.ChainByGenesisBlock(firstStreamableBlock.Number, firstStreamableBlock.Id); chain != nil { + resp.ChainName = chain.Name + resp.ChainNameAliases = chain.Aliases + } } - return nil -} - -// this is a simple helper, a full implementation would live in github.com/streamingfast/firehose-ethereum -func fillInfoResponseForEthereum(block *pbbstream.Block, resp *pbfirehose.InfoResponse) error { - resp.BlockIdEncoding = pbfirehose.InfoResponse_BLOCK_ID_ENCODING_HEX - var seenBlockType bool - for _, feature := range resp.BlockFeatures { - if feature == "extended" || feature == "base" || feature == "hybrid" { - seenBlockType = true + for _, protocol := range wellknown.WellKnownProtocols { + if protocol.BlockType == firstStreamableBlock.Payload.TypeUrl { + resp.BlockIdEncoding = protocol.BytesEncoding break } } - if !seenBlockType { - return fmt.Errorf("invalid block features, missing 'base', 'extended' or 'hybrid'") - } - return nil -} - -// this is a simple helper, a full implementation would live in github.com/pinax-network/firehose-antelope -func fillInfoResponseForAntelope(block *pbbstream.Block, resp *pbfirehose.InfoResponse) error { - resp.BlockIdEncoding = pbfirehose.InfoResponse_BLOCK_ID_ENCODING_HEX return nil } -func fillInfoResponseForCosmos(block *pbbstream.Block, resp *pbfirehose.InfoResponse) error { - resp.BlockIdEncoding = pbfirehose.InfoResponse_BLOCK_ID_ENCODING_HEX - return nil -} - -func fillInfoResponseForSolana(block *pbbstream.Block, resp *pbfirehose.InfoResponse) error { - resp.BlockIdEncoding = pbfirehose.InfoResponse_BLOCK_ID_ENCODING_BASE58 - return nil +func ox(s string) string { + return "0x" + s } diff --git a/go.mod b/go.mod index 6e64aa2..59f6925 100644 --- a/go.mod +++ b/go.mod @@ -30,7 +30,7 @@ require ( github.com/streamingfast/jsonpb v0.0.0-20210811021341-3670f0aa02d0 github.com/streamingfast/logging v0.0.0-20230608130331-f22c91403091 github.com/streamingfast/payment-gateway v0.0.0-20240426151444-581e930c76e2 - github.com/streamingfast/pbgo v0.0.6-0.20240821201153-468db4096ff0 + github.com/streamingfast/pbgo v0.0.6-0.20240823134334-812f6a16c5cb github.com/streamingfast/snapshotter v0.0.0-20230316190750-5bcadfde44d0 github.com/streamingfast/substreams v1.9.4-0.20240812210000-635f7bcba6cf github.com/stretchr/testify v1.8.4 diff --git a/go.sum b/go.sum index df9d1e2..36dae1b 100644 --- a/go.sum +++ b/go.sum @@ -572,8 +572,8 @@ github.com/streamingfast/overseer v0.2.1-0.20210326144022-ee491780e3ef h1:9IVFHR github.com/streamingfast/overseer v0.2.1-0.20210326144022-ee491780e3ef/go.mod h1:cq8CvbZ3ioFmGrHokSAJalS0lC+pVXLKhITScItUGXY= github.com/streamingfast/payment-gateway v0.0.0-20240426151444-581e930c76e2 h1:bliib3pAObbM+6cKYQFa8axbCY/x6RczQZrOxdM7OZA= github.com/streamingfast/payment-gateway v0.0.0-20240426151444-581e930c76e2/go.mod h1:DsnLrpKZ3DIDL6FmYVuxbC44fXvQdY7aCdSLMpbqZ8Q= -github.com/streamingfast/pbgo v0.0.6-0.20240821201153-468db4096ff0 h1:kfzU2GvvWt+RQtrHvfqv7zdRA4xv/3AusccIGG3+roM= -github.com/streamingfast/pbgo v0.0.6-0.20240821201153-468db4096ff0/go.mod h1:eDQjKBYg9BWE2BTaV3UZeLZ5xw05+ywA9RCFTmM1w5Y= +github.com/streamingfast/pbgo v0.0.6-0.20240823134334-812f6a16c5cb h1:Xqt4ned9ELmQMKcg7cFbm56MKG2gBjnE1M+2HObOs6w= +github.com/streamingfast/pbgo v0.0.6-0.20240823134334-812f6a16c5cb/go.mod h1:eDQjKBYg9BWE2BTaV3UZeLZ5xw05+ywA9RCFTmM1w5Y= github.com/streamingfast/protoreflect v0.0.0-20231205191344-4b629d20ce8d h1:33VIARqUqBUKXJcuQoOS1rVSms54tgxhhNCmrLptpLg= github.com/streamingfast/protoreflect v0.0.0-20231205191344-4b629d20ce8d/go.mod h1:aBJivEdekmFWYSQ29EE/fN9IanJWJXbtjy3ky0XD/jE= github.com/streamingfast/sf-tracing v0.0.0-20240430173521-888827872b90 h1:94HllkX4ttYVilo8ZJv05b5z8JiMmqBvv4+Jdgk/+2A= diff --git a/proto/generator/generator.go b/proto/generator/generator.go index c96d51f..375d1d8 100644 --- a/proto/generator/generator.go +++ b/proto/generator/generator.go @@ -20,25 +20,13 @@ import ( connect "connectrpc.com/connect" "github.com/iancoleman/strcase" "github.com/streamingfast/cli" + wellknown "github.com/streamingfast/firehose-core/well-known" "google.golang.org/protobuf/proto" ) //go:embed *.gotmpl var templates embed.FS -var wellKnownProtoRepos = []string{ - "buf.build/streamingfast/firehose-ethereum", - "buf.build/streamingfast/firehose-near", - "buf.build/streamingfast/firehose-solana", - "buf.build/streamingfast/firehose-bitcoin", - "buf.build/pinax/firehose-antelope", - "buf.build/pinax/firehose-arweave", - "buf.build/pinax/firehose-beacon", - "buf.build/streamingfast/firehose-starknet", - "buf.build/streamingfast/firehose-cosmos", - "buf.build/streamingfast/firehose-gear", -} - func main() { cli.Ensure(len(os.Args) == 3, "go run ./generator ") @@ -58,7 +46,8 @@ func main() { var protofiles []ProtoFile - for _, wellKnownProtoRepo := range wellKnownProtoRepos { + for _, protocol := range wellknown.WellKnownProtocols { + wellKnownProtoRepo := protocol.BufBuildURL request := connect.NewRequest(&reflectv1beta1.GetFileDescriptorSetRequest{ Module: wellKnownProtoRepo, }) diff --git a/well-known/chains.go b/well-known/chains.go new file mode 100644 index 0000000..b266032 --- /dev/null +++ b/well-known/chains.go @@ -0,0 +1,297 @@ +package wellknown + +import ( + pbfirehose "github.com/streamingfast/pbgo/sf/firehose/v2" +) + +type WellKnownProtocol struct { + Name string + BlockType string + BufBuildURL string + BytesEncoding pbfirehose.InfoResponse_BlockIdEncoding + KnownChains []*Chain +} + +type Chain struct { + Name string + Aliases []string + // Genesis block here is actually the "lowest possible" first streamable block through firehose blocks. + // In most cases, it matches the "genesis block" of the chain. + GenesisBlockID string + GenesisBlockNumber uint64 +} + +type WellKnownProtocolList []WellKnownProtocol + +var WellKnownProtocols = WellKnownProtocolList([]WellKnownProtocol{ + { + Name: "ethereum", + BlockType: "type.googleapis.com/sf.ethereum.type.v2.Block", + BufBuildURL: "buf.build/streamingfast/firehose-ethereum", + BytesEncoding: pbfirehose.InfoResponse_BLOCK_ID_ENCODING_HEX, + KnownChains: []*Chain{ + { + Name: "mainnet", + Aliases: []string{"ethereum"}, + GenesisBlockID: "d4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3", + GenesisBlockNumber: 0, + }, + { + Name: "sepolia", + Aliases: []string{}, + GenesisBlockID: "25a5cc106eea7138acab33231d7160d69cb777ee0c2c553fcddf5138993e6dd9", + GenesisBlockNumber: 0, + }, + { + Name: "holesky", + Aliases: []string{}, + GenesisBlockID: "b5f7f912443c940f21fd611f12828d75b534364ed9e95ca4e307729a4661bde4", + GenesisBlockNumber: 0, + }, + { + Name: "matic", + Aliases: []string{"polygon"}, + GenesisBlockID: "a9c28ce2141b56c474f1dc504bee9b01eb1bd7d1a507580d5519d4437a97de1b", + GenesisBlockNumber: 0, + }, + { + Name: "bsc", + Aliases: []string{"bnb", "bsc-mainnet"}, + GenesisBlockID: "0d21840abff46b96c84b2ac9e10e4f5cdaeb5693cb665db62a2f3b02d2d57b5b", + GenesisBlockNumber: 0, + }, + { + Name: "optimism", + Aliases: []string{}, + GenesisBlockID: "7ca38a1916c42007829c55e69d3e9a73265554b586a499015373241b8a3fa48b", + GenesisBlockNumber: 0, + }, + { + Name: "optimism-sepolia", + Aliases: []string{}, + GenesisBlockID: "102de6ffb001480cc9b8b548fd05c34cd4f46ae4aa91759393db90ea0409887d", + GenesisBlockNumber: 0, + }, + { + Name: "chapel", + Aliases: []string{"bsc-chapel", "bsc-testnet"}, + GenesisBlockID: "6d3c66c5357ec91d5c43af47e234a939b22557cbb552dc45bebbceeed90fbe34", + GenesisBlockNumber: 0, + }, + { + Name: "arbitrum-one", + Aliases: []string{"arb-one", "arbitrum"}, + GenesisBlockID: "7ee576b35482195fc49205cec9af72ce14f003b9ae69f6ba0faef4514be8b442", + GenesisBlockNumber: 0, + }, + // We do not auto-discover avalanche because the genesis block ID is the same as their testnet fuji and we can't differentiate them + //{ + // Name: "avalanche", + // Aliases: []string{"avax"}, + // GenesisBlockID: "31ced5b9beb7f8782b014660da0cb18cc409f121f408186886e1ca3e8eeca96b", + // GenesisBlockNumber: 0, + //}, + }, + }, + { + Name: "near", + BlockType: "type.googleapis.com/sf.near.type.v1.Block", + BufBuildURL: "buf.build/streamingfast/firehose-near", + BytesEncoding: pbfirehose.InfoResponse_BLOCK_ID_ENCODING_BASE58, + KnownChains: []*Chain{ + { + Name: "near-mainnet", + Aliases: []string{"near"}, + GenesisBlockID: "CFAAJTVsw5y4GmMKNmuTNybxFJtapKcrarsTh5TPUyQf", + GenesisBlockNumber: 9820214, + }, + { + Name: "near-testnet", + Aliases: []string{}, + GenesisBlockID: "fQURSjwQKZn8F98ayQjpndh85msJBu12FBkUY1gc5WA", + GenesisBlockNumber: 42376923, + }, + }, + }, + { + Name: "solana", + BlockType: "type.googleapis.com/sf.solana.type.v1.Block", + BufBuildURL: "buf.build/streamingfast/firehose-solana", + BytesEncoding: pbfirehose.InfoResponse_BLOCK_ID_ENCODING_BASE58, + KnownChains: []*Chain{ + { + Name: "solana-mainnet-beta", + Aliases: []string{"solana", "solana-mainnet"}, + GenesisBlockID: "4sGjMW1sUnHzSxGspuhpqLDx6wiyjNtZAMdL4VZHirAn", + GenesisBlockNumber: 0, + }, + }, + }, + { + Name: "bitcoin", + BlockType: "type.googleapis.com/sf.bitcoin.type.v1.Block", + BufBuildURL: "buf.build/streamingfast/firehose-bitcoin", + BytesEncoding: pbfirehose.InfoResponse_BLOCK_ID_ENCODING_HEX, + KnownChains: []*Chain{ + { + Name: "btc", + Aliases: []string{"bitcoin"}, + GenesisBlockID: "000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f", + GenesisBlockNumber: 0, + }, + }, + }, + { + Name: "antelope", + BlockType: "type.googleapis.com/sf.antelope.type.v1.Block", + BufBuildURL: "buf.build/pinax/firehose-antelope", + BytesEncoding: pbfirehose.InfoResponse_BLOCK_ID_ENCODING_HEX, + KnownChains: []*Chain{ + { + Name: "eos", + Aliases: []string{"eos-mainnet"}, + GenesisBlockID: "0000000267f3e2284b482f3afc2e724be1d6cbc1804532ec62d4e7af47c30693", + GenesisBlockNumber: 2, // even though the genesis block is 1, it is never available through firehose/substreams + }, + { + Name: "kylin", + Aliases: []string{}, + GenesisBlockID: "00000002a1ec7ae214b9e43a904b6c010fb1260c9e8a12e5967bdbe451152a07", + GenesisBlockNumber: 2, // even though the genesis block is 1, it is never available through firehose/substreams + }, + { + Name: "jungle4", + Aliases: []string{}, + GenesisBlockID: "00000002d61d836f51657f886a5bc55b18a731f7eace6565784328fbd051fc90", + GenesisBlockNumber: 2, // even though the genesis block is 1, it is never available through firehose/substreams + }, + }, + }, + { + Name: "arweave", + BlockType: "type.googleapis.com/sf.arweave.type.v1.Block", + BufBuildURL: "buf.build/pinax/firehose-arweave", + BytesEncoding: pbfirehose.InfoResponse_BLOCK_ID_ENCODING_HEX, // even though the usual encoding is base64url, firehose blocks are written with the hex-encoded version + KnownChains: []*Chain{ + { + Name: "arweave", + Aliases: []string{}, + GenesisBlockID: "ef0214ecaa252020230a5325719dfc2d9cec86123bc46926dad0c2251ed6be17b7112528dbe678fb2d31d6e6a0951244", + GenesisBlockNumber: 0, + }, + }, + }, + { + Name: "beacon", + BlockType: "type.googleapis.com/sf.beacon.type.v1.Block", + BufBuildURL: "buf.build/pinax/firehose-beacon", + BytesEncoding: pbfirehose.InfoResponse_BLOCK_ID_ENCODING_0X_HEX, + KnownChains: []*Chain{ + { + Name: "mainnet-cl", + Aliases: []string{}, + GenesisBlockID: "0x4d611d5b93fdab69013a7f0a2f961caca0c853f87cfe9595fe50038163079360", + GenesisBlockNumber: 0, + }, + { + Name: "sepolia-cl", + Aliases: []string{}, + GenesisBlockID: "0xfb9b64fe445f76696407e1e3cc390371edff147bf712db86db6197d4b31ede43", + GenesisBlockNumber: 0, + }, + { + Name: "holesky-cl", + Aliases: []string{}, + GenesisBlockID: "0xab09edd9380f8451c3ff5c809821174a36dce606fea8b5ea35ea936915dbf889", + GenesisBlockNumber: 0, + }, + }, + }, + { + Name: "starknet", + BlockType: "type.googleapis.com/sf.starknet.type.v1.Block", + BufBuildURL: "buf.build/streamingfast/firehose-starknet", + BytesEncoding: pbfirehose.InfoResponse_BLOCK_ID_ENCODING_0X_HEX, + KnownChains: []*Chain{ + { + Name: "starknet-mainnet", + Aliases: []string{}, + GenesisBlockID: "0x47c3637b57c2b079b93c61539950c17e868a28f46cdef28f88521067f21e943", + GenesisBlockNumber: 0, + }, + { + Name: "starknet-testnet", + Aliases: []string{}, + GenesisBlockID: "0x5c627d4aeb51280058bed93c7889bce78114d63baad1be0f0aeb32496d5f19c", + GenesisBlockNumber: 0, + }, + }, + }, + { + Name: "cosmos", + BlockType: "type.googleapis.com/sf.cosmos.type.v2.Block", + BufBuildURL: "buf.build/streamingfast/firehose-cosmos", + BytesEncoding: pbfirehose.InfoResponse_BLOCK_ID_ENCODING_HEX, + KnownChains: []*Chain{ + { + Name: "injective-mainnet", + Aliases: []string{}, + GenesisBlockID: "24c9714291a999b952859ee02ec9b233394fe743b07ea3578d432a4a2707b6af", + GenesisBlockNumber: 1, + }, + { + Name: "injective-testnet", + Aliases: []string{}, + GenesisBlockID: "a9effb99c7bc3ba8c18a487ffffd800c137bc2b2f47f73c350f3ca27077044a1", + GenesisBlockNumber: 37368800, // Not the real genesis block, but the other blocks are lost on the testnet + }, + }, + }, + { + Name: "gear", + BlockType: "type.googleapis.com/sf.gear.type.v1.Block", + BufBuildURL: "buf.build/streamingfast/firehose-gear", + BytesEncoding: pbfirehose.InfoResponse_BLOCK_ID_ENCODING_HEX, + KnownChains: []*Chain{ + { + Name: "vara-mainnet", + Aliases: []string{}, + GenesisBlockID: "fe1b4c55fd4d668101126434206571a7838a8b6b93a6d1b95d607e78e6c53763", + GenesisBlockNumber: 0, + }, + { + Name: "vara-testnet", + Aliases: []string{}, + GenesisBlockID: "525639f713f397dcf839bd022cd821f367ebcf179de7b9253531f8adbe5436d6", + GenesisBlockNumber: 0, + }, + }, + }, +}) + +func (p WellKnownProtocolList) ChainByGenesisBlock(blockNum uint64, blockID string) *Chain { + for _, protocol := range p { + for _, chain := range protocol.KnownChains { + if chain.GenesisBlockNumber == blockNum && chain.GenesisBlockID == blockID { + return chain + } + } + } + return nil +} + +func (p WellKnownProtocolList) ChainByName(name string) *Chain { + for _, protocol := range p { + for _, chain := range protocol.KnownChains { + if chain.Name == name { + return chain + } + for _, alias := range chain.Aliases { + if alias == name { + return chain + } + } + } + } + return nil +}