diff --git a/packages/indexer-agent/CHANGELOG.md b/packages/indexer-agent/CHANGELOG.md index afa3bb15d..3325f03b4 100644 --- a/packages/indexer-agent/CHANGELOG.md +++ b/packages/indexer-agent/CHANGELOG.md @@ -5,6 +5,16 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Changed +- Startup parameter `--ethereum` has been renamed to `--network-provider` +- The Agent can now be configured with multiple networks: The startup parameters + `--network-provider`, `--network-subgraph-endpoint`, `--network-subgraph-deployment` and + `--epochSubgraph` can be declared multiple times using the `:`, where the `` part can be either a human friendly chain alias (such as + `mainnet` or `arbitrum-one`), or a CAIP-2 (such as `eip155:1` or `eip155:42161`). + + A startup check ensures that parameters are consistent and usable by validating if they have the + same length and network identification pattern. ## [0.20.12] - 2023-02-19 ### Changed @@ -431,7 +441,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Update @graphprotocol/common-ts to 0.2.2 [Unreleased]: https://github.com/graphprotocol/indexer/compare/v0.20.14...HEAD -[0.20.14]: https://github.com/graphprotocol/indexer/compare/v0.20.12...v0.20.14 [0.20.12]: https://github.com/graphprotocol/indexer/compare/v0.20.11...v0.20.12 [0.20.11]: https://github.com/graphprotocol/indexer/compare/v0.20.9...v0.20.11 [0.20.9]: https://github.com/graphprotocol/indexer/compare/v0.20.7...v0.20.9 diff --git a/packages/indexer-agent/package.json b/packages/indexer-agent/package.json index 1a0738943..96423c512 100644 --- a/packages/indexer-agent/package.json +++ b/packages/indexer-agent/package.json @@ -43,12 +43,14 @@ "graphql-tag": "2.12.6", "isomorphic-fetch": "3.0.0", "jayson": "3.6.6", + "lodash.countby": "^4.6.0", "ngeohash": "0.6.3", "p-filter": "2.1.0", "p-map": "4.0.0", "p-queue": "6.6.2", "p-reduce": "2.1.0", "p-retry": "4.6.1", + "parsimmon": "^1.18.1", "umzug": "3.0.0", "yaml": "^2.0.0-10", "yargs": "17.4.1" @@ -57,8 +59,10 @@ "@types/bs58": "4.0.1", "@types/isomorphic-fetch": "0.0.36", "@types/jest": "27.4.1", + "@types/lodash.countby": "^4.6.7", "@types/ngeohash": "0.6.4", "@types/node": "17.0.23", + "@types/parsimmon": "^1.10.6", "@types/yargs": "17.0.10", "@typescript-eslint/eslint-plugin": "5.19.0", "@typescript-eslint/parser": "5.19.0", diff --git a/packages/indexer-agent/src/__tests__/input-parsers.ts b/packages/indexer-agent/src/__tests__/input-parsers.ts new file mode 100644 index 000000000..a4fc394ad --- /dev/null +++ b/packages/indexer-agent/src/__tests__/input-parsers.ts @@ -0,0 +1,91 @@ +import { parseTaggedUrl, parseTaggedIpfsHash } from '../commands/input-parsers' + +const testUrlString = 'https://example.com/path/to/resource' +const testUrl = new URL(testUrlString) +const testCid = 'QmRKs2ZfuwvmZA3QAWmCqrGUjV9pxtBUDP3wuc6iVGnjA2' + +describe('parseTaggedUrl tests', () => { + it('should parse a URL without network id', () => { + const expected = { networkId: null, url: testUrl } + const actual = parseTaggedUrl(testUrlString) + expect(actual).toEqual(expected) + }) + + it('should parse a URL prefixed with a network CAIP-2 id', () => { + const input = `eip155:1:${testUrlString}` + const expected = { + networkId: 'eip155:1', + url: testUrl, + } + const actual = parseTaggedUrl(input) + expect(actual).toEqual(expected) + }) + + it('should parse a URL prefixed with a network network alias', () => { + const input = `arbitrum-one:${testUrlString}` + const expected = { + networkId: 'eip155:42161', + url: testUrl, + } + const actual = parseTaggedUrl(input) + expect(actual).toEqual(expected) + }) + + it('should throw an error if the input is not a valid URL', () => { + expect(() => parseTaggedUrl('not-a-valid-url')).toThrow() + }) + + it('should throw an error if the input is not a valid URL, even if prefixed with a valid network id', () => { + expect(() => parseTaggedUrl('mainnet:not-a-valid-url')).toThrow() + }) + + it('should throw an error if the network id is not supported', () => { + const input = 'eip155:0:${testUrlString}' + expect(() => parseTaggedUrl(input)).toThrow() + }) + + it('should throw an error if the network id is malformed', () => { + const input = 'not/a/chain/alias:${testUrlString}' + expect(() => parseTaggedUrl(input)).toThrow() + }) +}) + +describe('parseTaggedIpfsHash tests', () => { + it('should parse an IPFS hash without network id', () => { + const expected = { networkId: null, cid: testCid } + const actual = parseTaggedIpfsHash(testCid) + expect(actual).toEqual(expected) + }) + + it('should parse an IPFS hash prefixed with a network id', () => { + const input = `eip155:1:${testCid}` + const expected = { networkId: 'eip155:1', cid: testCid } + const actual = parseTaggedIpfsHash(input) + expect(actual).toEqual(expected) + }) + + it('should parse an IPFS Hash prefixed with a network network alias', () => { + const input = `goerli:${testCid}` + const expected = { networkId: 'eip155:5', cid: testCid } + const actual = parseTaggedIpfsHash(input) + expect(actual).toEqual(expected) + }) + + it('should throw an error if the input is not a valid IPFS Hash', () => { + expect(() => parseTaggedIpfsHash('not-a-valid-ipfs-hash')).toThrow() + }) + + it('should throw an error if the input is not a valid IPFS Hash, even if prefixed with a valid network id', () => { + expect(() => parseTaggedIpfsHash('mainnet:not-a-valid-ipfs-hash')).toThrow() + }) + + it('should throw an error if the network id is not supported', () => { + const input = 'eip155:0:${testCid}' + expect(() => parseTaggedIpfsHash(input)).toThrow() + }) + + it('should throw an error if the network id is malformed', () => { + const input = 'not/a/chain/alias:${testCid}' + expect(() => parseTaggedIpfsHash(input)).toThrow() + }) +}) diff --git a/packages/indexer-agent/src/__tests__/start.ts b/packages/indexer-agent/src/__tests__/start.ts new file mode 100644 index 000000000..b8b0ae8f0 --- /dev/null +++ b/packages/indexer-agent/src/__tests__/start.ts @@ -0,0 +1,252 @@ +import { validateNetworkOptions, AgentOptions } from '../commands/start' + +const unbalancedOptionsErrorMessage = + 'Indexer-Agent was configured with an unbalanced argument number for these options: [--network-provider, --epoch-subgraph-endpoint, --network-subgraph-endpoint, --network-subgraph-deployment]. Ensure that every option cotains an equal number of arguments.' +const mixedNetworkIdentifiersErrorMessage = + 'Indexer-Agent was configured with mixed network identifiers for these options: [--network-provider, --epoch-subgraph-endpoint, --network-subgraph-endpoint, --network-subgraph-deployment]. Ensure that every network identifier is equally used among options.' +const duplicateNetworkIdentifiersErrorMessage = + 'Indexer-Agent was configured with duplicate network identifiers for these options: [--network-provider, --epoch-subgraph-endpoint, --network-subgraph-endpoint, --network-subgraph-deployment]. Ensure that each network identifier is used at most once.' +const cid = 'QmPK1s3pNYLi9ERiq3BDxKa4XosgWwFRQUydHUtz4YgpBq' + +describe('validateNetworkOptions tests', () => { + it('should parse unidentified network options correctly and reassign them back to their source', () => { + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore: Mock this value's type for this test + const options: AgentOptions = { + networkSubgraphEndpoint: ['https://subgraph1'], + networkSubgraphDeployment: [cid], + networkProvider: ['http://provider'], + epochSubgraphEndpoint: ['http://epoch-subgraph'], + } + validateNetworkOptions(options) + + expect(options.networkSubgraphEndpoint).toEqual([ + { + networkId: null, + url: new URL('https://subgraph1/'), + }, + ]) + expect(options.networkSubgraphDeployment).toEqual([ + { + networkId: null, + cid, + }, + ]) + expect(options.networkProvider).toEqual([ + { + networkId: null, + url: new URL('http://provider'), + }, + ]) + expect(options.epochSubgraphEndpoint).toEqual([ + { + networkId: null, + url: new URL('http://epoch-subgraph'), + }, + ]) + }) + + it('should parse network options correctly and reassign them back to their source', () => { + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore: Mock this value's type for this test + const options: AgentOptions = { + networkSubgraphEndpoint: ['mainnet:https://subgraph1'], + networkSubgraphDeployment: [`mainnet:${cid}`], + networkProvider: ['mainnet:http://provider'], + epochSubgraphEndpoint: ['mainnet:http://epoch-subgraph'], + } + validateNetworkOptions(options) + + expect(options.networkSubgraphEndpoint).toEqual([ + { + networkId: 'eip155:1', + url: new URL('https://subgraph1/'), + }, + ]) + expect(options.networkSubgraphDeployment).toEqual([ + { + networkId: 'eip155:1', + cid, + }, + ]) + expect(options.networkProvider).toEqual([ + { + networkId: 'eip155:1', + url: new URL('http://provider'), + }, + ]) + expect(options.epochSubgraphEndpoint).toEqual([ + { + networkId: 'eip155:1', + url: new URL('http://epoch-subgraph'), + }, + ]) + }) + + it('should parse multiple network option pairs correctly', () => { + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore: Mock this value's type for this test + const options: AgentOptions = { + networkSubgraphEndpoint: [ + 'mainnet:https://subgraph-1', + 'goerli:https://subgraph-2', + ], + networkSubgraphDeployment: [`mainnet:${cid}`, `goerli:${cid}`], + networkProvider: [ + 'mainnet:http://provider-1', + 'goerli:http://provider-2', + ], + epochSubgraphEndpoint: [ + 'mainnet:http://epoch-subgraph-1', + 'goerli:http://epoch-subgraph-2', + ], + } + validateNetworkOptions(options) + + expect(options.networkSubgraphEndpoint).toEqual([ + { + networkId: 'eip155:1', + url: new URL('https://subgraph-1'), + }, + { + networkId: 'eip155:5', + url: new URL('https://subgraph-2'), + }, + ]) + expect(options.networkSubgraphDeployment).toEqual([ + { + networkId: 'eip155:1', + cid, + }, + { + networkId: 'eip155:5', + cid, + }, + ]) + expect(options.networkProvider).toEqual([ + { + networkId: 'eip155:1', + url: new URL('http://provider-1'), + }, + { + networkId: 'eip155:5', + url: new URL('http://provider-2'), + }, + ]) + expect(options.epochSubgraphEndpoint).toEqual([ + { + networkId: 'eip155:1', + url: new URL('http://epoch-subgraph-1'), + }, + { + networkId: 'eip155:5', + url: new URL('http://epoch-subgraph-2'), + }, + ]) + }) + + it('should throw an error if neither networkSubgraphEndpoint nor networkSubgraphDeployment is provided', () => { + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore: Mock this value's type for this test + const options: AgentOptions = { + networkSubgraphEndpoint: undefined, + networkSubgraphDeployment: undefined, + } + expect(() => validateNetworkOptions(options)).toThrowError( + 'At least one of --network-subgraph-endpoint and --network-subgraph-deployment must be provided', + ) + }) + + it('should throw an error if the length of network options is not consistent', () => { + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore: Mock this value's type for this test + const options: AgentOptions = { + networkSubgraphEndpoint: [ + 'https://network-subgraph1', + 'https://network-subgraph2', + ], + networkSubgraphDeployment: [cid], + networkProvider: ['http://provider'], + epochSubgraphEndpoint: ['http://epoch-subgraph'], + } + expect(() => validateNetworkOptions(options)).toThrowError( + unbalancedOptionsErrorMessage, + ) + }) + + describe('should throw an error if the network identifiers are not balanced', () => { + it('by omission', () => { + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore: Mock this value's type for this test + const options: AgentOptions = { + networkSubgraphEndpoint: ['https://network-subgraph'], + networkSubgraphDeployment: [`mainnet:${cid}`], + networkProvider: ['mainnet:http://provider'], + epochSubgraphEndpoint: ['mainnet:http://epoch-subgraph'], + } + expect(() => validateNetworkOptions(options)).toThrowError( + mixedNetworkIdentifiersErrorMessage, + ) + }) + + it('by difference', () => { + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore: Mock this value's type for this test + const options: AgentOptions = { + networkSubgraphEndpoint: ['goerli:https://network-subgraph'], + networkSubgraphDeployment: [`mainnet:${cid}`], + networkProvider: ['mainnet:http://provider'], + epochSubgraphEndpoint: ['mainnet:http://epoch-subgraph'], + } + expect(() => validateNetworkOptions(options)).toThrowError( + mixedNetworkIdentifiersErrorMessage, + ) + }) + + it('by duplication', () => { + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore: Mock this value's type for this test + const options: AgentOptions = { + networkSubgraphEndpoint: [ + 'mainnet:https://network-subgraph-1', + 'mainnet:https://network-subgraph-2', + ], + networkSubgraphDeployment: [ + `mainnet:${cid}`, + `mainnet:${cid.replace('a', 'b')}`, + ], + networkProvider: [ + 'mainnet:http://provider-1', + 'mainnet:http://provider-2', + ], + epochSubgraphEndpoint: [ + 'mainnet:http://epoch-subgraph-1', + 'mainnet:http://epoch-subgraph-2', + ], + } + expect(() => validateNetworkOptions(options)).toThrowError( + duplicateNetworkIdentifiersErrorMessage, + ) + }) + }) + + it('should throw an error if identified options are mixed with unidentified ones', () => { + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore: Mock this value's type for this test + const options: AgentOptions = { + networkSubgraphEndpoint: [ + 'mainnet:https://subgraph-1', + 'https://subgraph-2', + ], + networkSubgraphDeployment: [`mainnet:${cid}`, `goerli:${cid}`], + networkProvider: ['mainnet:http://provider-1', 'http://provider-2'], + epochSubgraphEndpoint: [ + 'mainnet:http://epoch-subgraph-1', + 'http://epoch-subgraph-2', + ], + } + expect(() => validateNetworkOptions(options)).toThrow( + mixedNetworkIdentifiersErrorMessage, + ) + }) +}) diff --git a/packages/indexer-agent/src/commands/input-parsers.ts b/packages/indexer-agent/src/commands/input-parsers.ts new file mode 100644 index 000000000..018d67474 --- /dev/null +++ b/packages/indexer-agent/src/commands/input-parsers.ts @@ -0,0 +1,97 @@ +import P from 'parsimmon' +import { resolveChainId } from '@graphprotocol/indexer-common' + +interface MaybeTaggedUrl { + networkId: string | null + url: URL +} + +interface MaybeTaggedIpfsHash { + networkId: string | null + cid: string +} + +// Checks if the provided network identifier is supported by the Indexer Agent. +function validateNetworkIdentifier(n: string): P.Parser { + try { + const valid = resolveChainId(n) + return P.succeed(valid) + } catch { + return P.fail('a supported network identifier') + } +} + +// A basic URL parser. +const url = P.regex(/^https?:.*/) + .map(x => new URL(x)) + .desc('a valid URL') + +// Intermediary parser to tag either CAIP-2 ids or network aliases like 'mainnet' and 'arbitrum-one'. +const caip2Id = P.string('eip155:').then( + P.regex(/[0-9]+/).chain(validateNetworkIdentifier), +) +// A valid human friendly network name / alias. +const alias = P.regex(/[a-z-]+/).chain(validateNetworkIdentifier) + +// Either a CAIP-2 or an alias. +const tag = P.alt(caip2Id, alias) + +// A tag followed by a colon. +const prefixTag = tag.skip(P.string(':')) + +// Intermediary parser that can detect a 'network identifier and an URL separated by a colon. +// Returns a `MaybeTaggedUrl` tagged with the network identifier. +const taggedUrl = P.seqMap(prefixTag, url, (networkId, url) => ({ + networkId, + url, +})) + +// Intermediary parser to convert an URL to a `MaybeTaggedUrl`, in its untagged form. +const untaggedUrl = url.map(url => ({ + networkId: null, + url, +})) + +// Final parser that can handle both tagged and untagged URLs +const maybeTaggedUrl = P.alt(taggedUrl, untaggedUrl) + +// A basic `base58btc` parser for CIDv0 +const base58 = P.regex(/^Qm[1-9A-HJ-NP-Za-km-z]{44,}$/).desc( + 'An IPFS Content Identifer (Qm...)', +) + +// Intermediary parser that can detect a 'network identifier and an IPFS hash separated by a colon. +// Returns a `MaybeTaggedIpfsHash` tagged with the network identifier.. +const taggedIpfs = P.seqMap(prefixTag, base58, (networkId, cid) => ({ + networkId, + cid, +})) + +// Intermediary parser to convert an IPFS Hash to a `MaybeTaggedIpfsHash`, in its untagged form. +const untaggedIpfs = base58.map(cid => ({ networkId: null, cid })) + +// Final parser that can handle both tagged and untagged IPFS Hashes. +const maybeTaggedIpfsHash = P.alt(taggedIpfs, untaggedIpfs) + +// Generic function that takes a parser of type T and attempts to parse it from a string. If it +// fails, then it will throw an error with an explanation of what was expected, as well as the +// portion of the input that was parsed and what's remaining to parse. +function parse(parser: P.Parser, input: string): T { + const parseResult = parser.parse(input) + if (parseResult.status) { + return parseResult.value + } + const expected = parseResult.expected[0] + const parsed = input.slice(0, parseResult.index.offset) + const remaining = input.slice(parseResult.index.offset) + throw new Error( + `Failed to parse "${input}". Expected: ${expected}. Parsed up to: "${parsed}". Remaining: "${remaining}"`, + ) +} + +export function parseTaggedUrl(input: string): MaybeTaggedUrl { + return parse(maybeTaggedUrl, input) +} +export function parseTaggedIpfsHash(input: string): MaybeTaggedIpfsHash { + return parse(maybeTaggedIpfsHash, input) +} diff --git a/packages/indexer-agent/src/commands/start.ts b/packages/indexer-agent/src/commands/start.ts index 8a50e71f4..35f1a8eaf 100644 --- a/packages/indexer-agent/src/commands/start.ts +++ b/packages/indexer-agent/src/commands/start.ts @@ -3,6 +3,7 @@ import path from 'path' import { Argv } from 'yargs' import { parse as yaml_parse } from 'yaml' import { SequelizeStorage, Umzug } from 'umzug' +import countBy from 'lodash.countby' import { connectContracts, @@ -40,14 +41,20 @@ import { Network as NetworkMetadata } from '@ethersproject/networks' import { startCostModelAutomation } from '../cost' import { createSyncingServer } from '../syncing-server' import { monitorEthBalance } from '../utils' +import { parseTaggedUrl, parseTaggedIpfsHash } from './input-parsers' + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export type AgentOptions = { [key: string]: any } & Argv['argv'] export default { command: 'start', describe: 'Start the agent', builder: (yargs: Argv): Argv => { return yargs - .option('ethereum', { + .option('network-provider', { + alias: 'ethereum', description: 'Ethereum node or provider URL', + array: true, type: 'string', required: true, group: 'Ethereum', @@ -147,11 +154,13 @@ export default { }) .option('network-subgraph-deployment', { description: 'Network subgraph deployment', + array: true, type: 'string', group: 'Network Subgraph', }) .option('network-subgraph-endpoint', { description: 'Endpoint to query the network subgraph from', + array: true, type: 'string', group: 'Network Subgraph', }) @@ -163,6 +172,7 @@ export default { }) .option('epoch-subgraph-endpoint', { description: 'Endpoint to query the epoch block oracle subgraph from', + array: true, type: 'string', required: true, group: 'Protocol', @@ -343,35 +353,6 @@ export default { default: false, group: 'Disputes', }) - .check(argv => { - if ( - !argv['network-subgraph-endpoint'] && - !argv['network-subgraph-deployment'] - ) { - return `At least one of --network-subgraph-endpoint and --network-subgraph-deployment must be provided` - } - if (argv['indexer-geo-coordinates']) { - const [geo1, geo2] = argv['indexer-geo-coordinates'] - if (!+geo1 || !+geo2) { - return 'Invalid --indexer-geo-coordinates provided. Must be of format e.g.: 31.780715 -41.179504' - } - } - if (argv['gas-increase-timeout']) { - if (argv['gas-increase-timeout'] < 30000) { - return 'Invalid --gas-increase-timeout provided. Must be at least 30 seconds' - } - } - if (argv['gas-increase-factor'] <= 1.0) { - return 'Invalid --gas-increase-factor provided. Must be > 1.0' - } - if ( - !Number.isInteger(argv['rebate-claim-max-batch-size']) || - argv['rebate-claim-max-batch-size'] <= 0 - ) { - return 'Invalid --rebate-claim-max-batch-size provided. Must be > 0 and an integer.' - } - return true - }) .option('collect-receipts-endpoint', { description: 'Client endpoint for collecting receipts', type: 'string', @@ -399,11 +380,37 @@ export default { return yaml_parse(fs.readFileSync(cfgFilePath, 'utf-8')) }, }) + .check(argv => { + try { + validateNetworkOptions(argv) + } catch (error) { + return error.message + } + + if (argv['indexer-geo-coordinates']) { + const [geo1, geo2] = argv['indexer-geo-coordinates'] + if (!+geo1 || !+geo2) { + return 'Invalid --indexer-geo-coordinates provided. Must be of format e.g.: 31.780715 -41.179504' + } + } + if (argv['gas-increase-timeout']) { + if (argv['gas-increase-timeout'] < 30000) { + return 'Invalid --gas-increase-timeout provided. Must be at least 30 seconds' + } + } + if (argv['gas-increase-factor'] <= 1.0) { + return 'Invalid --gas-increase-factor provided. Must be > 1.0' + } + if ( + !Number.isInteger(argv['rebate-claim-max-batch-size']) || + argv['rebate-claim-max-batch-size'] <= 0 + ) { + return 'Invalid --rebate-claim-max-batch-size provided. Must be > 0 and an integer.' + } + return true + }) }, - handler: async ( - // eslint-disable-next-line @typescript-eslint/no-explicit-any - argv: { [key: string]: any } & Argv['argv'], - ): Promise => { + handler: async (argv: AgentOptions): Promise => { const logger = createLogger({ name: 'IndexerAgent', async: false, @@ -493,12 +500,12 @@ export default { // Parse the Network Subgraph optional argument const networkSubgraphDeploymentId = argv.networkSubgraphDeployment - ? new SubgraphDeploymentID(argv.networkSubgraphDeployment) + ? new SubgraphDeploymentID(argv.networkSubgraphDeployment[0].cid) // FIXME: Use multiple network subgraphs. : undefined const networkSubgraph = await NetworkSubgraph.create({ logger, - endpoint: argv.networkSubgraphEndpoint, + endpoint: argv.networkSubgraphEndpoint?.[0].url.toString(), // FIXME: Use multiple network subgraphs. deployment: networkSubgraphDeploymentId !== undefined ? { @@ -512,7 +519,7 @@ export default { const networkProvider = await Network.provider( logger, metrics, - argv.ethereum, + argv.networkProvider[0].url.toString(), // FIXME: Use multiple providers. argv.ethereumPollingInterval, ) @@ -557,7 +564,9 @@ export default { const indexerAddress = toAddress(argv.indexerAddress) - const epochSubgraph = await EpochSubgraph.create(argv.epochSubgraphEndpoint) + const epochSubgraph = await EpochSubgraph.create( + argv.epochSubgraphEndpoint[0].url.toString(), // FIXME: use multiple epoch subgraphs + ) const networkMonitor = new NetworkMonitor( resolveChainId(networkMeta.chainId), @@ -725,7 +734,7 @@ export default { try { await validateNetworkId( networkMeta, - argv.networkSubgraphDeployment, + argv.networkSubgraphDeployment[0].cid, // FIXME: Use multiple network subgraphs. indexingStatusResolver, logger, ) @@ -808,3 +817,71 @@ async function validateNetworkId( throw new Error(errorMsg) } } + +export function validateNetworkOptions(argv: AgentOptions) { + // Check if at least one of those two options is being used + if (!argv.networkSubgraphEndpoint && !argv.networkSubgraphDeployment) { + throw new Error( + 'At least one of --network-subgraph-endpoint and --network-subgraph-deployment must be provided', + ) + } + + // Parse each option group, making a special case for the Network Subgraph options that can be + // partially defined. + const providers = argv.networkProvider.map(parseTaggedUrl) + const epochSubgraphs = argv.epochSubgraphEndpoint.map(parseTaggedUrl) + const networkSubgraphEndpoints = + argv.networkSubgraphEndpoint?.map(parseTaggedUrl) + const networkSubgraphDeployments = + argv.networkSubgraphDeployment?.map(parseTaggedIpfsHash) + + // Refine which option lists to check, while formatting a string with the used ones. + const arraysToCheck = [providers, epochSubgraphs] + let usedOptions = '[--network-provider, --epoch-subgraph-endpoint' + if (networkSubgraphEndpoints !== undefined) { + arraysToCheck.push(networkSubgraphEndpoints) + usedOptions += ', --network-subgraph-endpoint' + } + if (networkSubgraphDeployments !== undefined) { + arraysToCheck.push(networkSubgraphDeployments) + usedOptions += ', --network-subgraph-deployment' + } + usedOptions += ']' + + // Check for consistent length across network options + const commonSize = new Set(arraysToCheck.map(a => a.length)).size + if (commonSize !== 1) { + throw new Error( + `Indexer-Agent was configured with an unbalanced argument number for these options: ${usedOptions}. ` + + 'Ensure that every option cotains an equal number of arguments.', + ) + } + + // Check for consistent network identification + const networkIdCount = countBy(arraysToCheck.flat(), a => a.networkId) + const commonIdCount = new Set(Object.values(networkIdCount)).size + if (commonIdCount !== 1) { + throw new Error( + `Indexer-Agent was configured with mixed network identifiers for these options: ${usedOptions}. ` + + 'Ensure that every network identifier is equally used among options.', + ) + } + + // Check for duplicated network identification + for (const optionGroup of arraysToCheck) { + const usedNetworks = countBy(optionGroup, option => option.networkId) + const maxUsed = Math.max(...Object.values(usedNetworks)) + if (maxUsed > 1) { + throw new Error( + `Indexer-Agent was configured with duplicate network identifiers for these options: ${usedOptions}. ` + + 'Ensure that each network identifier is used at most once.', + ) + } + } + + // Validation finished. Assign the parsed values to their original sources. + argv.networkProvider = providers + argv.epochSubgraphEndpoint = epochSubgraphs + argv.networkSubgraphEndpoint = networkSubgraphEndpoints + argv.networkSubgraphDeployment = networkSubgraphDeployments +} diff --git a/packages/indexer-common/src/indexer-management/types.ts b/packages/indexer-common/src/indexer-management/types.ts index 14183c08e..afe75b8b7 100644 --- a/packages/indexer-common/src/indexer-management/types.ts +++ b/packages/indexer-common/src/indexer-management/types.ts @@ -154,7 +154,7 @@ export function epochElapsedBlocks(networkEpoch: NetworkEpoch): number { return networkEpoch.startBlockNumber - networkEpoch.latestBlock } -const Caip2ByChainAlias: { [key: string]: string } = { +export const Caip2ByChainAlias: { [key: string]: string } = { mainnet: 'eip155:1', goerli: 'eip155:5', gnosis: 'eip155:100', @@ -168,7 +168,7 @@ const Caip2ByChainAlias: { [key: string]: string } = { fantom: 'eip155:250', } -const Caip2ByChainId: { [key: number]: string } = { +export const Caip2ByChainId: { [key: number]: string } = { 1: 'eip155:1', 5: 'eip155:5', 100: 'eip155:100', diff --git a/yarn.lock b/yarn.lock index 3f0851367..59b34d390 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2828,6 +2828,18 @@ resolved "https://registry.npmjs.org/@types/json5/-/json5-0.0.29.tgz#ee28707ae94e11d2b827bcbe5270bcea7f3e71ee" integrity sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ== +"@types/lodash.countby@^4.6.7": + version "4.6.7" + resolved "https://registry.npmjs.org/@types/lodash.countby/-/lodash.countby-4.6.7.tgz#9dfa94ff43823c314c70056c18c00adfb8fa7cc3" + integrity sha512-RkkfnOXscBXuRc9Iay+tMHtaztdRCtU7doEwi5yLEV/7UHRyy5yS+ppZCzFl06s6lWt6fcHKAN8F6kyes6js5g== + dependencies: + "@types/lodash" "*" + +"@types/lodash@*": + version "4.14.194" + resolved "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.194.tgz#b71eb6f7a0ff11bff59fc987134a093029258a76" + integrity sha512-r22s9tAS7imvBt2lyHC9B8AGwWnXaYb1tY09oyLkXDs4vArpYJzw09nj8MLx5VfciBPGIb+ZwG0ssYnEPJxn/g== + "@types/lodash@^4.14.159": version "4.14.182" resolved "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.182.tgz" @@ -2895,6 +2907,11 @@ resolved "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.0.tgz" integrity sha512-//oorEZjL6sbPcKUaCdIGlIUeH26mgzimjBB77G6XRgnDl/L5wOnpyBGRe/Mmf5CVW3PwEBE1NjiMZ/ssFh4wA== +"@types/parsimmon@^1.10.6": + version "1.10.6" + resolved "https://registry.npmjs.org/@types/parsimmon/-/parsimmon-1.10.6.tgz#8fcf95990514d2a7624aa5f630c13bf2427f9cdd" + integrity sha512-FwAQwMRbkhx0J6YELkwIpciVzCcgEqXEbIrIn3a2P5d3kGEHQ3wVhlN3YdVepYP+bZzCYO6OjmD4o9TGOZ40rA== + "@types/pbkdf2@^3.0.0": version "3.1.0" resolved "https://registry.npmjs.org/@types/pbkdf2/-/pbkdf2-3.1.0.tgz" @@ -7520,6 +7537,11 @@ lodash.camelcase@^4.3.0: resolved "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz" integrity sha1-soqmKIorn8ZRA1x3EfZathkDMaY= +lodash.countby@^4.6.0: + version "4.6.0" + resolved "https://registry.npmjs.org/lodash.countby/-/lodash.countby-4.6.0.tgz#5351f24de16724a0059b561f920b0d80af78a33c" + integrity sha512-RhdqSKPeVL9zWY5jSYHA2PHrV+lm2x/NfZd1uCUMEJXZoqFJ14MlSjnm+otBVdmlrkJ3trLKM07wuMszZ5GIbA== + lodash.ismatch@^4.4.0: version "4.4.0" resolved "https://registry.npmjs.org/lodash.ismatch/-/lodash.ismatch-4.4.0.tgz" @@ -8718,6 +8740,11 @@ parseurl@~1.3.3: resolved "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz" integrity sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ== +parsimmon@^1.18.1: + version "1.18.1" + resolved "https://registry.npmjs.org/parsimmon/-/parsimmon-1.18.1.tgz#d8dd9c28745647d02fc6566f217690897eed7709" + integrity sha512-u7p959wLfGAhJpSDJVYXoyMCXWYwHia78HhRBWqk7AIbxdmlrfdp5wX0l3xv/iTSH5HvhN9K7o26hwwpgS5Nmw== + path-exists@^3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz"