Refactor Configuration

packages/indexer-agent/CHANGELOG.md

@@ -5,6 +5,16 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Changed
+- Startup parameter `--ethereum` has been renamed to `--network-provider`
+- The Agent can now be configured with multiple networks: The startup parameters
+  `--network-provider`, `--network-subgraph-endpoint`, `--network-subgraph-deployment` and
+  `--epochSubgraph` can be declared multiple times using the `<network identifier>:<parameter
+  value>`, where the `<network_identifier>` part can be either a human friendly chain alias (such as
+  `mainnet` or `arbitrum-one`), or a CAIP-2 (such as `eip155:1` or `eip155:42161`).
+
+  A startup check ensures that parameters are consistent and usable by validating if they have the
+  same length and network identification pattern.
 
 ## [0.20.12] - 2023-02-19
 ### Changed
@@ -431,7 +441,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Update @graphprotocol/common-ts to 0.2.2
 
 [Unreleased]: https://github.com/graphprotocol/indexer/compare/v0.20.14...HEAD
-[0.20.14]: https://github.com/graphprotocol/indexer/compare/v0.20.12...v0.20.14
 [0.20.12]: https://github.com/graphprotocol/indexer/compare/v0.20.11...v0.20.12
 [0.20.11]: https://github.com/graphprotocol/indexer/compare/v0.20.9...v0.20.11
 [0.20.9]: https://github.com/graphprotocol/indexer/compare/v0.20.7...v0.20.9

packages/indexer-agent/package.json

@@ -43,12 +43,14 @@
     "graphql-tag": "2.12.6",
     "isomorphic-fetch": "3.0.0",
     "jayson": "3.6.6",
+    "lodash.countby": "^4.6.0",
     "ngeohash": "0.6.3",
     "p-filter": "2.1.0",
     "p-map": "4.0.0",
     "p-queue": "6.6.2",
     "p-reduce": "2.1.0",
     "p-retry": "4.6.1",
+    "parsimmon": "^1.18.1",
     "umzug": "3.0.0",
     "yaml": "^2.0.0-10",
     "yargs": "17.4.1"
@@ -57,8 +59,10 @@
     "@types/bs58": "4.0.1",
     "@types/isomorphic-fetch": "0.0.36",
     "@types/jest": "27.4.1",
+    "@types/lodash.countby": "^4.6.7",
     "@types/ngeohash": "0.6.4",
     "@types/node": "17.0.23",
+    "@types/parsimmon": "^1.10.6",
     "@types/yargs": "17.0.10",
     "@typescript-eslint/eslint-plugin": "5.19.0",
     "@typescript-eslint/parser": "5.19.0",

packages/indexer-agent/src/__tests__/input-parsers.ts

@@ -0,0 +1,91 @@
+import { parseTaggedUrl, parseTaggedIpfsHash } from '../commands/input-parsers'
+
+const testUrlString = 'https://example.com/path/to/resource'
+const testUrl = new URL(testUrlString)
+const testCid = 'QmRKs2ZfuwvmZA3QAWmCqrGUjV9pxtBUDP3wuc6iVGnjA2'
+
+describe('parseTaggedUrl tests', () => {
+  it('should parse a URL without network id', () => {
+    const expected = { networkId: null, url: testUrl }
+    const actual = parseTaggedUrl(testUrlString)
+    expect(actual).toEqual(expected)
+  })
+
+  it('should parse a URL prefixed with a network CAIP-2 id', () => {
+    const input = `eip155:1:${testUrlString}`
+    const expected = {
+      networkId: 'eip155:1',
+      url: testUrl,
+    }
+    const actual = parseTaggedUrl(input)
+    expect(actual).toEqual(expected)
+  })
+
+  it('should parse a URL prefixed with a network network alias', () => {
+    const input = `arbitrum-one:${testUrlString}`
+    const expected = {
+      networkId: 'eip155:42161',
+      url: testUrl,
+    }
+    const actual = parseTaggedUrl(input)
+    expect(actual).toEqual(expected)
+  })
+
+  it('should throw an error if the input is not a valid URL', () => {
+    expect(() => parseTaggedUrl('not-a-valid-url')).toThrow()
+  })
+
+  it('should throw an error if the input is not a valid URL, even if prefixed with a valid network id', () => {
+    expect(() => parseTaggedUrl('mainnet:not-a-valid-url')).toThrow()
+  })
+
+  it('should throw an error if the network id is not supported', () => {
+    const input = 'eip155:0:${testUrlString}'
+    expect(() => parseTaggedUrl(input)).toThrow()
+  })
+
+  it('should throw an error if the network id is malformed', () => {
+    const input = 'not/a/chain/alias:${testUrlString}'
+    expect(() => parseTaggedUrl(input)).toThrow()
+  })
+})
+
+describe('parseTaggedIpfsHash tests', () => {
+  it('should parse an IPFS hash without network id', () => {
+    const expected = { networkId: null, cid: testCid }
+    const actual = parseTaggedIpfsHash(testCid)
+    expect(actual).toEqual(expected)
+  })
+
+  it('should parse an IPFS hash prefixed with a network id', () => {
+    const input = `eip155:1:${testCid}`
+    const expected = { networkId: 'eip155:1', cid: testCid }
+    const actual = parseTaggedIpfsHash(input)
+    expect(actual).toEqual(expected)
+  })
+
+  it('should parse an IPFS Hash prefixed with a network network alias', () => {
+    const input = `goerli:${testCid}`
+    const expected = { networkId: 'eip155:5', cid: testCid }
+    const actual = parseTaggedIpfsHash(input)
+    expect(actual).toEqual(expected)
+  })
+
+  it('should throw an error if the input is not a valid IPFS Hash', () => {
+    expect(() => parseTaggedIpfsHash('not-a-valid-ipfs-hash')).toThrow()
+  })
+
+  it('should throw an error if the input is not a valid IPFS Hash, even if prefixed with a valid network id', () => {
+    expect(() => parseTaggedIpfsHash('mainnet:not-a-valid-ipfs-hash')).toThrow()
+  })
+
+  it('should throw an error if the network id is not supported', () => {
+    const input = 'eip155:0:${testCid}'
+    expect(() => parseTaggedIpfsHash(input)).toThrow()
+  })
+
+  it('should throw an error if the network id is malformed', () => {
+    const input = 'not/a/chain/alias:${testCid}'
+    expect(() => parseTaggedIpfsHash(input)).toThrow()
+  })
+})

packages/indexer-agent/src/__tests__/start.ts

@@ -0,0 +1,252 @@
+import { validateNetworkOptions, AgentOptions } from '../commands/start'
+
+const unbalancedOptionsErrorMessage =
+  'Indexer-Agent was configured with an unbalanced argument number for these options: [--network-provider, --epoch-subgraph-endpoint, --network-subgraph-endpoint, --network-subgraph-deployment]. Ensure that every option cotains an equal number of arguments.'
+const mixedNetworkIdentifiersErrorMessage =
+  'Indexer-Agent was configured with mixed network identifiers for these options: [--network-provider, --epoch-subgraph-endpoint, --network-subgraph-endpoint, --network-subgraph-deployment]. Ensure that every network identifier is equally used among options.'
+const duplicateNetworkIdentifiersErrorMessage =
+  'Indexer-Agent was configured with duplicate network identifiers for these options: [--network-provider, --epoch-subgraph-endpoint, --network-subgraph-endpoint, --network-subgraph-deployment]. Ensure that each network identifier is used at most once.'
+const cid = 'QmPK1s3pNYLi9ERiq3BDxKa4XosgWwFRQUydHUtz4YgpBq'
+
+describe('validateNetworkOptions tests', () => {
+  it('should parse unidentified network options correctly and reassign them back to their source', () => {
+    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+    // @ts-ignore: Mock this value's type for this test
+    const options: AgentOptions = {
+      networkSubgraphEndpoint: ['https://subgraph1'],
+      networkSubgraphDeployment: [cid],
+      networkProvider: ['http://provider'],
+      epochSubgraphEndpoint: ['http://epoch-subgraph'],
+    }
+    validateNetworkOptions(options)
+
+    expect(options.networkSubgraphEndpoint).toEqual([
+      {
+        networkId: null,
+        url: new URL('https://subgraph1/'),
+      },
+    ])
+    expect(options.networkSubgraphDeployment).toEqual([
+      {
+        networkId: null,
+        cid,
+      },
+    ])
+    expect(options.networkProvider).toEqual([
+      {
+        networkId: null,
+        url: new URL('http://provider'),
+      },
+    ])
+    expect(options.epochSubgraphEndpoint).toEqual([
+      {
+        networkId: null,
+        url: new URL('http://epoch-subgraph'),
+      },
+    ])
+  })
+
+  it('should parse network options correctly and reassign them back to their source', () => {
+    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+    // @ts-ignore: Mock this value's type for this test
+    const options: AgentOptions = {
+      networkSubgraphEndpoint: ['mainnet:https://subgraph1'],
+      networkSubgraphDeployment: [`mainnet:${cid}`],
+      networkProvider: ['mainnet:http://provider'],
+      epochSubgraphEndpoint: ['mainnet:http://epoch-subgraph'],
+    }
+    validateNetworkOptions(options)
+
+    expect(options.networkSubgraphEndpoint).toEqual([
+      {
+        networkId: 'eip155:1',
+        url: new URL('https://subgraph1/'),
+      },
+    ])
+    expect(options.networkSubgraphDeployment).toEqual([
+      {
+        networkId: 'eip155:1',
+        cid,
+      },
+    ])
+    expect(options.networkProvider).toEqual([
+      {
+        networkId: 'eip155:1',
+        url: new URL('http://provider'),
+      },
+    ])
+    expect(options.epochSubgraphEndpoint).toEqual([
+      {
+        networkId: 'eip155:1',
+        url: new URL('http://epoch-subgraph'),
+      },
+    ])
+  })
+
+  it('should parse multiple network option pairs correctly', () => {
+    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+    // @ts-ignore: Mock this value's type for this test
+    const options: AgentOptions = {
+      networkSubgraphEndpoint: [
+        'mainnet:https://subgraph-1',
+        'goerli:https://subgraph-2',
+      ],
+      networkSubgraphDeployment: [`mainnet:${cid}`, `goerli:${cid}`],
+      networkProvider: [
+        'mainnet:http://provider-1',
+        'goerli:http://provider-2',
+      ],
+      epochSubgraphEndpoint: [
+        'mainnet:http://epoch-subgraph-1',
+        'goerli:http://epoch-subgraph-2',
+      ],
+    }
+    validateNetworkOptions(options)
+
+    expect(options.networkSubgraphEndpoint).toEqual([
+      {
+        networkId: 'eip155:1',
+        url: new URL('https://subgraph-1'),
+      },
+      {
+        networkId: 'eip155:5',
+        url: new URL('https://subgraph-2'),
+      },
+    ])
+    expect(options.networkSubgraphDeployment).toEqual([
+      {
+        networkId: 'eip155:1',
+        cid,
+      },
+      {
+        networkId: 'eip155:5',
+        cid,
+      },
+    ])
+    expect(options.networkProvider).toEqual([
+      {
+        networkId: 'eip155:1',
+        url: new URL('http://provider-1'),
+      },
+      {
+        networkId: 'eip155:5',
+        url: new URL('http://provider-2'),
+      },
+    ])
+    expect(options.epochSubgraphEndpoint).toEqual([
+      {
+        networkId: 'eip155:1',
+        url: new URL('http://epoch-subgraph-1'),
+      },
+      {
+        networkId: 'eip155:5',
+        url: new URL('http://epoch-subgraph-2'),
+      },
+    ])
+  })
+
+  it('should throw an error if neither networkSubgraphEndpoint nor networkSubgraphDeployment is provided', () => {
+    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+    // @ts-ignore: Mock this value's type for this test
+    const options: AgentOptions = {
+      networkSubgraphEndpoint: undefined,
+      networkSubgraphDeployment: undefined,
+    }
+    expect(() => validateNetworkOptions(options)).toThrowError(
+      'At least one of --network-subgraph-endpoint and --network-subgraph-deployment must be provided',
+    )
+  })
+
+  it('should throw an error if the length of network options is not consistent', () => {
+    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+    // @ts-ignore: Mock this value's type for this test
+    const options: AgentOptions = {
+      networkSubgraphEndpoint: [
+        'https://network-subgraph1',
+        'https://network-subgraph2',
+      ],
+      networkSubgraphDeployment: [cid],
+      networkProvider: ['http://provider'],
+      epochSubgraphEndpoint: ['http://epoch-subgraph'],
+    }
+    expect(() => validateNetworkOptions(options)).toThrowError(
+      unbalancedOptionsErrorMessage,
+    )
+  })
+
+  describe('should throw an error if the network identifiers are not balanced', () => {
+    it('by omission', () => {
+      // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+      // @ts-ignore: Mock this value's type for this test
+      const options: AgentOptions = {
+        networkSubgraphEndpoint: ['https://network-subgraph'],
+        networkSubgraphDeployment: [`mainnet:${cid}`],
+        networkProvider: ['mainnet:http://provider'],
+        epochSubgraphEndpoint: ['mainnet:http://epoch-subgraph'],
+      }
+      expect(() => validateNetworkOptions(options)).toThrowError(
+        mixedNetworkIdentifiersErrorMessage,
+      )
+    })
+
+    it('by difference', () => {
+      // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+      // @ts-ignore: Mock this value's type for this test
+      const options: AgentOptions = {
+        networkSubgraphEndpoint: ['goerli:https://network-subgraph'],
+        networkSubgraphDeployment: [`mainnet:${cid}`],
+        networkProvider: ['mainnet:http://provider'],
+        epochSubgraphEndpoint: ['mainnet:http://epoch-subgraph'],
+      }
+      expect(() => validateNetworkOptions(options)).toThrowError(
+        mixedNetworkIdentifiersErrorMessage,
+      )
+    })
+
+    it('by duplication', () => {
+      // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+      // @ts-ignore: Mock this value's type for this test
+      const options: AgentOptions = {
+        networkSubgraphEndpoint: [
+          'mainnet:https://network-subgraph-1',
+          'mainnet:https://network-subgraph-2',
+        ],
+        networkSubgraphDeployment: [
+          `mainnet:${cid}`,
+          `mainnet:${cid.replace('a', 'b')}`,
+        ],
+        networkProvider: [
+          'mainnet:http://provider-1',
+          'mainnet:http://provider-2',
+        ],
+        epochSubgraphEndpoint: [
+          'mainnet:http://epoch-subgraph-1',
+          'mainnet:http://epoch-subgraph-2',
+        ],
+      }
+      expect(() => validateNetworkOptions(options)).toThrowError(
+        duplicateNetworkIdentifiersErrorMessage,
+      )
+    })
+  })
+
+  it('should throw an error if identified options are mixed with unidentified ones', () => {
+    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+    // @ts-ignore: Mock this value's type for this test
+    const options: AgentOptions = {
+      networkSubgraphEndpoint: [
+        'mainnet:https://subgraph-1',
+        'https://subgraph-2',
+      ],
+      networkSubgraphDeployment: [`mainnet:${cid}`, `goerli:${cid}`],
+      networkProvider: ['mainnet:http://provider-1', 'http://provider-2'],
+      epochSubgraphEndpoint: [
+        'mainnet:http://epoch-subgraph-1',
+        'http://epoch-subgraph-2',
+      ],
+    }
+    expect(() => validateNetworkOptions(options)).toThrow(
+      mixedNetworkIdentifiersErrorMessage,
+    )
+  })
+})