diff --git a/README.md b/README.md index 18beb67bd..37fcc27ed 100644 --- a/README.md +++ b/README.md @@ -190,6 +190,9 @@ Indexer Infrastructure --auto-allocation-min-batch-size Minimum number of allocation transactions inside a batch for AUTO management mode [number] [default: 1] + --auto-graft-resolver-depth Maximum depth of grafting dependency to + automatically + resolve [number] [default: 0] Network Subgraph --network-subgraph-deployment Network subgraph deployment [string] diff --git a/docs/errors.md b/docs/errors.md index 124bc9f52..b29647ae1 100644 --- a/docs/errors.md +++ b/docs/errors.md @@ -854,4 +854,12 @@ Failed to resolve POI: User provided POI does not match reference fetched from t Check sync and health status of the subgraph to access the issue. If needed, provide a POI or use `--force` to bypass POI checks. +## IE069 +**Summary** + +Failed to deploy subgraph deployment graft base. + +**Solution** + +Same error as IE026, but auto-deploying as the base for a grafted subgraph. Please make sure the auto graft depth resolver has correct limit, and that the graft base deployment has synced to the graft block before trying again - Set indexing rules for periodic reconciles. diff --git a/packages/indexer-agent/src/__tests__/indexer.ts b/packages/indexer-agent/src/__tests__/indexer.ts index f965ed864..0a5afcaf5 100644 --- a/packages/indexer-agent/src/__tests__/indexer.ts +++ b/packages/indexer-agent/src/__tests__/indexer.ts @@ -139,6 +139,7 @@ const setup = async () => { }) const indexNodeIDs = ['node_1'] + const graftDepth = 1 indexerManagementClient = await createIndexerManagementClient({ models, address: toAddress(address), @@ -168,6 +169,7 @@ const setup = async () => { parseGRT('1000'), address, AllocationManagementMode.AUTO, + graftDepth, ) } diff --git a/packages/indexer-agent/src/commands/start.ts b/packages/indexer-agent/src/commands/start.ts index 04e60acd9..1783b89e5 100644 --- a/packages/indexer-agent/src/commands/start.ts +++ b/packages/indexer-agent/src/commands/start.ts @@ -328,6 +328,12 @@ export default { .map((id: string) => id.trim()) .filter((id: string) => id.length > 0), }) + .option('auto-graft-resolver-depth', { + description: `Maximum depth of grafting dependency to automatically resolve`, + type: 'number', + default: 0, + group: 'Indexer Infrastructure', + }) .option('poi-disputable-epochs', { description: 'The number of epochs in the past to look for potential POI disputes', @@ -368,6 +374,12 @@ export default { ) { return 'Invalid --rebate-claim-max-batch-size provided. Must be > 0 and an integer.' } + if ( + !Number.isInteger(argv['auto-graft-resolver-depth']) || + argv['auto-graft-resolver-depth'] < 0 + ) { + return 'Invalid --auto-graft-resolver-depth provided. Must be >= 0 and an integer.' + } return true }) .option('vector-node', { @@ -791,6 +803,7 @@ export default { receiptCollector, allocationManagementMode, autoAllocationMinBatchSize: argv.autoAllocationMinBatchSize, + autoGraftResolverDepth: argv.autoGraftResolverDepth, }) await createIndexerManagementServer({ @@ -809,6 +822,7 @@ export default { argv.defaultAllocationAmount, indexerAddress, allocationManagementMode, + argv.autoGraftResolverDepth, ) const networkSubgraphDeployment = argv.networkSubgraphDeployment ? new SubgraphDeploymentID(argv.networkSubgraphDeployment) diff --git a/packages/indexer-agent/src/indexer.ts b/packages/indexer-agent/src/indexer.ts index 404fdb2b6..1ba5b9bde 100644 --- a/packages/indexer-agent/src/indexer.ts +++ b/packages/indexer-agent/src/indexer.ts @@ -86,6 +86,7 @@ export class Indexer { defaultAllocationAmount: BigNumber indexerAddress: string allocationManagementMode: AllocationManagementMode + autoGraftResolverDepth: number constructor( logger: Logger, @@ -96,12 +97,14 @@ export class Indexer { defaultAllocationAmount: BigNumber, indexerAddress: string, allocationManagementMode: AllocationManagementMode, + autoGraftResolverDepth: number, ) { this.indexerManagement = indexerManagement this.statusResolver = statusResolver this.logger = logger this.indexerAddress = indexerAddress this.allocationManagementMode = allocationManagementMode + this.autoGraftResolverDepth = autoGraftResolverDepth if (adminEndpoint.startsWith('https')) { // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -827,23 +830,59 @@ export class Indexer { } } + graftBase = (err: Error, depth: number): SubgraphDeploymentID | undefined => { + if ( + err.message.includes( + 'subgraph validation error: [the graft base is invalid: deployment not found: ', + ) + ) { + const graftBaseQm = err.message.substring( + err.message.indexOf('Qm'), + err.message.indexOf('Qm') + 46, + ) + if (depth >= this.autoGraftResolverDepth) { + this.logger.warn( + `Grafting base depth reached auto-graft-resolver-depth limit, stop auto-grafting`, + { + base: graftBaseQm, + depth, + }, + ) + return + } + return new SubgraphDeploymentID(graftBaseQm) + } + } + async deploy( name: string, deployment: SubgraphDeploymentID, node_id: string, + depth: number, ): Promise { try { this.logger.info(`Deploy subgraph deployment`, { name, deployment: deployment.display, }) - const response = await this.rpc.request('subgraph_deploy', { + let response = await this.rpc.request('subgraph_deploy', { name, ipfs_hash: deployment.ipfsHash, node_id: node_id, }) if (response.error) { - throw response.error + const baseDeployment = this.graftBase(response.error, depth) + if (baseDeployment) { + await this.ensure(name, baseDeployment, depth + 1) + // Try deploy target deployment again, ideally do this after checking graft base sync status for graft block + response = await this.rpc.request('subgraph_deploy', { + name, + ipfs_hash: deployment.ipfsHash, + node_id: node_id, + }) + } else { + throw indexerError(IndexerErrorCode.IE026, response.error) + } } this.logger.info(`Successfully deployed subgraph deployment`, { name, @@ -917,7 +956,12 @@ export class Indexer { } } - async ensure(name: string, deployment: SubgraphDeploymentID): Promise { + async ensure( + name: string, + deployment: SubgraphDeploymentID, + depth?: number, + ): Promise { + depth = depth ?? 0 try { // Randomly assign to unused nodes if they exist, // otherwise use the node with lowest deployments assigned @@ -937,7 +981,7 @@ export class Indexer { return nodeA.deployments.length - nodeB.deployments.length })[0].id await this.create(name) - await this.deploy(name, deployment, targetNode) + await this.deploy(name, deployment, targetNode, depth) await this.reassign(deployment, targetNode) } catch (error) { const err = indexerError(IndexerErrorCode.IE020, error) diff --git a/packages/indexer-common/src/errors.ts b/packages/indexer-common/src/errors.ts index 064d04fbb..e1b9afded 100644 --- a/packages/indexer-common/src/errors.ts +++ b/packages/indexer-common/src/errors.ts @@ -79,6 +79,7 @@ export enum IndexerErrorCode { IE066 = 'IE066', IE067 = 'IE067', IE068 = 'IE068', + IE069 = 'IE069', } export const INDEXER_ERROR_MESSAGES: Record = { @@ -150,7 +151,8 @@ export const INDEXER_ERROR_MESSAGES: Record = { IE065: 'Failed to unallocate: Allocation has already been closed', IE066: 'Failed to allocate: allocation ID already exists on chain', IE067: 'Failed to query POI for current epoch start block', - IE068: 'User-provided POI did not match reference POI from graph-node', + IE068: 'User-provided POI did not match with reference POI from graph-node', + IE069: 'Failed to deploy the graft base for the target deployment', } export type IndexerErrorCause = unknown diff --git a/packages/indexer-common/src/indexer-management/client.ts b/packages/indexer-common/src/indexer-management/client.ts index 8eb63a8c3..1800354a2 100644 --- a/packages/indexer-common/src/indexer-management/client.ts +++ b/packages/indexer-common/src/indexer-management/client.ts @@ -438,6 +438,7 @@ export interface IndexerManagementClientOptions { receiptCollector?: AllocationReceiptCollector allocationManagementMode?: AllocationManagementMode autoAllocationMinBatchSize?: number + autoGraftResolverDepth?: number } export class IndexerManagementClient extends Client { @@ -502,6 +503,7 @@ export const createIndexerManagementClient = async ( receiptCollector, allocationManagementMode, autoAllocationMinBatchSize, + autoGraftResolverDepth, } = options const schema = buildSchema(print(SCHEMA_SDL)) const resolvers = { @@ -515,7 +517,11 @@ export const createIndexerManagementClient = async ( const dai: WritableEventual = mutable() - const subgraphManager = new SubgraphManager(deploymentManagementEndpoint, indexNodeIDs) + const subgraphManager = new SubgraphManager( + deploymentManagementEndpoint, + indexNodeIDs, + autoGraftResolverDepth, + ) let allocationManager: AllocationManager | undefined = undefined let actionManager: ActionManager | undefined = undefined let networkMonitor: NetworkMonitor | undefined = undefined diff --git a/packages/indexer-common/src/indexer-management/subgraphs.ts b/packages/indexer-common/src/indexer-management/subgraphs.ts index 7d90386f8..148288fea 100644 --- a/packages/indexer-common/src/indexer-management/subgraphs.ts +++ b/packages/indexer-common/src/indexer-management/subgraphs.ts @@ -2,6 +2,11 @@ import { indexerError, IndexerErrorCode, IndexerManagementModels, + IndexingDecisionBasis, + IndexingRuleAttributes, + SubgraphIdentifierType, + upsertIndexingRule, + fetchIndexingRules, } from '@graphprotocol/indexer-common' import { Logger, SubgraphDeploymentID } from '@graphprotocol/common-ts' import jayson, { Client as RpcClient } from 'jayson/promise' @@ -10,8 +15,9 @@ import pTimeout from 'p-timeout' export class SubgraphManager { client: RpcClient indexNodeIDs: string[] + autoGraftResolverDepth: number - constructor(endpoint: string, indexNodeIDs: string[]) { + constructor(endpoint: string, indexNodeIDs: string[], autoGraftResolverDepth?: number) { if (endpoint.startsWith('https')) { // eslint-disable-next-line @typescript-eslint/no-explicit-any this.client = jayson.Client.https(endpoint as any) @@ -20,6 +26,35 @@ export class SubgraphManager { this.client = jayson.Client.http(endpoint as any) } this.indexNodeIDs = indexNodeIDs + this.autoGraftResolverDepth = autoGraftResolverDepth ?? 0 + } + + graftBase = ( + logger: Logger, + err: Error, + depth: number, + ): SubgraphDeploymentID | undefined => { + if ( + err.message.includes( + 'subgraph validation error: [the graft base is invalid: deployment not found', + ) + ) { + const graftBaseQm = err.message.substring( + err.message.indexOf('Qm'), + err.message.indexOf('Qm') + 46, + ) + if (depth >= this.autoGraftResolverDepth) { + logger.warn( + `Grafting depth for deployment reached auto-graft-resolver-depth limit`, + { + deployment: graftBaseQm, + depth, + }, + ) + return + } + return new SubgraphDeploymentID(graftBaseQm) + } } async createSubgraph(logger: Logger, name: string): Promise { @@ -44,24 +79,25 @@ export class SubgraphManager { name: string, deployment: SubgraphDeploymentID, indexNode: string | undefined, + depth: number, ): Promise { - try { - let targetNode: string - if (indexNode) { - targetNode = indexNode - if (!this.indexNodeIDs.includes(targetNode)) { - logger.warn( - `Specified deployment target node not present in indexNodeIDs supplied at startup, proceeding with deploy to target node anyway.`, - { - targetNode: indexNode, - indexNodeIDs: this.indexNodeIDs, - }, - ) - } - } else { - targetNode = - this.indexNodeIDs[Math.floor(Math.random() * this.indexNodeIDs.length)] + let targetNode: string + if (indexNode) { + targetNode = indexNode + if (!this.indexNodeIDs.includes(targetNode)) { + logger.warn( + `Specified deployment target node not present in indexNodeIDs supplied at startup, proceeding with deploy to target node anyway.`, + { + targetNode: indexNode, + indexNodeIDs: this.indexNodeIDs, + }, + ) } + } else { + targetNode = this.indexNodeIDs[Math.floor(Math.random() * this.indexNodeIDs.length)] + } + + try { logger.info(`Deploy subgraph`, { name, deployment: deployment.display, @@ -76,7 +112,22 @@ export class SubgraphManager { const response = await pTimeout(requestPromise, 120000) if (response.error) { - throw response.error + const baseDeployment = this.graftBase(logger, response.error, depth) + if (baseDeployment) { + logger.debug( + `Found graft base deployment, ensure deployment and a matching offchain indexing rules`, + ) + // Only add offchain after ensure + await this.ensure(logger, models, name, baseDeployment, targetNode, depth + 1) + + // ensure targetDeployment again after graft base syncs, can use + // "subgraph validation error: [the graft base is invalid: failed to graft onto `Qm...` since it has not processed any blocks (only processed block ___)]" + await this.ensure(logger, models, name, deployment, targetNode, depth) + + throw indexerError(IndexerErrorCode.IE069, response.error) + } else { + throw indexerError(IndexerErrorCode.IE026, response.error) + } } logger.info(`Successfully deployed subgraph`, { name, @@ -84,8 +135,16 @@ export class SubgraphManager { endpoints: response.result, }) - // TODO: Insert an offchain indexing rule if one matching this deployment doesn't yet exist // Will be useful for supporting deploySubgraph resolver + const indexingRules = (await fetchIndexingRules(models, false)).map(rule => new SubgraphDeploymentID(rule.identifier)) + if (!indexingRules.includes(deployment)){ + const offchainIndexingRule = { + identifier: deployment.ipfsHash, + identifierType: SubgraphIdentifierType.DEPLOYMENT, + decisionBasis: IndexingDecisionBasis.OFFCHAIN, + } as Partial + await upsertIndexingRule(logger, models, offchainIndexingRule) + } } catch (error) { const err = indexerError(IndexerErrorCode.IE026, error) logger.error(`Failed to deploy subgraph deployment`, { @@ -196,10 +255,12 @@ export class SubgraphManager { name: string, deployment: SubgraphDeploymentID, indexNode: string | undefined, + depth?: number, ): Promise { + depth = depth ?? 0 try { await this.createSubgraph(logger, name) - await this.deploy(logger, models, name, deployment, indexNode) + await this.deploy(logger, models, name, deployment, indexNode, depth) await this.reassign(logger, deployment, indexNode) } catch (error) { const err = indexerError(IndexerErrorCode.IE020, error)