diff --git a/CONFIGURATION.md b/CONFIGURATION.md index 01c09c39f..644d06910 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -56,9 +56,11 @@ ORACLE_JSONRPC_ERROR_CODES | Override default JSON rpc error codes that can trig ORACLE_HOME_EVENTS_REPROCESSING | If set to `true`, home events happened in the past will be refetched and processed once again, to ensure that nothing was missed on the first pass. | `bool` ORACLE_HOME_EVENTS_REPROCESSING_BATCH_SIZE | Batch size for one `eth_getLogs` request when reprocessing old logs in the home chain. Defaults to `1000` | `integer` ORACLE_HOME_EVENTS_REPROCESSING_BLOCK_DELAY | Block confirmations number, after which old logs are being reprocessed in the home chain. Defaults to `500` | `integer` +ORACLE_HOME_RPC_SYNC_STATE_CHECK_INTERVAL | Interval for checking JSON RPC sync state, by requesting the latest block number. Oracle will switch to the fallback JSON RPC in case sync process is stuck | `integer` ORACLE_FOREIGN_EVENTS_REPROCESSING | If set to `true`, foreign events happened in the past will be refetched and processed once again, to ensure that nothing was missed on the first pass. | `bool` ORACLE_FOREIGN_EVENTS_REPROCESSING_BATCH_SIZE | Batch size for one `eth_getLogs` request when reprocessing old logs in the foreign chain. Defaults to `500` | `integer` ORACLE_FOREIGN_EVENTS_REPROCESSING_BLOCK_DELAY | Block confirmations number, after which old logs are being reprocessed in the foreign chain. Defaults to `250` | `integer` +ORACLE_FOREIGN_RPC_SYNC_STATE_CHECK_INTERVAL | Interval for checking JSON RPC sync state, by requesting the latest block number. Oracle will switch to the fallback JSON RPC in case sync process is stuck | `integer` ## Monitor configuration diff --git a/oracle/config/base.config.js b/oracle/config/base.config.js index 35e527064..bbb108cf0 100644 --- a/oracle/config/base.config.js +++ b/oracle/config/base.config.js @@ -7,7 +7,15 @@ const { HOME_AMB_ABI, FOREIGN_AMB_ABI } = require('../../commons') -const { web3Home, web3Foreign } = require('../src/services/web3') +const { + web3Home, + web3Foreign, + web3HomeRedundant, + web3HomeFallback, + web3ForeignRedundant, + web3ForeignFallback, + web3ForeignArchive +} = require('../src/services/web3') const { add0xPrefix, privateKeyToAddress } = require('../src/utils/utils') const { EXIT_CODES } = require('../src/utils/constants') @@ -27,9 +35,11 @@ const { ORACLE_HOME_EVENTS_REPROCESSING, ORACLE_HOME_EVENTS_REPROCESSING_BATCH_SIZE, ORACLE_HOME_EVENTS_REPROCESSING_BLOCK_DELAY, + ORACLE_HOME_RPC_SYNC_STATE_CHECK_INTERVAL, ORACLE_FOREIGN_EVENTS_REPROCESSING, ORACLE_FOREIGN_EVENTS_REPROCESSING_BATCH_SIZE, - ORACLE_FOREIGN_EVENTS_REPROCESSING_BLOCK_DELAY + ORACLE_FOREIGN_EVENTS_REPROCESSING_BLOCK_DELAY, + ORACLE_FOREIGN_RPC_SYNC_STATE_CHECK_INTERVAL } = process.env let homeAbi @@ -63,9 +73,12 @@ const homeConfig = { bridgeAddress: COMMON_HOME_BRIDGE_ADDRESS, bridgeABI: homeAbi, pollingInterval: parseInt(ORACLE_HOME_RPC_POLLING_INTERVAL, 10), + syncCheckInterval: parseInt(ORACLE_HOME_RPC_SYNC_STATE_CHECK_INTERVAL, 10) || 60000, startBlock: parseInt(ORACLE_HOME_START_BLOCK, 10) || 0, blockPollingLimit: parseInt(ORACLE_HOME_RPC_BLOCK_POLLING_LIMIT, 10), web3: web3Home, + web3Redundant: web3HomeRedundant, + web3Fallback: web3HomeFallback, bridgeContract: homeContract, eventContract: homeContract, reprocessingOptions: { @@ -81,9 +94,13 @@ const foreignConfig = { bridgeAddress: COMMON_FOREIGN_BRIDGE_ADDRESS, bridgeABI: foreignAbi, pollingInterval: parseInt(ORACLE_FOREIGN_RPC_POLLING_INTERVAL, 10), + syncCheckInterval: parseInt(ORACLE_FOREIGN_RPC_SYNC_STATE_CHECK_INTERVAL, 10) || 60000, startBlock: parseInt(ORACLE_FOREIGN_START_BLOCK, 10) || 0, blockPollingLimit: parseInt(ORACLE_FOREIGN_RPC_BLOCK_POLLING_LIMIT, 10), web3: web3Foreign, + web3Redundant: web3ForeignRedundant, + web3Fallback: web3ForeignFallback, + web3Archive: web3ForeignArchive || web3Foreign, bridgeContract: foreignContract, eventContract: foreignContract, reprocessingOptions: { diff --git a/oracle/config/foreign-sender.config.js b/oracle/config/foreign-sender.config.js index 7971c7459..177c1b39d 100644 --- a/oracle/config/foreign-sender.config.js +++ b/oracle/config/foreign-sender.config.js @@ -1,17 +1,14 @@ const baseConfig = require('./base.config') const { DEFAULT_TRANSACTION_RESEND_INTERVAL } = require('../src/utils/constants') -const { web3Foreign, web3ForeignRedundant, web3ForeignFallback } = require('../src/services/web3') const { ORACLE_FOREIGN_TX_RESEND_INTERVAL } = process.env module.exports = { ...baseConfig, + main: baseConfig.foreign, queue: 'foreign-prioritized', id: 'foreign', name: 'sender-foreign', - web3: web3Foreign, - web3Redundant: web3ForeignRedundant, - web3Fallback: web3ForeignFallback, resendInterval: parseInt(ORACLE_FOREIGN_TX_RESEND_INTERVAL, 10) || DEFAULT_TRANSACTION_RESEND_INTERVAL } diff --git a/oracle/config/home-sender.config.js b/oracle/config/home-sender.config.js index 70ac51a85..e177d9d00 100644 --- a/oracle/config/home-sender.config.js +++ b/oracle/config/home-sender.config.js @@ -1,17 +1,14 @@ const baseConfig = require('./base.config') const { DEFAULT_TRANSACTION_RESEND_INTERVAL } = require('../src/utils/constants') -const { web3Home, web3HomeRedundant, web3HomeFallback } = require('../src/services/web3') const { ORACLE_HOME_TX_RESEND_INTERVAL } = process.env module.exports = { ...baseConfig, + main: baseConfig.home, queue: 'home-prioritized', id: 'home', name: 'sender-home', - web3: web3Home, - web3Redundant: web3HomeRedundant, - web3Fallback: web3HomeFallback, resendInterval: parseInt(ORACLE_HOME_TX_RESEND_INTERVAL, 10) || DEFAULT_TRANSACTION_RESEND_INTERVAL } diff --git a/oracle/config/information-request-watcher.config.js b/oracle/config/information-request-watcher.config.js index 6aa5a7533..449ffecb4 100644 --- a/oracle/config/information-request-watcher.config.js +++ b/oracle/config/information-request-watcher.config.js @@ -1,11 +1,9 @@ const baseConfig = require('./base.config') -const { web3ForeignArchive } = require('../src/services/web3') const id = `${baseConfig.id}-information-request` module.exports = { ...baseConfig, - web3ForeignArchive: web3ForeignArchive || baseConfig.foreign.web3, main: baseConfig.home, event: 'UserRequestForInformation', sender: 'home', diff --git a/oracle/src/events/processAMBInformationRequests/index.js b/oracle/src/events/processAMBInformationRequests/index.js index b3711b060..eeef38d5c 100644 --- a/oracle/src/events/processAMBInformationRequests/index.js +++ b/oracle/src/events/processAMBInformationRequests/index.js @@ -35,11 +35,13 @@ Object.keys(asyncCalls).forEach(method => { }) function processInformationRequestsBuilder(config) { - const { home, foreign, web3ForeignArchive } = config + const { home, foreign } = config let validatorContract = null let blockFinder = null + foreign.web3Archive.currentProvider.startSyncStateChecker(foreign.syncCheckInterval) + return async function processInformationRequests(informationRequests) { const txToSend = [] @@ -49,13 +51,15 @@ function processInformationRequestsBuilder(config) { if (blockFinder === null) { rootLogger.debug('Initializing block finder') - blockFinder = await makeBlockFinder('foreign', foreign.web3) + blockFinder = await makeBlockFinder('foreign', foreign.web3Archive) } + // latest foreign block is requested from an archive RPC, to ensure that it is synced with the network + // block confirmations can be requested from the regular JSON RPC const foreignBlockNumber = - (await getBlockNumber(foreign.web3)) - (await getRequiredBlockConfirmations(foreign.bridgeContract)) + (await getBlockNumber(foreign.web3Archive)) - (await getRequiredBlockConfirmations(foreign.bridgeContract)) const homeBlock = await getBlock(home.web3, informationRequests[0].blockNumber) - const lastForeignBlock = await getBlock(foreign.web3, foreignBlockNumber) + const lastForeignBlock = await getBlock(foreign.web3Archive, foreignBlockNumber) if (homeBlock.timestamp > lastForeignBlock.timestamp) { rootLogger.debug( @@ -85,7 +89,7 @@ function processInformationRequestsBuilder(config) { logger.info({ requestSelector, method: asyncCallMethod, data }, 'Processing async request') const call = asyncCalls[asyncCallMethod] - let [status, result] = await call(web3ForeignArchive, data, foreignClosestBlock).catch(e => { + let [status, result] = await call(foreign.web3Archive, data, foreignClosestBlock).catch(e => { if (e instanceof HttpListProviderError) { throw e } diff --git a/oracle/src/sender.js b/oracle/src/sender.js index 46aa607bc..489c8fdf4 100644 --- a/oracle/src/sender.js +++ b/oracle/src/sender.js @@ -32,8 +32,8 @@ if (process.argv.length < 3) { const config = require(path.join('../config/', process.argv[2])) -const { web3, web3Fallback } = config -const web3Redundant = ORACLE_TX_REDUNDANCY === 'true' ? config.web3Redundant : web3 +const { web3, web3Fallback, syncCheckInterval } = config.main +const web3Redundant = ORACLE_TX_REDUNDANCY === 'true' ? config.main.web3Redundant : web3 const nonceKey = `${config.id}:nonce` let chainId = 0 @@ -43,6 +43,7 @@ async function initialize() { const checkHttps = checkHTTPS(process.env.ORACLE_ALLOW_HTTP_FOR_RPC, logger) web3.currentProvider.urls.forEach(checkHttps(config.id)) + web3.currentProvider.startSyncStateChecker(syncCheckInterval) GasPrice.start(config.id, web3) diff --git a/oracle/src/services/HttpListProvider.js b/oracle/src/services/HttpListProvider.js index efd5d9728..66bda92fe 100644 --- a/oracle/src/services/HttpListProvider.js +++ b/oracle/src/services/HttpListProvider.js @@ -1,5 +1,6 @@ const fetch = require('node-fetch') const promiseRetry = require('promise-retry') +const { utils } = require('web3') const { FALLBACK_RPC_URL_SWITCH_TIMEOUT } = require('../utils/constants') const { onInjected } = require('./injectedLogger') @@ -39,19 +40,54 @@ function HttpListProvider(urls, options = {}) { this.options = { ...defaultOptions, ...options } this.currentIndex = 0 this.lastTimeUsedPrimary = 0 + this.latestBlock = 0 + this.syncStateCheckerIntervalId = 0 onInjected(logger => { this.logger = logger.child({ module: `HttpListProvider:${this.options.name}` }) }) } -HttpListProvider.prototype.switchToFallbackRPC = function() { - if (this.urls.length < 2) { - return +HttpListProvider.prototype.startSyncStateChecker = function(syncCheckInterval) { + if (this.urls.length > 1 && syncCheckInterval > 0 && this.syncStateCheckerIntervalId === 0) { + this.syncStateCheckerIntervalId = setInterval(this.checkLatestBlock.bind(this), syncCheckInterval) } +} +HttpListProvider.prototype.checkLatestBlock = function() { + const payload = { jsonrpc: '2.0', id: 1, method: 'eth_blockNumber', params: [] } + this.send(payload, (error, result) => { + if (error) { + this.logger.warn({ oldBlock: this.latestBlock }, 'Failed to request latest block from all RPC urls') + } else if (result.error) { + this.logger.warn( + { oldBlock: this.latestBlock, error: result.error.message }, + 'Failed to make eth_blockNumber request due to unknown error, switching to fallback RPC' + ) + this.switchToFallbackRPC() + } else { + const blockNumber = utils.hexToNumber(result.result) + if (blockNumber > this.latestBlock) { + this.logger.debug({ oldBlock: this.latestBlock, newBlock: blockNumber }, 'Updating latest block number') + this.latestBlock = blockNumber + } else { + this.logger.warn( + { oldBlock: this.latestBlock, newBlock: blockNumber }, + 'Latest block on the node was not updated since last request, switching to fallback RPC' + ) + this.switchToFallbackRPC() + } + } + }) +} + +HttpListProvider.prototype.switchToFallbackRPC = function(index) { const prevIndex = this.currentIndex - const newIndex = (prevIndex + 1) % this.urls.length + const newIndex = index || (prevIndex + 1) % this.urls.length + if (this.urls.length < 2 || prevIndex === newIndex) { + return + } + this.logger.info( { index: newIndex, oldURL: this.urls[prevIndex], newURL: this.urls[newIndex] }, 'Switching to fallback JSON-RPC URL' @@ -80,11 +116,7 @@ HttpListProvider.prototype.send = async function send(payload, callback) { // if some of URLs failed to respond, current URL index is updated to the first URL that responded if (currentIndex !== index) { - this.logger.info( - { index, oldURL: this.urls[currentIndex], newURL: this.urls[index] }, - 'Switching to fallback JSON-RPC URL' - ) - this.currentIndex = index + this.switchToFallbackRPC(index) } callback(null, result) } catch (e) { diff --git a/oracle/src/utils/constants.js b/oracle/src/utils/constants.js index 789e71fc4..48ff67160 100644 --- a/oracle/src/utils/constants.js +++ b/oracle/src/utils/constants.js @@ -27,7 +27,6 @@ module.exports = { MIN_GAS_PRICE_BUMP_FACTOR: 0.1, DEFAULT_TRANSACTION_RESEND_INTERVAL: 20 * 60 * 1000, FALLBACK_RPC_URL_SWITCH_TIMEOUT: 60 * 60 * 1000, - BLOCK_NUMBER_PROGRESS_ITERATIONS_LIMIT: 10, SENDER_QUEUE_MAX_PRIORITY: 10, SENDER_QUEUE_SEND_PRIORITY: 5, SENDER_QUEUE_CHECK_STATUS_PRIORITY: 1, diff --git a/oracle/src/watcher.js b/oracle/src/watcher.js index e7ef51ac6..76e8bc4f0 100644 --- a/oracle/src/watcher.js +++ b/oracle/src/watcher.js @@ -6,11 +6,7 @@ const logger = require('./services/logger') const { getShutdownFlag } = require('./services/shutdownState') const { getBlockNumber, getRequiredBlockConfirmations, getEvents } = require('./tx/web3') const { checkHTTPS, watchdog } = require('./utils/utils') -const { - EXIT_CODES, - BLOCK_NUMBER_PROGRESS_ITERATIONS_LIMIT, - MAX_HISTORY_BLOCK_TO_REPROCESS -} = require('./utils/constants') +const { EXIT_CODES, MAX_HISTORY_BLOCK_TO_REPROCESS } = require('./utils/constants') if (process.argv.length < 3) { logger.error('Please check the number of arguments, config file was not provided') @@ -38,21 +34,21 @@ const { pollingInterval, chain, reprocessingOptions, - blockPollingLimit + blockPollingLimit, + syncCheckInterval } = config.main const lastBlockRedisKey = `${config.id}:lastProcessedBlock` const lastReprocessedBlockRedisKey = `${config.id}:lastReprocessedBlock` const seenEventsRedisKey = `${config.id}:seenEvents` let lastProcessedBlock = Math.max(startBlock - 1, 0) let lastReprocessedBlock -let lastSeenBlockNumber = 0 -let sameBlockNumberCounter = 0 async function initialize() { try { const checkHttps = checkHTTPS(process.env.ORACLE_ALLOW_HTTP_FOR_RPC, logger) web3.currentProvider.urls.forEach(checkHttps(chain)) + web3.currentProvider.startSyncStateChecker(syncCheckInterval) await getLastProcessedBlock() await getLastReprocessedBlock() @@ -225,28 +221,6 @@ async function getLastBlockToProcess(web3, bridgeContract) { getBlockNumber(web3), getRequiredBlockConfirmations(bridgeContract) ]) - - if (lastBlockNumber < lastSeenBlockNumber) { - sameBlockNumberCounter = 0 - logger.warn({ lastBlockNumber, lastSeenBlockNumber }, 'Received block number less than already seen block') - web3.currentProvider.switchToFallbackRPC() - } else if (lastBlockNumber === lastSeenBlockNumber) { - sameBlockNumberCounter++ - if (sameBlockNumberCounter > 1) { - logger.info({ lastBlockNumber, sameBlockNumberCounter }, 'Received the same block number more than twice') - if (sameBlockNumberCounter >= BLOCK_NUMBER_PROGRESS_ITERATIONS_LIMIT) { - sameBlockNumberCounter = 0 - logger.warn( - { lastBlockNumber, n: BLOCK_NUMBER_PROGRESS_ITERATIONS_LIMIT }, - 'Received the same block number for too many times. Probably node is not synced anymore' - ) - web3.currentProvider.switchToFallbackRPC() - } - } - } else { - sameBlockNumberCounter = 0 - lastSeenBlockNumber = lastBlockNumber - } return lastBlockNumber - requiredBlockConfirmations }