fix: update validator as UNRECOGNIZED when not found onchain (staging)

ci/config.json.ci

@@ -171,5 +171,10 @@
   },
   "jobRedecodeTx": {
     "limitRecordGet": 100
+  },
+  "crawlIbcTao": {
+    "key": "crawlIbcTao",
+    "millisecondRepeatJob": 2000,
+    "blocksPerCall": 100
   }
 }

config.json

@@ -171,5 +171,10 @@
   },
   "jobRedecodeTx": {
     "limitRecordGet": 100
+  },
+  "crawlIbcTao": {
+    "key": "crawlIbcTao",
+    "millisecondRepeatJob": 2000,
+    "blocksPerCall": 100
   }
 }

migrations/20230802071931_ibc_tao_model.ts

@@ -0,0 +1,36 @@
+import { Knex } from 'knex';
+
+export async function up(knex: Knex): Promise<void> {
+  await knex.schema.createTable('ibc_client', (table) => {
+    table.increments();
+    table.string('client_id').notNullable().unique();
+    table.string('counterparty_chain_id').notNullable();
+    table.jsonb('client_state').notNullable();
+    table.jsonb('consensus_state').notNullable();
+    table.string('client_type').notNullable();
+  });
+  await knex.schema.createTable('ibc_connection', (table) => {
+    table.increments();
+    table.integer('ibc_client_id').index();
+    table.string('connection_id').notNullable().unique();
+    table.string('counterparty_client_id').notNullable();
+    table.string('counterparty_connection_id').notNullable();
+    table.foreign('ibc_client_id').references('ibc_client.id');
+  });
+  await knex.schema.createTable('ibc_channel', (table) => {
+    table.increments();
+    table.integer('ibc_connection_id').index();
+    table.string('channel_id').notNullable().unique();
+    table.string('port_id').notNullable().index();
+    table.string('counterparty_port_id').notNullable();
+    table.string('counterparty_channel_id').notNullable();
+    table.string('state').notNullable();
+    table.foreign('ibc_connection_id').references('ibc_connection.id');
+  });
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.schema.dropTableIfExists('ibc_client');
+  await knex.schema.dropTableIfExists('ibc_connection');
+  await knex.schema.dropTableIfExists('ibc_channel');
+}

src/common/constant.ts

@@ -71,6 +71,7 @@ export const BULL_JOB_NAME = {
   REINDEX_CW721_HISTORY: 'reindex:cw721-history',
   HANDLE_MIGRATE_CONTRACT: 'handle:migrate-contract',
   JOB_REDECODE_TX: 'job:redecode-tx',
+  CRAWL_IBC_TAO: 'crawl:ibc-tao',
 };
 
 export const SERVICE = {
@@ -248,6 +249,10 @@ export const SERVICE = {
       key: 'DailyStatsJobsService',
       name: 'v1.DailyStatsJobsService',
     },
+    CrawlIBCTaoService: {
+      key: 'CrawlIBCTaoService',
+      name: 'v1.CrawlIBCTaoService',
+    },
   },
 };
 

src/models/event.ts

@@ -6,7 +6,7 @@ import { EventAttribute } from './event_attribute';
 import { TransactionMessage } from './transaction_message';
 
 export class Event extends BaseModel {
-  [relation: string]: any;
+  [relation: string]: any | any[];
 
   id!: string;
 
@@ -94,5 +94,13 @@ export class Event extends BaseModel {
     TX: 'tx',
     TRANSFER: 'transfer',
     MIGRATE: 'migrate',
+    CREATE_CLIENT: 'create_client',
+    CONNECTION_OPEN_ACK: 'connection_open_ack',
+    CONNECTION_OPEN_CONFIRM: 'connection_open_confirm',
+    CHANNEL_OPEN_ACK: 'channel_open_ack',
+    CHANNEL_OPEN_CONFIRM: 'channel_open_confirm',
+    CHANNEL_CLOSE_INIT: 'channel_close_init',
+    CHANNEL_CLOSE_CONFIRM: 'channel_close_confirm',
+    CHANNEL_CLOSE: 'channel_close',
   };
 }

src/models/event_attribute.ts

@@ -98,6 +98,15 @@ export class EventAttribute extends BaseModel {
     FROM: 'from',
     FEE: 'fee',
     FEE_PAYER: 'fee_payer',
+    CLIENT_ID: 'client_id',
+    CLIENT_TYPE: 'client_type',
+    CONNECTION_ID: 'connection_id',
+    COUNTERPARTY_CLIENT_ID: 'counterparty_client_id',
+    COUNTERPARTY_CONNECTION_ID: 'counterparty_connection_id',
+    CHANNEL_ID: 'channel_id',
+    PORT_ID: 'port_id',
+    COUNTERPARTY_PORT_ID: 'counterparty_port_id',
+    COUNTERPARTY_CHANNEL_ID: 'counterparty_channel_id',
   };
 
   static ATTRIBUTE_COMPOSITE_KEY = {

src/models/ibc_channel.ts

@@ -0,0 +1,64 @@
+/* eslint-disable import/no-cycle */
+import { Model } from 'objection';
+import BaseModel from './base';
+import { IbcConnection } from './ibc_connection';
+
+export class IbcChannel extends BaseModel {
+  id!: number;
+
+  ibc_connection_id!: number;
+
+  channel_id!: string;
+
+  port_id!: string;
+
+  counterparty_port_id!: string;
+
+  counterparty_channel_id!: string;
+
+  state!: string;
+
+  static get tableName() {
+    return 'ibc_channel';
+  }
+
+  static get jsonSchema() {
+    return {
+      type: 'object',
+      required: [
+        'ibc_connection_id',
+        'channel_id',
+        'port_id',
+        'counterparty_port_id',
+        'counterparty_channel_id',
+        'state',
+      ],
+      properties: {
+        ibc_connection_id: { type: 'number' },
+        channel_id: { type: 'string' },
+        port_id: { type: 'string' },
+        counterparty_port_id: { type: 'string' },
+        counterparty_channel_id: { type: 'string' },
+        state: { type: 'string' },
+      },
+    };
+  }
+
+  static get relationMappings() {
+    return {
+      ibc_connection: {
+        relation: Model.BelongsToOneRelation,
+        modelClass: IbcConnection,
+        join: {
+          from: 'ibc_channel.ibc_connection_id',
+          to: 'ibc_connection.id',
+        },
+      },
+    };
+  }
+
+  static STATUS = {
+    OPEN: 'OPEN',
+    CLOSE: 'CLOSE',
+  };
+}

src/models/ibc_client.ts

@@ -0,0 +1,40 @@
+/* eslint-disable import/no-cycle */
+import BaseModel from './base';
+
+export class IbcClient extends BaseModel {
+  id!: number;
+
+  client_id!: string;
+
+  counterparty_chain_id!: string;
+
+  client_state!: any;
+
+  consensus_state!: any;
+
+  client_type!: string;
+
+  static get tableName() {
+    return 'ibc_client';
+  }
+
+  static get jsonSchema() {
+    return {
+      type: 'object',
+      required: [
+        'client_id',
+        'counterparty_chain_id',
+        'client_state',
+        'consensus_state',
+        'client_type',
+      ],
+      properties: {
+        client_id: { type: 'string' },
+        counterparty_chain_id: { type: 'string' },
+        client_state: { type: 'object' },
+        consensus_state: { type: 'object' },
+        client_type: { type: 'string' },
+      },
+    };
+  }
+}

src/models/ibc_connection.ts

@@ -0,0 +1,51 @@
+/* eslint-disable import/no-cycle */
+import { Model } from 'objection';
+import BaseModel from './base';
+import { IbcClient } from './ibc_client';
+
+export class IbcConnection extends BaseModel {
+  id!: number;
+
+  ibc_client_id!: number;
+
+  connection_id!: string;
+
+  counterparty_client_id!: string;
+
+  counterparty_connection_id!: string;
+
+  static get tableName() {
+    return 'ibc_connection';
+  }
+
+  static get jsonSchema() {
+    return {
+      type: 'object',
+      required: [
+        'ibc_client_id',
+        'connection_id',
+        'counterparty_client_id',
+        'counterparty_connection_id',
+      ],
+      properties: {
+        ibc_client_id: { type: 'number' },
+        connection_id: { type: 'string' },
+        counterparty_client_id: { type: 'string' },
+        counterparty_connection_id: { type: 'string' },
+      },
+    };
+  }
+
+  static get relationMappings() {
+    return {
+      ibc_client: {
+        relation: Model.BelongsToOneRelation,
+        modelClass: IbcClient,
+        join: {
+          from: 'ibc_connection.ibc_client_id',
+          to: 'ibc_client.id',
+        },
+      },
+    };
+  }
+}

src/models/index.ts

@@ -25,3 +25,6 @@ export * from './feegrant_history';
 export * from './daily_statistics';
 export * from './account_statistics';
 export * from './cw20_total_holder_stats';
+export * from './ibc_client';
+export * from './ibc_connection';
+export * from './ibc_channel';

src/models/validator.ts

@@ -73,6 +73,9 @@ export class Validator extends BaseModel {
     return {
       BONDED: 'BOND_STATUS_BONDED',
       UNBONDED: 'BOND_STATUS_UNBONDED',
+      UNSPECIFIED: 'BOND_STATUS_UNSPECIFIED',
+      UNBONDING: 'BOND_STATUS_UNBONDING',
+      UNRECOGNIZED: 'UNRECOGNIZED',
     };
   }
 

src/services/crawl-validator/crawl_validator.service.ts

@@ -70,7 +70,6 @@ export default class CrawlValidatorService extends BullableService {
       .select('value')
       .limit(1)
       .offset(0);
-
     await knex.transaction(async (trx) => {
       if (resultTx.length > 0) {
         await this.updateValidators(trx);
@@ -110,8 +109,10 @@ export default class CrawlValidatorService extends BullableService {
       }
     }
 
-    const validatorInDB: Validator[] = await knex('validator').select('*');
-
+    const validatorInDB: Validator[] = await knex('validator')
+      .select('*')
+      .whereNot('status', Validator.STATUS.UNRECOGNIZED);
+    const offchainMapped: Map<string, boolean> = new Map();
     await Promise.all(
       validators.map(async (validator) => {
         const foundValidator = validatorInDB.find(
@@ -123,6 +124,9 @@ export default class CrawlValidatorService extends BullableService {
         if (!foundValidator) {
           validatorEntity = Validator.createNewValidator(validator);
         } else {
+          // mark this offchain validator is mapped with onchain
+          offchainMapped.set(validator.operator_address, true);
+
           validatorEntity = foundValidator;
           validatorEntity.jailed = validator.jailed;
           validatorEntity.status = validator.status;
@@ -146,6 +150,26 @@ export default class CrawlValidatorService extends BullableService {
 
     updateValidators = await this.loadCustomInfo(updateValidators);
 
+    // loop all validator not found onchain, update status is UNRECOGNIZED
+    validatorInDB
+      .filter((val: any) => !offchainMapped.get(val.operator_address))
+      .forEach(async (validatorNotOnchain: any) => {
+        this.logger.debug(
+          'Account not found onchain: ',
+          validatorNotOnchain.operator_address
+        );
+        validatorNotOnchain.status = Validator.STATUS.UNRECOGNIZED;
+        validatorNotOnchain.tokens = 0;
+        validatorNotOnchain.delegator_shares = 0;
+        validatorNotOnchain.percent_voting_power = 0;
+
+        validatorNotOnchain.jailed_until =
+          validatorNotOnchain.jailed_until.toISOString();
+        validatorNotOnchain.unbonding_time =
+          validatorNotOnchain.unbonding_time.toISOString();
+        updateValidators.push(validatorNotOnchain);
+      });
+
     await Validator.query()
       .insert(updateValidators)
       .onConflict('operator_address')

src/services/cw20/cw20.service.ts

@@ -240,12 +240,16 @@ export default class Cw20Service extends BullableService {
         'smart_contract.id as smart_contract_id',
         'tx.height as height',
         'smart_contract_event.id as smart_contract_event_id'
-      );
+      )
+      .orderBy('smart_contract_event.id', 'asc');
   }
 
   async handleStatistic(startBlock: number) {
     const systemDate = (
-      await Block.query().where('height', startBlock).first().throwIfNotFound()
+      await Block.query()
+        .where('height', startBlock + 1)
+        .first()
+        .throwIfNotFound()
     ).time;
     const lastUpdatedDate = (
       await CW20TotalHolderStats.query().max('date').first()
@@ -263,13 +267,15 @@ export default class Cw20Service extends BullableService {
 
   async handleTotalHolderStatistic(systemDate: Date) {
     const totalHolder = await Cw20Contract.query()
-      .alias('cw20_contract')
       .joinRelated('holders')
       .where('cw20_contract.track', true)
-      .andWhere('holders.amount', '>', 0)
-      .count()
-      .groupBy('holders.cw20_contract_id')
-      .select('holders.cw20_contract_id');
+      .groupBy('cw20_contract.id')
+      .select(
+        'cw20_contract.id as cw20_contract_id',
+        knex.raw(
+          'count(CASE when holders.amount > 0 THEN 1 ELSE null END) as count'
+        )
+      );
     if (totalHolder.length > 0) {
       await CW20TotalHolderStats.query().insert(
         totalHolder.map((e) =>