Merge branch 'develop' of github.com:aura-nw/horoscope-v2 into feat/i…

…bc_tao_genesis_dev

ci/config.json.ci

@@ -108,7 +108,10 @@
   "cw20": {
     "blocksPerCall": 100,
     "millisecondRepeatJob": 2000,
-    "key": "cw20"
+    "key": "cw20",
+    "reindexHistory": {
+      "limitRecordGet": 500
+    }
   },
   "dashboardStatistics": {
     "millisecondCrawl": 10000,

config.json

@@ -103,7 +103,10 @@
   "cw20": {
     "blocksPerCall": 100,
     "millisecondRepeatJob": 2000,
-    "key": "cw20"
+    "key": "cw20",
+    "reindexHistory": {
+      "limitRecordGet": 500
+    }
   },
   "crawlContractEvent": {
     "key": "crawlContractEvent",

src/common/constant.ts

@@ -73,6 +73,8 @@ export const BULL_JOB_NAME = {
   JOB_REDECODE_TX: 'job:redecode-tx',
   CRAWL_IBC_TAO: 'crawl:ibc-tao',
   CRAWL_GENESIS_IBC_TAO: 'crawl:genesis-ibc-tao',
+  REINDEX_CW20_CONTRACT: 'reindex:cw20-contract',
+  REINDEX_CW20_HISTORY: 'reindex:cw20-history',
 };
 
 export const SERVICE = {
@@ -254,6 +256,14 @@ export const SERVICE = {
       key: 'CrawlIBCTaoService',
       name: 'v1.CrawlIBCTaoService',
     },
+    Cw20ReindexingService: {
+      key: 'Cw20ReindexingService',
+      name: 'v1.Cw20ReindexingService',
+      Reindexing: {
+        key: 'reindexing',
+        path: 'v1.Cw20ReindexingService.reindexing',
+      },
+    },
   },
 };
 

src/models/cw20_activity.ts

@@ -5,6 +5,8 @@ import { Cw20Contract } from './cw20_contract';
 import { SmartContract } from './smart_contract';
 
 export class Cw20Event extends BaseModel {
+  static softDelete = false;
+
   [relation: string]: any;
 
   id!: number;

src/models/cw20_contract.ts

@@ -26,6 +26,8 @@ export interface IContractInfo {
   name?: string;
 }
 export class Cw20Contract extends BaseModel {
+  static softDelete = false;
+
   [relation: string]: any;
 
   id!: number;

src/models/cw20_holder.ts

@@ -5,6 +5,8 @@ import { Cw20Contract } from './cw20_contract';
 import { SmartContract } from './smart_contract';
 
 export class CW20Holder extends BaseModel {
+  static softDelete = false;
+
   [relation: string]: any;
 
   id?: number;

src/models/cw20_total_holder_stats.ts

@@ -3,6 +3,8 @@ import BaseModel from './base';
 import { Cw20Contract } from './cw20_contract';
 
 export class CW20TotalHolderStats extends BaseModel {
+  static softDelete = false;
+
   [relation: string]: any;
 
   date!: Date;

src/models/validator.ts

@@ -73,6 +73,9 @@ export class Validator extends BaseModel {
     return {
       BONDED: 'BOND_STATUS_BONDED',
       UNBONDED: 'BOND_STATUS_UNBONDED',
+      UNSPECIFIED: 'BOND_STATUS_UNSPECIFIED',
+      UNBONDING: 'BOND_STATUS_UNBONDING',
+      UNRECOGNIZED: 'UNRECOGNIZED',
     };
   }
 

src/services/crawl-validator/crawl_validator.service.ts

@@ -70,7 +70,6 @@ export default class CrawlValidatorService extends BullableService {
       .select('value')
       .limit(1)
       .offset(0);
-
     await knex.transaction(async (trx) => {
       if (resultTx.length > 0) {
         await this.updateValidators(trx);
@@ -110,8 +109,10 @@ export default class CrawlValidatorService extends BullableService {
       }
     }
 
-    const validatorInDB: Validator[] = await knex('validator').select('*');
-
+    const validatorInDB: Validator[] = await knex('validator')
+      .select('*')
+      .whereNot('status', Validator.STATUS.UNRECOGNIZED);
+    const offchainMapped: Map<string, boolean> = new Map();
     await Promise.all(
       validators.map(async (validator) => {
         const foundValidator = validatorInDB.find(
@@ -123,6 +124,9 @@ export default class CrawlValidatorService extends BullableService {
         if (!foundValidator) {
           validatorEntity = Validator.createNewValidator(validator);
         } else {
+          // mark this offchain validator is mapped with onchain
+          offchainMapped.set(validator.operator_address, true);
+
           validatorEntity = foundValidator;
           validatorEntity.jailed = validator.jailed;
           validatorEntity.status = validator.status;
@@ -146,6 +150,26 @@ export default class CrawlValidatorService extends BullableService {
 
     updateValidators = await this.loadCustomInfo(updateValidators);
 
+    // loop all validator not found onchain, update status is UNRECOGNIZED
+    validatorInDB
+      .filter((val: any) => !offchainMapped.get(val.operator_address))
+      .forEach(async (validatorNotOnchain: any) => {
+        this.logger.debug(
+          'Account not found onchain: ',
+          validatorNotOnchain.operator_address
+        );
+        validatorNotOnchain.status = Validator.STATUS.UNRECOGNIZED;
+        validatorNotOnchain.tokens = 0;
+        validatorNotOnchain.delegator_shares = 0;
+        validatorNotOnchain.percent_voting_power = 0;
+
+        validatorNotOnchain.jailed_until =
+          validatorNotOnchain.jailed_until.toISOString();
+        validatorNotOnchain.unbonding_time =
+          validatorNotOnchain.unbonding_time.toISOString();
+        updateValidators.push(validatorNotOnchain);
+      });
+
     await Validator.query()
       .insert(updateValidators)
       .onConflict('operator_address')

src/services/cw20/cw20.service.ts

@@ -1,4 +1,5 @@
 import { Service } from '@ourparentcenter/moleculer-decorators-extended';
+import { Queue } from 'bullmq';
 import { Knex } from 'knex';
 import _ from 'lodash';
 import { ServiceBroker } from 'moleculer';
@@ -34,6 +35,13 @@ export const CW20_ACTION = {
   BURN_FROM: 'burn_from',
   SEND_FROM: 'send_from',
 };
+export interface ICw20ReindexingHistoryParams {
+  smartContractId: number;
+  startBlock: number;
+  endBlock: number;
+  prevId: number;
+  contractAddress: string;
+}
 @Service({
   name: SERVICE.V1.Cw20.key,
   version: 1,
@@ -205,41 +213,38 @@ export default class Cw20Service extends BullableService {
     }
   }
 
-  async getCw20ContractEvents(startBlock: number, endBlock: number) {
+  async getCw20ContractEvents(
+    startBlock: number,
+    endBlock: number,
+    smartContractId?: number,
+    page?: { prevId: number; limit: number }
+  ) {
     return SmartContractEvent.query()
-      .alias('smart_contract_event')
-      .withGraphJoined(
-        '[message(selectMessage), tx(selectTransaction), attributes(selectAttribute), smart_contract(selectSmartContract).code(selectCode)]'
-      )
-      .modifiers({
-        selectCode(builder) {
-          builder.select('type');
-        },
-        selectTransaction(builder) {
-          builder.select('hash', 'height');
-        },
-        selectMessage(builder) {
-          builder.select('sender', 'content');
-        },
-        selectAttribute(builder) {
-          builder.select('key', 'value');
-        },
-        selectSmartContract(builder) {
-          builder.select('address', 'id');
-        },
-      })
+      .withGraphFetched('attributes(selectAttribute)')
+      .joinRelated('[message, tx, smart_contract.code]')
       .where('smart_contract:code.type', 'CW20')
       .where('tx.height', '>', startBlock)
       .andWhere('tx.height', '<=', endBlock)
+      .modify((builder) => {
+        if (smartContractId) {
+          builder.andWhere('smart_contract.id', smartContractId);
+        }
+        if (page) {
+          builder
+            .andWhere('smart_contract_event.id', '>', page.prevId)
+            .orderBy('smart_contract_event.id', 'asc')
+            .limit(page.limit);
+        }
+      })
       .select(
         'message.sender as sender',
         'smart_contract.address as contract_address',
         'smart_contract_event.action',
-        'smart_contract_event.event_id as event_id',
-        'smart_contract_event.index',
+        'smart_contract_event.event_id',
         'smart_contract.id as smart_contract_id',
-        'tx.height as height',
-        'smart_contract_event.id as smart_contract_event_id'
+        'smart_contract_event.id as smart_contract_event_id',
+        'tx.hash',
+        'tx.height'
       )
       .orderBy('smart_contract_event.id', 'asc');
   }
@@ -309,4 +314,45 @@ export default class Cw20Service extends BullableService {
     }
     return super._start();
   }
+
+  @QueueHandler({
+    queueName: BULL_JOB_NAME.REINDEX_CW20_HISTORY,
+    jobName: BULL_JOB_NAME.REINDEX_CW20_HISTORY,
+  })
+  public async reindexHistory(_payload: ICw20ReindexingHistoryParams) {
+    const { smartContractId, startBlock, endBlock, prevId, contractAddress } =
+      _payload;
+    // insert data from event_attribute_backup to event_attribute
+    const { limitRecordGet } = config.cw20.reindexHistory;
+    const events = await this.getCw20ContractEvents(
+      startBlock,
+      endBlock,
+      smartContractId,
+      { limit: limitRecordGet, prevId }
+    );
+    if (events.length > 0) {
+      await knex.transaction(async (trx) => {
+        await this.handleCw20Histories(events, trx);
+      });
+      await this.createJob(
+        BULL_JOB_NAME.REINDEX_CW20_HISTORY,
+        BULL_JOB_NAME.REINDEX_CW20_HISTORY,
+        {
+          smartContractId,
+          startBlock,
+          endBlock,
+          prevId: events[events.length - 1].smart_contract_event_id,
+          contractAddress,
+        } satisfies ICw20ReindexingHistoryParams,
+        {
+          removeOnComplete: true,
+        }
+      );
+    } else {
+      const queue: Queue = this.getQueueManager().getQueue(
+        BULL_JOB_NAME.REINDEX_CW20_CONTRACT
+      );
+      (await queue.getJob(contractAddress))?.remove();
+    }
+  }
 }