Feat/reindex cw20 dev ( develop )

ci/config.json.ci

@@ -108,7 +108,10 @@
   "cw20": {
     "blocksPerCall": 100,
     "millisecondRepeatJob": 2000,
-    "key": "cw20"
+    "key": "cw20",
+    "reindexHistory": {
+      "limitRecordGet": 500
+    }
   },
   "dashboardStatistics": {
     "millisecondCrawl": 10000,

config.json

@@ -103,7 +103,10 @@
   "cw20": {
     "blocksPerCall": 100,
     "millisecondRepeatJob": 2000,
-    "key": "cw20"
+    "key": "cw20",
+    "reindexHistory": {
+      "limitRecordGet": 500
+    }
   },
   "crawlContractEvent": {
     "key": "crawlContractEvent",

src/common/constant.ts

@@ -72,6 +72,8 @@ export const BULL_JOB_NAME = {
   HANDLE_MIGRATE_CONTRACT: 'handle:migrate-contract',
   JOB_REDECODE_TX: 'job:redecode-tx',
   CRAWL_IBC_TAO: 'crawl:ibc-tao',
+  REINDEX_CW20_CONTRACT: 'reindex:cw20-contract',
+  REINDEX_CW20_HISTORY: 'reindex:cw20-history',
 };
 
 export const SERVICE = {
@@ -253,6 +255,14 @@ export const SERVICE = {
       key: 'CrawlIBCTaoService',
       name: 'v1.CrawlIBCTaoService',
     },
+    Cw20ReindexingService: {
+      key: 'Cw20ReindexingService',
+      name: 'v1.Cw20ReindexingService',
+      Reindexing: {
+        key: 'reindexing',
+        path: 'v1.Cw20ReindexingService.reindexing',
+      },
+    },
   },
 };
 

src/models/cw20_activity.ts

@@ -5,6 +5,8 @@ import { Cw20Contract } from './cw20_contract';
 import { SmartContract } from './smart_contract';
 
 export class Cw20Event extends BaseModel {
+  static softDelete = false;
+
   [relation: string]: any;
 
   id!: number;

src/models/cw20_contract.ts

@@ -26,6 +26,8 @@ export interface IContractInfo {
   name?: string;
 }
 export class Cw20Contract extends BaseModel {
+  static softDelete = false;
+
   [relation: string]: any;
 
   id!: number;

src/models/cw20_holder.ts

@@ -5,6 +5,8 @@ import { Cw20Contract } from './cw20_contract';
 import { SmartContract } from './smart_contract';
 
 export class CW20Holder extends BaseModel {
+  static softDelete = false;
+
   [relation: string]: any;
 
   id?: number;

src/models/cw20_total_holder_stats.ts

@@ -3,6 +3,8 @@ import BaseModel from './base';
 import { Cw20Contract } from './cw20_contract';
 
 export class CW20TotalHolderStats extends BaseModel {
+  static softDelete = false;
+
   [relation: string]: any;
 
   date!: Date;

src/services/cw20/cw20.service.ts

@@ -1,4 +1,5 @@
 import { Service } from '@ourparentcenter/moleculer-decorators-extended';
+import { Queue } from 'bullmq';
 import { Knex } from 'knex';
 import _ from 'lodash';
 import { ServiceBroker } from 'moleculer';
@@ -34,6 +35,13 @@ export const CW20_ACTION = {
   BURN_FROM: 'burn_from',
   SEND_FROM: 'send_from',
 };
+export interface ICw20ReindexingHistoryParams {
+  smartContractId: number;
+  startBlock: number;
+  endBlock: number;
+  prevId: number;
+  contractAddress: string;
+}
 @Service({
   name: SERVICE.V1.Cw20.key,
   version: 1,
@@ -205,41 +213,38 @@ export default class Cw20Service extends BullableService {
     }
   }
 
-  async getCw20ContractEvents(startBlock: number, endBlock: number) {
+  async getCw20ContractEvents(
+    startBlock: number,
+    endBlock: number,
+    smartContractId?: number,
+    page?: { prevId: number; limit: number }
+  ) {
     return SmartContractEvent.query()
-      .alias('smart_contract_event')
-      .withGraphJoined(
-        '[message(selectMessage), tx(selectTransaction), attributes(selectAttribute), smart_contract(selectSmartContract).code(selectCode)]'
-      )
-      .modifiers({
-        selectCode(builder) {
-          builder.select('type');
-        },
-        selectTransaction(builder) {
-          builder.select('hash', 'height');
-        },
-        selectMessage(builder) {
-          builder.select('sender', 'content');
-        },
-        selectAttribute(builder) {
-          builder.select('key', 'value');
-        },
-        selectSmartContract(builder) {
-          builder.select('address', 'id');
-        },
-      })
+      .withGraphFetched('attributes(selectAttribute)')
+      .joinRelated('[message, tx, smart_contract.code]')
       .where('smart_contract:code.type', 'CW20')
       .where('tx.height', '>', startBlock)
       .andWhere('tx.height', '<=', endBlock)
+      .modify((builder) => {
+        if (smartContractId) {
+          builder.andWhere('smart_contract.id', smartContractId);
+        }
+        if (page) {
+          builder
+            .andWhere('smart_contract_event.id', '>', page.prevId)
+            .orderBy('smart_contract_event.id', 'asc')
+            .limit(page.limit);
+        }
+      })
       .select(
         'message.sender as sender',
         'smart_contract.address as contract_address',
         'smart_contract_event.action',
-        'smart_contract_event.event_id as event_id',
-        'smart_contract_event.index',
+        'smart_contract_event.event_id',
         'smart_contract.id as smart_contract_id',
-        'tx.height as height',
-        'smart_contract_event.id as smart_contract_event_id'
+        'smart_contract_event.id as smart_contract_event_id',
+        'tx.hash',
+        'tx.height'
       )
       .orderBy('smart_contract_event.id', 'asc');
   }
@@ -309,4 +314,45 @@ export default class Cw20Service extends BullableService {
     }
     return super._start();
   }
+
+  @QueueHandler({
+    queueName: BULL_JOB_NAME.REINDEX_CW20_HISTORY,
+    jobName: BULL_JOB_NAME.REINDEX_CW20_HISTORY,
+  })
+  public async reindexHistory(_payload: ICw20ReindexingHistoryParams) {
+    const { smartContractId, startBlock, endBlock, prevId, contractAddress } =
+      _payload;
+    // insert data from event_attribute_backup to event_attribute
+    const { limitRecordGet } = config.cw20.reindexHistory;
+    const events = await this.getCw20ContractEvents(
+      startBlock,
+      endBlock,
+      smartContractId,
+      { limit: limitRecordGet, prevId }
+    );
+    if (events.length > 0) {
+      await knex.transaction(async (trx) => {
+        await this.handleCw20Histories(events, trx);
+      });
+      await this.createJob(
+        BULL_JOB_NAME.REINDEX_CW20_HISTORY,
+        BULL_JOB_NAME.REINDEX_CW20_HISTORY,
+        {
+          smartContractId,
+          startBlock,
+          endBlock,
+          prevId: events[events.length - 1].smart_contract_event_id,
+          contractAddress,
+        } satisfies ICw20ReindexingHistoryParams,
+        {
+          removeOnComplete: true,
+        }
+      );
+    } else {
+      const queue: Queue = this.getQueueManager().getQueue(
+        BULL_JOB_NAME.REINDEX_CW20_CONTRACT
+      );
+      (await queue.getJob(contractAddress))?.remove();
+    }
+  }
 }

src/services/cw20/cw20_reindexing.service.ts

@@ -0,0 +1,169 @@
+import {
+  Action,
+  Service,
+} from '@ourparentcenter/moleculer-decorators-extended';
+import _ from 'lodash';
+import { Context, ServiceBroker } from 'moleculer';
+import config from '../../../config.json' assert { type: 'json' };
+import BullableService, { QueueHandler } from '../../base/bullable.service';
+import { BULL_JOB_NAME, IContextUpdateCw20, SERVICE } from '../../common';
+import {
+  CW20Holder,
+  CW20TotalHolderStats,
+  Cw20Contract,
+  Cw20Event,
+  IHolderEvent,
+  SmartContract,
+} from '../../models';
+import { ICw20ReindexingHistoryParams } from './cw20.service';
+
+export interface IAddressParam {
+  contractAddress: string;
+}
+interface ICw20ReindexingParams {
+  contractAddress: string;
+  smartContractId: number;
+}
+@Service({
+  name: SERVICE.V1.Cw20ReindexingService.key,
+  version: 1,
+})
+export default class Cw20ReindexingContract extends BullableService {
+  public constructor(public broker: ServiceBroker) {
+    super(broker);
+  }
+
+  @Action({
+    name: SERVICE.V1.Cw20ReindexingService.Reindexing.key,
+    params: {
+      contractAddress: 'string',
+    },
+  })
+  public async reindexing(ctx: Context<IAddressParam>) {
+    const { contractAddress } = ctx.params;
+    const smartContract = await SmartContract.query()
+      .withGraphJoined('code')
+      .where('address', contractAddress)
+      .first()
+      .throwIfNotFound();
+
+    // check whether contract is Cw20 type -> throw error to user
+    if (smartContract.code.type === 'CW20') {
+      await this.createJob(
+        BULL_JOB_NAME.REINDEX_CW20_CONTRACT,
+        BULL_JOB_NAME.REINDEX_CW20_CONTRACT,
+        {
+          contractAddress,
+          smartContractId: smartContract.id,
+        } satisfies ICw20ReindexingParams,
+        {
+          jobId: contractAddress,
+        }
+      );
+    } else {
+      throw new Error(
+        `Smart contract ${ctx.params.contractAddress} is not CW20 type`
+      );
+    }
+  }
+
+  @QueueHandler({
+    queueName: BULL_JOB_NAME.REINDEX_CW20_CONTRACT,
+    jobName: BULL_JOB_NAME.REINDEX_CW20_CONTRACT,
+  })
+  async jobHandler(_payload: ICw20ReindexingParams): Promise<void> {
+    const { smartContractId, contractAddress } = _payload;
+    const cw20Contract = await Cw20Contract.query()
+      .withGraphJoined('smart_contract')
+      .where('smart_contract.address', contractAddress)
+      .select(['cw20_contract.id'])
+      .first();
+    // query
+    const contractInfo = (
+      await Cw20Contract.getContractsInfo([contractAddress])
+    )[0];
+    let track = true;
+    let initBalances: IHolderEvent[] = [];
+    // get init address holder, init amount
+    try {
+      initBalances = await Cw20Contract.getInstantiateBalances(contractAddress);
+    } catch (error) {
+      track = false;
+    }
+    const minUpdatedHeightOwner =
+      _.min(initBalances.map((holder) => holder.event_height)) || 0;
+    const maxUpdatedHeightOwner =
+      _.max(initBalances.map((holder) => holder.event_height)) || 0;
+    if (cw20Contract) {
+      await Cw20Event.query()
+        .delete()
+        .where('cw20_contract_id', cw20Contract.id);
+      await CW20TotalHolderStats.query()
+        .delete()
+        .where('cw20_contract_id', cw20Contract.id);
+      await CW20Holder.query()
+        .delete()
+        .where('cw20_contract_id', cw20Contract.id);
+      await Cw20Contract.query().deleteById(cw20Contract.id);
+    }
+    const newCw20Contract = await Cw20Contract.query().insertGraph({
+      ...Cw20Contract.fromJson({
+        smart_contract_id: smartContractId,
+        symbol: contractInfo?.symbol,
+        minter: contractInfo?.minter,
+        marketing_info: contractInfo?.marketing_info,
+        name: contractInfo?.name,
+        total_supply: initBalances.reduce(
+          (acc: string, curr: { address: string; amount: string }) =>
+            (BigInt(acc) + BigInt(curr.amount)).toString(),
+          '0'
+        ),
+        track,
+        decimal: contractInfo?.decimal,
+        last_updated_height: minUpdatedHeightOwner,
+      }),
+      holders: initBalances.map((e) => ({
+        address: e.address,
+        amount: e.amount,
+        last_updated_height: e.event_height,
+      })),
+    });
+    // handle from minUpdatedHeightOwner to blockHeight
+    await this.broker.call(
+      SERVICE.V1.Cw20UpdateByContract.UpdateByContract.path,
+      {
+        cw20Contracts: [
+          {
+            id: newCw20Contract.id,
+            last_updated_height: newCw20Contract.last_updated_height,
+          },
+        ],
+        startBlock: minUpdatedHeightOwner,
+        endBlock: maxUpdatedHeightOwner,
+      } satisfies IContextUpdateCw20
+    );
+    // insert histories
+    await this.createJob(
+      BULL_JOB_NAME.REINDEX_CW20_HISTORY,
+      BULL_JOB_NAME.REINDEX_CW20_HISTORY,
+      {
+        smartContractId,
+        startBlock: config.crawlBlock.startBlock,
+        endBlock: maxUpdatedHeightOwner,
+        prevId: 0,
+        contractAddress,
+      } satisfies ICw20ReindexingHistoryParams,
+      {
+        removeOnComplete: true,
+      }
+    );
+  }
+
+  async _start(): Promise<void> {
+    await this.broker.waitForServices([
+      SERVICE.V1.Cw20.name,
+      SERVICE.V1.Cw20UpdateByContract.name,
+    ]);
+    return super._start();
+  }
+}

src/services/cw20/cw20_update_by_contract.service.ts

@@ -78,7 +78,7 @@ export default class Cw20UpdateByContractService extends BullableService {
     const { startBlock, endBlock } = ctx.params;
     // eslint-disable-next-line no-restricted-syntax
     for (const cw20Contract of ctx.params.cw20Contracts) {
-      const startUpdateBlock = Math.min(
+      const startUpdateBlock = Math.max(
         startBlock,
         cw20Contract.last_updated_height
       );