Merge pull request #266 from lidofinance/feat/val-1135-e2e-impr

hardhat e2e for dsm v3

.github/workflows/tests_and_checks.yml

@@ -1,27 +1,54 @@
 name: Tests and checks
 on: pull_request
 jobs:
-  test:
-    name: App tests
+  unit_tests:
+    name: Unit Tests
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repo
         uses: actions/checkout@v3
       - name: Set up node
         uses: actions/setup-node@v3
         with:
-          node-version: 14
+          node-version: 18
           cache: 'yarn'
       - name: Install dependencies
         run: yarn install --immutable
       - name: Run lint
         run: yarn lint
       - name: Run tests
         run: yarn test
+        env:
+          RPC_URL: ${{ secrets.RPC_URL }}
+          CHAIN_ID: 17000
+          LOCATOR_DEVNET_ADDRESS: "0x28FAB2059C713A7F9D8c86Db49f9bb0e96Af1ef8"
+          WALLET_PRIVATE_KEY: ${{ secrets.WALLET_PRIVATE_KEY }}
+
+  e2e_tests:
+    name: E2E Tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v3
+      - name: Set up node
+        uses: actions/setup-node@v3
+        with:
+          node-version: 18
+          cache: 'yarn'
+      - name: Install dependencies
+        run: yarn install --immutable
       - name: Run e2e transports
         run: docker compose -f docker-compose.test.yml up -d
+      - name: Pull kapi
+        run: docker pull lidofinance/lido-keys-api:staging
+      - name: Pull psql
+        run: docker pull postgres:14-alpine
       - name: Run e2e tests
         run: yarn test:e2e
         env:
           RPC_URL: ${{ secrets.RPC_URL }}
           WALLET_PRIVATE_KEY: ${{ secrets.WALLET_PRIVATE_KEY }}
+          KEYS_API_URL: http://127.0.0.1:3000
+          CHAIN_ID: 17000
+          LOCATOR_DEVNET_ADDRESS: "0x28FAB2059C713A7F9D8c86Db49f9bb0e96Af1ef8"
+

hardhat.config.ts

@@ -0,0 +1,31 @@
+import { HardhatUserConfig } from 'hardhat/config';
+import * as dotenv from 'dotenv';
+dotenv.config();
+
+// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+const RPC_URL = process.env.RPC_URL!;
+// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+const secretKey = process.env.WALLET_PRIVATE_KEY!;
+const CHAIN_ID = process.env.CHAIN_ID || '17000';
+
+const config: HardhatUserConfig = {
+  networks: {
+    hardhat: {
+      forking: {
+        url: RPC_URL,
+      },
+      chainId: parseInt(CHAIN_ID, 10),
+      accounts: [
+        {
+          privateKey: secretKey,
+          balance: (BigInt(1e18) * BigInt(100)).toString(),
+        },
+      ],
+    },
+  },
+  solidity: {
+    version: '0.8.4',
+  },
+};
+
+export default config;

package.json

@@ -17,11 +17,11 @@
     "start:debug": "nest start --debug --watch",
     "start:prod": "node --max-old-space-size=4096 dist/main",
     "lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix",
-    "test": "jest",
+    "test": "jest --detectOpenHandles",
     "test:watch": "jest --watch",
     "test:cov": "jest --coverage",
     "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand",
-    "test:e2e": "yarn cache:clear && jest --detectOpenHandles --forceExit --config ./test/jest-e2e.json",
+    "test:e2e": "yarn cache:clear && jest --detectOpenHandles --runInBand --forceExit --config ./test/jest-e2e.json",
     "typechain": "typechain --target=ethers-v5 --out-dir ./src/generated ./src/abi/*.json",
     "postinstall": "yarn typechain",
     "docker:build": "docker build -t $npm_package_config_dockerorg/$npm_package_name:$npm_package_version .",
@@ -48,6 +48,7 @@
     "class-transformer": "^0.5.1",
     "class-validator": "^0.14.0",
     "compare-versions": "^6.1.0",
+    "dockerode": "^4.0.2",
     "ethers": "5.7.2",
     "glob": "^7.1.2",
     "kafkajs": "^1.15.0",
@@ -67,7 +68,9 @@
     "@nestjs/cli": "^8.2.5",
     "@nestjs/schematics": "^8.0.0",
     "@nestjs/testing": "^8.0.11",
+    "@nomicfoundation/hardhat-ethers": "^3.0.6",
     "@typechain/ethers-v5": "^7.1.2",
+    "@typechain/hardhat": "^9.1.0",
     "@types/app-root-path": "^1.2.4",
     "@types/cache-manager": "^3.4.3",
     "@types/glob": "^8.1.0",
@@ -82,7 +85,9 @@
     "eslint-config-prettier": "^8.3.0",
     "eslint-plugin-prettier": "^3.4.0",
     "ganache": "7.9.0",
+    "hardhat": "^2.22.14",
     "jest": "^27.0.6",
+    "pg": "^8.13.0",
     "prettier": "^2.3.2",
     "supertest": "^6.1.3",
     "ts-jest": "^27.0.3",

src/contracts/data-bus/data-bus.client.spec.ts

@@ -1,14 +1,7 @@
 import { Block } from '@ethersproject/providers';
 import { ethers } from 'ethers';
 import { formatBytes32String } from 'ethers/lib/utils';
-import { Server } from 'ganache';
-import {
-  CHAIN_ID,
-  FORK_BLOCK,
-  GANACHE_PORT,
-  UNLOCKED_ACCOUNTS,
-} from '../../../test/constants';
-import { makeServer } from '../../../test/server';
+import { TEST_SERVER_PORT } from './utils/constants';
 import { DataBusClient } from './data-bus.client';
 import {
   MessageDepositV1,
@@ -19,6 +12,11 @@ import {
   MessagesNames,
   MessageUnvetV1,
 } from './data-bus.serializer';
+import { HardhatServer } from '../../../test/helpers/hardhat-server';
+import { accountImpersonate, setBalance } from '../../../test/helpers/provider';
+import { getSecurityOwner } from '../../../test/helpers/dsm';
+
+jest.setTimeout(40_000);
 
 export const randomInt = (min: number, max: number) =>
   Math.floor(Math.random() * (max - min + 1)) + min;
@@ -100,25 +98,30 @@ describe('DataBus', () => {
   let provider: ethers.providers.JsonRpcProvider;
   let owner: ethers.Signer;
   let sdk: DataBusClient;
-  let server: Server<'ethereum'>;
   let variants: ReturnType<typeof getVariants>;
+  let hardhatServer: HardhatServer;
+  let dsmOwnerAddress: string;
 
   const setupServer = async () => {
-    server = makeServer(FORK_BLOCK, CHAIN_ID, UNLOCKED_ACCOUNTS, true);
-    await server.listen(GANACHE_PORT);
+    hardhatServer = new HardhatServer();
+    await hardhatServer.start();
   };
 
   beforeEach(async () => {
     await setupServer();
+    dsmOwnerAddress = await getSecurityOwner();
+    await accountImpersonate(dsmOwnerAddress);
+    await setBalance(dsmOwnerAddress, 100);
+
     // Set up Ganache provider (ensure Ganache is running on port 8545)
     provider = new ethers.providers.JsonRpcProvider(
-      'http://localhost:' + GANACHE_PORT,
+      'http://127.0.0.1:' + TEST_SERVER_PORT,
     );
     variants = getVariants(await provider.getBlock('latest'));
 
     // Get the first account as the owner
-    const accounts = await provider.listAccounts();
-    owner = provider.getSigner(accounts[0]);
+    // const accounts = await provider.listAccounts();
+    owner = provider.getSigner(dsmOwnerAddress); //accounts[0]);
 
     // Deploy the DataBus contract from bytecode
     const dataBusBytecode =
@@ -142,7 +145,7 @@ describe('DataBus', () => {
   });
 
   afterEach(async () => {
-    await server.close();
+    await hardhatServer.stop();
   });
 
   it('should measure gas for sendPingMessage', async () => {

src/contracts/data-bus/utils/constants.ts

@@ -0,0 +1 @@
+export const TEST_SERVER_PORT = 8545;

src/guardian/guardian-message/guardian-message.service.ts

@@ -74,6 +74,10 @@ export class GuardianMessageService {
     });
   }
 
+  /**
+   * Sends an unvetting message to the message broker
+   * @param message - MessageUnvet object
+   */
   public sendUnvetMessage(message: Omit<MessageUnvet, 'type'>) {
     return this.sendMessageFromGuardian({
       ...message,

src/guardian/guardian.service.ts

@@ -213,8 +213,10 @@ export class GuardianService implements OnModuleInit {
       this.handleKeys(stakingModulesData, blockData, lidoKeys)
         .catch(this.logger.error)
         .finally(() => {
-          this.logger.log('End of unvetting and deposits processing by Guardian');
-          endTimer()
+          this.logger.log(
+            'End of unvetting and deposits processing by Guardian',
+          );
+          endTimer();
         });
     } catch (error) {
       this.logger.error('Guardian cycle processing error');

src/keys-api/interfaces/SRModuleKeyListResponse.ts

@@ -0,0 +1,13 @@
+import { SRModule } from '.';
+import { ELBlockSnapshot } from './ELBlockSnapshot';
+import { RegistryKey } from './RegistryKey';
+
+export type SRModuleKeyListResponse = {
+  data: {
+    keys: Array<RegistryKey>;
+    module: SRModule;
+  };
+  meta: {
+    elBlockSnapshot: ELBlockSnapshot;
+  };
+};

src/keys-api/keys-api.service.ts

@@ -8,6 +8,7 @@ import { Configuration } from 'common/config';
 import { GroupedByModuleOperatorListResponse } from './interfaces/GroupedByModuleOperatorListResponse';
 import { InconsistentLastChangedBlockHash } from 'common/custom-errors';
 import { SRModuleListResponse } from './interfaces/SRModuleListResponse';
+import { SRModuleKeyListResponse } from './interfaces/SRModuleKeyListResponse';
 import { ELBlockSnapshot } from './interfaces/ELBlockSnapshot';
 import { DeepReadonly } from 'common/ts-utils';
 
@@ -157,6 +158,16 @@ export class KeysApiService {
     return result;
   }
 
+  /**
+   * The /v1/modules/{module_id}/keys endpoint returns full list of keys
+   */
+  public async getModuleKeys(moduleId: number, opId: number) {
+    const result = await this.fetch<SRModuleKeyListResponse>(
+      `/v1/modules/${moduleId}/keys?operatorIndex=${opId}`,
+    );
+    return result;
+  }
+
   /**
    * Verifies the consistency of metadata by comparing hashes.
    * @param firstRequestHash - Hash of the first request

src/provider/index.ts

@@ -3,4 +3,4 @@ export * from './provider.constants';
 export * from './provider.mock';
 export * from './provider.module';
 export * from './provider.service';
-export * from './provider.ganache';
+export * from './provider.test';

src/provider/provider.ganache.ts → src/provider/provider.test.ts

@@ -4,12 +4,12 @@ import { Configuration } from 'common/config';
 import { RpcBatchProvider, RpcProvider } from './interfaces';
 import { ProviderService } from './provider.service';
 
-export const GANACHE_PORT = 8545;
-export const GANACHE_URL = `http://127.0.0.1:${GANACHE_PORT}`;
+export const TEST_SERVER_PORT = 8545;
+export const TEST_SERVER_URL = `http://127.0.0.1:${TEST_SERVER_PORT}`;
 
 const getProviderFactory = () => {
   return async (): Promise<RpcProvider> => {
-    class FormatterGanache extends Formatter {
+    class FormatterTest extends Formatter {
       blockTag(blockTag: any): any {
         if (typeof blockTag === 'object' && blockTag != null) {
           return 'latest';
@@ -24,25 +24,25 @@ const getProviderFactory = () => {
 
       static getFormatter(): Formatter {
         if (this._formatter == null) {
-          this._formatter = new FormatterGanache();
+          this._formatter = new FormatterTest();
         }
         return this._formatter;
       }
 
       clone() {
-        return new Provider(GANACHE_URL);
+        return new Provider(TEST_SERVER_URL);
       }
     }
 
-    return new Provider(GANACHE_URL);
+    return new Provider(TEST_SERVER_URL);
   };
 };
 
 @Module({})
-export class GanacheProviderModule {
+export class TestProviderModule {
   static forRoot(): DynamicModule {
     return {
-      module: GanacheProviderModule,
+      module: TestProviderModule,
       global: true,
       providers: [
         ProviderService,

src/staking-module-data-collector/staking-module-data-collector.service.ts

@@ -210,6 +210,7 @@ export class StakingModuleDataCollectorService {
       (key) =>
         !key.used && key.vetted && key.moduleAddress === stakingModuleAddress,
     );
+
     return vettedUnusedKeys;
   }
 }

test/constants.ts

@@ -1,4 +1,3 @@
-import { CHAINS } from '@lido-sdk/constants';
 import { SecretKey } from '@chainsafe/blst';
 
 import { fromHexString } from '@chainsafe/ssz';
@@ -9,38 +8,12 @@ export const TESTS_TIMEOUT = 30_000;
 // Needs to be higher on gh actions for reliable runs
 export const SLEEP_FOR_RESULT = 3_000;
 
-// Addresses
-export const SECURITY_MODULE = '0x808DE3b26Be9438F12E9B45528955EA94C17f217';
-// https://holesky.etherscan.io/address/0x808DE3b26Be9438F12E9B45528955EA94C17f217#readContract
-// getOwner
-export const SECURITY_MODULE_OWNER =
-  '0xE92329EC7ddB11D25e25b3c21eeBf11f15eB325d';
-
-export const SECURITY_MODULE_V2 = '0x045dd46212a178428c088573a7d102b9d89a022a';
-// https://holesky.etherscan.io/address/0x045dd46212a178428c088573a7d102b9d89a022a#readContract
-// getOwner
-export const SECURITY_MODULE_OWNER_V2 =
-  '0xDA6bEE5441f2e6b364F3b25E85d5f3C29Bfb669E';
-export const STAKING_ROUTER = '0xd6EbF043D30A7fe46D1Db32BA90a0A51207FE229';
-export const NOP_REGISTRY = '0x595F64Ddc3856a3b5Ff4f4CC1d1fb4B46cFd2bAC';
-export const DEPOSIT_CONTRACT = '0x4242424242424242424242424242424242424242';
-export const SIMPLE_DVT = '0x11a93807078f8BB880c1BD0ee4C387537de4b4b6';
-export const CSM = '0x4562c3e63c2e586cD1651B958C22F88135aCAd4f';
-export const SANDBOX = '0xD6C2ce3BB8bea2832496Ac8b5144819719f343AC';
 // Withdrawal credentials
-export const LIDO_WC =
-  '0x010000000000000000000000f0179dec45a37423ead4fad5fcb136197872ead9';
 export const BAD_WC =
   '0x010000000000000000000000b9d7934878b5fb9610b3fe8a5e441e8fad7e291f';
 
-// Fork node config
-export const CHAIN_ID = CHAINS.Holesky;
-
-export const FORK_BLOCK = 1894357;
-export const FORK_BLOCK_V2 = 1817726;
-export const UNLOCKED_ACCOUNTS = [SECURITY_MODULE_OWNER];
-export const UNLOCKED_ACCOUNTS_V2 = [SECURITY_MODULE_OWNER_V2];
-export const GANACHE_PORT = 8545;
+export const TEST_SERVER_PORT = 8545;
+export const TEST_SERVER_URL = 'http://127.0.0.1:8545';
 
 // BLS key for the validator
 export const BLS_PRIV_KEY =