Merge pull request #60 from nemgrouplimited/dev

Release 1.0.1

.prettierignore

@@ -0,0 +1 @@
+src/models/NodeRewards

CHANGELOG.md

@@ -3,6 +3,22 @@ All notable changes to this project will be documented in this file.
 
 The changelog format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
+## [v1.0.1][v1.0.1] - 25-May-2021
+
+### Milestone: [[email protected]][[email protected]]
+
+Package  | Version  | Link
+---|---|---
+REST Core| v2.3.6 | [catapult-rest][[email protected]]
+SDK Core| v1.0.1 | [symbol-sdk][[email protected]]
+
+- Added the network type check to the Node Monitor Service [#53](https://github.com/nemgrouplimited/symbol-statistics-service/issues/53)
+- Added version of REST to the Node List [#55](https://github.com/nemgrouplimited/symbol-statistics-service/issues/53)
+- Fixed the peer node status check issue [#58](https://github.com/nemgrouplimited/symbol-statistics-service/pull/58)
+- Improved DB collection store [#58](https://github.com/nemgrouplimited/symbol-statistics-service/pull/58)
+- Included the initial nodes from config to the list [#58](https://github.com/nemgrouplimited/symbol-statistics-service/pull/58)
+- Added missing utf8 dependency
+
 ## [v1.0.0][v1.0.0] - 14-Mar-2021
 
 ### Milestone: [[email protected]][[email protected]]
@@ -71,7 +87,8 @@ REST Core| v2.1.0 | [catapult-rest](https://github.com/nemtech/catapult-rest/rel
 ### Fixes
 - Cors error. [#13](https://github.com/nemgrouplimited/symbol-statistics-service/issues/13)
 
-
+[v1.0.1]: https://github.com/nemfoundation/symbol-statistics-service/releases/tag/v1.0.1
+[v1.0.0]: https://github.com/nemfoundation/symbol-statistics-service/releases/tag/v1.0.0
 [v0.3.0]: https://github.com/nemfoundation/symbol-statistics-service/releases/tag/v0.3.0
 [v0.2.0]: https://github.com/nemfoundation/symbol-statistics-service/releases/tag/v0.2.0
 [v0.1.0]: https://github.com/nemfoundation/symbol-statistics-service/releases/tag/v0.1.0
@@ -83,6 +100,8 @@ REST Core| v2.1.0 | [catapult-rest](https://github.com/nemtech/catapult-rest/rel
 [[email protected]]: https://www.npmjs.com/package/symbol-sdk/v/0.23.3-alpha-202102181227
 [[email protected]]: https://www.npmjs.com/package/symbol-sdk/v/0.23.3
 [[email protected]]: https://www.npmjs.com/package/symbol-sdk/v/1.0.0
+[[email protected]]: https://www.npmjs.com/package/symbol-sdk/v/1.0.1
 
 [[email protected]]: https://github.com/nemtech/catapult-rest/releases/tag/v2.3.3
 [[email protected]]: https://github.com/nemtech/catapult-rest/releases/tag/v2.3.4
+[[email protected]]: https://github.com/nemtech/catapult-rest/releases/tag/v2.3.6

package.json

@@ -1,6 +1,6 @@
 {
     "name": "symbol-statistics-service",
-    "version": "1.0.0",
+    "version": "1.0.1",
     "description": "",
     "scripts": {
         "dev": "nodemon",
@@ -19,7 +19,7 @@
         "@src": "dist"
     },
     "devDependencies": {
-        "@openapitools/openapi-generator-cli": "^2.1.12",
+        "@openapitools/openapi-generator-cli": "^2.2.6",
         "@types/mongoose": "^5.7.14",
         "@types/node": "^14.14.10",
         "@types/winston": "^2.4.4",
@@ -29,7 +29,7 @@
         "eslint-config-prettier": "^6.12.0",
         "eslint-plugin-prettier": "^3.1.4",
         "nodemon": "^2.0.3",
-        "openapi-to-postmanv2": "^2.0.0",
+        "openapi-to-postmanv2": "^2.6.0",
         "prettier": "^2.1.2",
         "rimraf": "^3.0.2",
         "swagger-cli": "^4.0.4",
@@ -48,8 +48,9 @@
         "express": "^4.17.1",
         "module-alias": "^2.2.2",
         "mongoose": "^5.9.16",
-        "symbol-sdk": "^1.0.0",
+        "symbol-sdk": "^1.0.1",
         "tcp-ping": "^0.1.1",
+        "utf8": "^3.0.0",
         "uuid": "^8.0.0",
         "winston": "^3.3.3"
     }

src/config/config.json

@@ -5,19 +5,17 @@
 	"CHAIN_HEIGHT_MONITOR_SCHEDULE_INTERVAL": 15000,
 	"GEOLOCATION_MONITOR_SCHEDULE_INTERVAL": 30000,
 	"NODES": [
-		"http://api-01.ap-northeast-1.0.10.0.x.symboldev.network:3000",
-		"http://api-01.ap-southeast-1.0.10.0.x.symboldev.network:3000",
-		"http://api-01.eu-central-1.0.10.0.x.symboldev.network:3000",
-		"http://api-01.eu-west-1.0.10.0.x.symboldev.network:3000",
-		"http://api-01.us-east-1.0.10.0.x.symboldev.network:3000",
-		"http://api-01.us-west-1.0.10.0.x.symboldev.network:3000",
-		"http://api-01.us-west-2.0.10.0.x.symboldev.network:3000",
-		"http://api-01.us-west-2.testnet.symboldev.network:3000",
-		"http://api-01.us-west-1.testnet.symboldev.network:3000",
-		"http://api-01.ap-northeast-1.testnet.symboldev.network:3000"
+		"http://ngl-dual-001.testnet.symboldev.network:3000",
+		"http://ngl-dual-101.testnet.symboldev.network:3000",
+		"http://ngl-dual-201.testnet.symboldev.network:3000",
+		"http://ngl-dual-301.testnet.symboldev.network:3000",
+		"http://ngl-dual-401.testnet.symboldev.network:3000",
+		"http://ngl-dual-501.testnet.symboldev.network:3000",
+		"http://ngl-dual-601.testnet.symboldev.network:3000"
 	],
 	"API_NODE_PORT": 3000,
 	"PEER_NODE_PORT": 7900,
 	"REQUEST_TIMEOUT": 5000,
-	"CONTROLLER_ENDPOINT": "http://34.252.126.146:7890"
+	"CONTROLLER_ENDPOINT": "http://34.252.126.146:7890",
+	"NETWORK_IDENTIFIER": 152
 }

src/config/index.ts

@@ -1,5 +1,6 @@
 import * as config from './config.json';
 import * as utils from '@src/utils';
+import { NetworkType } from 'symbol-sdk';
 
 interface Network {
 	PORT: number;
@@ -20,6 +21,7 @@ interface Monitor {
 	API_NODE_PORT: number;
 	PEER_NODE_PORT: number;
 	REQUEST_TIMEOUT: number;
+	NETWORK_IDENTIFIER: NetworkType;
 }
 
 interface NodeRewardsConfig {
@@ -55,6 +57,7 @@ export const monitor: Monitor = {
 	API_NODE_PORT: Number(process.env.API_NODE_PORT) || config.API_NODE_PORT,
 	PEER_NODE_PORT: Number(process.env.PEER_NODE_PORT) || config.PEER_NODE_PORT,
 	REQUEST_TIMEOUT: Number(process.env.REQUEST_TIMEOUT) || config.REQUEST_TIMEOUT,
+	NETWORK_IDENTIFIER: Number(process.env.NETWORK_IDENTIFIER) || config.NETWORK_IDENTIFIER,
 };
 
 export const nodeRewards: NodeRewardsConfig = {
@@ -91,6 +94,9 @@ export const verifyConfig = (cfg: Config): boolean => {
 
 	if (isNaN(cfg.monitor.REQUEST_TIMEOUT) || cfg.monitor.REQUEST_TIMEOUT <= 0) error = 'Invalid "REQUEST_TIMEOUT"';
 
+	if (isNaN(cfg.monitor.NETWORK_IDENTIFIER) || !Object.values(NetworkType).includes(cfg.monitor.NETWORK_IDENTIFIER))
+		error = 'Invalid "NETWORK_IDENTIFIER"';
+
 	if (error) throw 'Invalid config. ' + error;
 	return true;
 };

src/models/Node.ts

@@ -70,6 +70,10 @@ const NodeSchema: Schema = new Schema({
 			type: String,
 			required: false,
 		},
+		restVersion: {
+			type: String,
+			required: false,
+		},
 		lastStatusCheck: {
 			type: Number,
 			required: false,
@@ -176,3 +180,9 @@ NodeSchema.set('toObject', {
 });
 
 export const Node = mongoose.model<NodeDocument>('Node', NodeSchema);
+
+export const validateNodeModel = (node: any): boolean => {
+	if (!node || typeof node !== 'object') return false;
+
+	return !new Node(node).validateSync();
+};

src/services/ApiNodeService.ts

@@ -10,6 +10,7 @@ export interface ApiStatus {
 	chainHeight?: number;
 	finalizationHeight?: number;
 	nodePublicKey?: string;
+	restVersion?: string;
 	lastStatusCheck: number;
 }
 
@@ -44,12 +45,14 @@ export class ApiNodeService {
 		try {
 			const nodeInfo = (await HTTP.get(`http://${host}:${port}/node/info`)).data;
 			const chainInfo = (await HTTP.get(`http://${host}:${port}/chain/info`)).data;
+			const nodeServer = (await HTTP.get(`http://${host}:${port}/node/server`)).data;
 
 			return {
 				isAvailable: true,
 				chainHeight: chainInfo.height,
 				finalizationHeight: chainInfo.latestFinalizedBlock.height,
 				nodePublicKey: nodeInfo.nodePublicKey,
+				restVersion: nodeServer.serverInfo.restVersion,
 				lastStatusCheck: Date.now(),
 			};
 		} catch (e) {

src/services/DataBase.ts

@@ -3,7 +3,7 @@ import * as winston from 'winston';
 import { basename } from '@src/utils';
 import { Logger } from '@src/infrastructure';
 import { INode, NodeDocument, Node } from '@src/models/Node';
-import { IHostDetail, HostDetail } from '@src/models/HostDetail';
+import { IHostDetail, HostDetail, HostDetailDocument } from '@src/models/HostDetail';
 import { INodesStats, NodesStatsDocument, NodesStats } from '@src/models/NodesStats';
 import { INodeHeightStats, NodeHeightStatsDocument, NodeHeightStats } from '@src/models/NodeHeightStats';
 import { NodeCountSeries } from '@src/models/NodeCountSeries';
@@ -40,17 +40,7 @@ export class DataBase {
 	};
 
 	static updateNodeList = async (nodeList: INode[]): Promise<void> => {
-		// Replace this part with mongo transactions
-		const prevNodeList = await DataBase.getNodeList();
-
-		await Node.deleteMany();
-		await Node.insertMany(nodeList);
-		const currentNodeList = await DataBase.getNodeList();
-
-		if (currentNodeList.length !== nodeList.length) {
-			await Node.deleteMany();
-			await Node.insertMany(prevNodeList);
-		}
+		await DataBase.updateCollection<NodeDocument>(Node, nodeList, 'Node');
 	};
 
 	static updateNode = async (node: INode): Promise<void> => {
@@ -66,13 +56,11 @@ export class DataBase {
 	};
 
 	static updateNodesStats = async (nodeList: INodesStats): Promise<void> => {
-		await NodesStats.deleteMany();
-		await NodesStats.create(nodeList);
+		await DataBase.updateCollection<NodesStatsDocument>(NodesStats, [nodeList], 'NodesStats');
 	};
 
 	static updateNodeHeightStats = async (nodeHeightStats: INodeHeightStats): Promise<void> => {
-		await NodeHeightStats.deleteMany();
-		await NodeHeightStats.create(nodeHeightStats);
+		await DataBase.updateCollection<NodeHeightStatsDocument>(NodeHeightStats, [nodeHeightStats], 'NodeHeightStats');
 	};
 
 	static getNodesHostDetail = async (): Promise<IHostDetail[]> => {
@@ -84,11 +72,44 @@ export class DataBase {
 	};
 
 	static updateNodesHostDetail = async (hostDetail: IHostDetail[]): Promise<void> => {
-		await HostDetail.deleteMany();
-		await HostDetail.insertMany(hostDetail);
+		await DataBase.updateCollection<HostDetailDocument>(HostDetail, hostDetail, 'HostDetail');
 	};
 
 	static getNodeCountSeries = async (): Promise<AbstractTimeSeriesDocument[]> => {
 		return NodeCountSeries.find().exec();
 	};
+
+	private static updateCollection = async <T extends mongoose.Document>(
+		model: mongoose.Model<T>,
+		documents: Array<any>,
+		collectionName: string,
+	) => {
+		const prevState = await model.find().exec();
+		let error = Error();
+
+		try {
+			await model.deleteMany();
+		} catch (e) {
+			logger.error(`Update collection "${collectionName}" failed. Error during "model.deleteMany()". ${error.message}`);
+			throw error;
+		}
+
+		try {
+			await model.insertMany(documents);
+		} catch (e) {
+			logger.error(`Update collection "${collectionName}" failed. Error during "model.insertMany()". ${error.message}`);
+			await model.insertMany(prevState);
+			throw error;
+		}
+
+		const currentState = await model.find().exec();
+
+		if (documents.length !== currentState.length) {
+			logger.error(
+				`Update collection "${collectionName}" failed. Collectin.length(${currentState.length}) !== documentsToInsert.length(${documents.length})`,
+			);
+			await model.insertMany(prevState);
+			throw new Error(`Failed to update collection "${collectionName}. Length verification failed`);
+		}
+	};
 }

src/services/HostInfo.ts

@@ -43,7 +43,7 @@ export class HostInfo {
 				zip: data.zip,
 			};
 		} catch (e) {
-			logger.error(`Failed to get host info ${e.message}`);
+			logger.error(`Failed to get host ${host} info ${e.message}`);
 			return null;
 		}
 	};