first commit

.gitignore

@@ -0,0 +1,4 @@
+.vscode
+.env
+*.log
+node_modules

README.md

@@ -0,0 +1,77 @@
+# CAP Translation Gateway
+Andrei Besleaga (Nicolae) - 2024
+
+The Common Alerting Protocol (CAP) provides an open, non-proprietary digital message format for all types of alerts and notifications. It does not address any particular application or telecommunications method. However is it possible that it may be used without a standard language that should be always available for the final user.
+
+This is a proof of concept, work in progress, application for a distributed mesh gateway system for automated translation and adding of sections for certain data from Common Alert Protocol to universal English language and different encoding (JSON) to end result.
+
+Application will use translation services (APIs) to translate info data submitted to English, if non existent, and add the corresponding information to the final XML result to deliver it to the final user.
+
+### Backend consists of:
+
+1. API Gateway (REST API service) which is a NodeJs Express server for communication over HTTP(S) with clients
+2. CAP Translator microservice for operations
+3. Admin microservices for demo of other various facilities to be implemented
+
+Backend uses Node COTE — A Node.js library for building zero-configuration microservices (which creates a mesh network of microservices), chosen for:
+- Zero dependency: Multiple Microservices with only JavaScript and Node.js 
+- Zero-configuration: no IP addresses, no ports, no routing to configure
+- Decentralized: No fixed parts, no "manager" nodes, no single point of failure
+- Auto-discovery: Services discover each other without a central bookkeeper
+- Fault-tolerant: Don't lose any requests when a service is down
+- Scalable: Horizontally scale to any number of machines
+- Performant: Process thousands of messages per second
+- Humanized API: simple to get started with a reasonable API
+
+All services should be run with PM2 process manager (which also takes care of caught/uncaught exceptions and failures of runtime code and can log all for debug/dev purposes and also gives full availability to the microservice - with restart and resources management - unless deployed to a cloud appservice or similar and other managed mechanism can be used).
+
+The cap-gateway directory requires node only for monitor.js as example of inter-microservices communications only.
+
+## Installation
+git clone https://github.com/andreibesleaga/cap-gateway
+
+cd cap-gateway
+
+npm install
+
+cd responder-microservices
+
+npm install
+
+npm start
+
+cd ../gateway
+
+npm install
+
+npm start
+
+### For the quickest start:
+
+Have PM2 installed globally and type: pm2 start all.json
+This will run all the services you need, and you can monitor your services with pm2 monit or use any pm2 commands at your disposal.
+
+Or cd to gateway and run once the : npm start, then cd to the responder-microservices and run as many times the microservices as necessary with : npm start
+
+### RESTful API endpoints:
+
+responder-microservices
+
+cap-microservice:
+POST /app/cap/translate : returns the posted XML CAP data with added english translation section (and optional result encoding JSON);
+
+admin-microservice:
+POST /app/admin/services/pubsub : publish messages for inter micro services communication;
+(request body params: message - original XML CAP data, exportJson - optional boolean to return just JSON encoded result instead of XML)
+
+All API endpoints calls to be tested with Postman, ThunderClient directly from VSCode or other REST API tools.
+
+
+## Architecture considerations
+
+Current app is implemented a standard REST API with distributed mesh microservices and uses third party APIs for traslating the language and add new english section in the result, for automated use of the protocol where needed to be available in english, without further development of existing software, by routing requests through the gateway, or where the application will use a new proposed encoding format of CAP protocol to JSON instead of XML encoding.
+
+API gateway is the entry point (to be deployed on cloud for availability/scalability) which balances all the requests to services.
+The responder microservice can be started as many times as necessary in the mesh network so that many responders will be available and balance the requests.
+
+Everything would be deployed on a cloud system and maybe using the cloud specific APIs/Resources for the operations as they are already managed/monitored and the needed resources increased when necessary so that the app would be always available and performant in case of high peak usage.

cap-gateway/all.json

@@ -0,0 +1,22 @@
+{
+    "apps": [
+        {
+            "name"      : "gateway",
+            "script"    : "gateway/index.js",
+            "out_file"  : "./LOGS/gateway.std.log",
+            "error_file": "./LOGS/gateway.err.log"
+        },
+        {
+            "name"      : "admin-service",
+            "script"    : "responder-microservices/index.js",
+            "out_file"  : "./LOGS/admin.std.log",
+            "error_file": "./LOGS/admin.err.log"
+        },
+        {
+            "name"      : "cap-service",
+            "script"    : "responder-microservices/index.js",
+            "out_file"  : "./LOGS/cap.std.log",
+            "error_file": "./LOGS/cap.err.log"
+        }
+    ]
+}

cap-gateway/gateway/.eslintrc.json

@@ -0,0 +1,12 @@
+{
+  "env": {
+    "node": true,
+    "es2021": true
+  },
+  "extends": ["eslint:recommended", "prettier"],
+  "parserOptions": {
+    "ecmaVersion": "latest",
+    "sourceType": "module"
+  },
+  "rules": { "no-console": "off", "comma-dangle": "off", "max-len": "off", "linebreak-style": "off" }
+}

cap-gateway/gateway/.gitignore

@@ -0,0 +1,4 @@
+.vscode
+.env
+*.log
+node_modules

cap-gateway/gateway/.prettierrc

@@ -0,0 +1,12 @@
+{
+  "printWidth": 120,
+  "singleQuote": true,
+  "semi": true,
+  "arrowParens": "avoid",
+  "endOfLine": "lf",
+  "bracketSpacing": true,
+  "trailingComma": "es5",
+  "useTabs": false,
+  "tabWidth": 2,
+  "editor.formatOnSave": true
+}

cap-gateway/gateway/README.md

@@ -0,0 +1,12 @@
+Api Gateway Service - Main NodeJS microservice Express HTTP(S) Server which routes all requests (register all microservices routes)
+
+- input sanitization middleware (should also validate the request CAP XML Schema);
+- .env constants to be set for each microservice when app deployed - copy from sample.env to .env and change for each;
+- NODE_ENV variable to be set to: production, when deployed, for each microservice;
+- \_health - route to check if server responds - to be checked on pipeline and restart the service if not responding;
+- PM2 installed as a package/global and used to run and automatically restart services on crashes - also logs to .log - check package.json starter scripts;
+
+- servers have APPLOG_microservice.log logs when started with PM2;
+- api-gateway started with PM2 (not clustered because of COTE services discovery);
+- api-gateway has access-logs file in dir, if path defined in .env with a combined Apache format logging of requests and responses times;
+- constants.js - program constants settings;

cap-gateway/gateway/constants.js

@@ -0,0 +1,22 @@
+// API Gateway Endpoints - app URLs routed in controllers to coresponding microservices
+export const Endpoints = Object.freeze({
+  admin: {
+    pubsub: '/app/admin/services/pubsub', // start admin pubsub messager to be received by all microservices subscribed
+  },
+  cap: {
+    translate: '/app/cap/translate', //  sends a message;
+  },
+});
+
+// HTTP CODES contants used in the main app server
+export const HTTP_CODE = Object.freeze({
+  OK: 200,
+  BadRequest: 400,
+  Unauthorized: 401,
+  ServerError: 500,
+  NotImplemented: 501,
+  ServiceUnavailable: 503,
+});
+
+// microservice call timeout value
+export const SERVICE_TIMEOUT = 3000;

cap-gateway/gateway/controllers/admin.js

@@ -0,0 +1,19 @@
+import { HTTP_CODE, Endpoints, SERVICE_TIMEOUT } from '../constants.js';
+import { sanitizeParams } from '../middlewares/sanitize.js';
+import  cote  from 'cote';
+
+const Endpoint = Endpoints.admin;
+const getError = error => ({ error: error?.response?.data ?? error?.response?.message ?? error?.message ?? error?.response});
+
+export function registerAdminCalls(app) {
+  app.post(Endpoint.pubsub, sanitizeParams, async (req, res) => {
+    try {
+      const requester = new cote.Requester({ name: 'admin_requester_pubsub', timeout: SERVICE_TIMEOUT});
+      const request = { type:'start' };
+      let r = await requester.send(request);
+      return (r.status!==undefined && r.error!==undefined) ? res.status(r.status).json({ error: r.error }) : res.send(r);      
+    } catch (error) {
+      return res.status(error?.response?.status ?? HTTP_CODE.ServerError).json(getError(error));
+    }
+  });
+}

cap-gateway/gateway/controllers/cap.js

@@ -0,0 +1,19 @@
+import { HTTP_CODE, Endpoints, SERVICE_TIMEOUT } from '../constants.js';
+import { sanitizeParams } from '../middlewares/sanitize.js';
+import  cote  from 'cote';
+
+const Endpoint = Endpoints.cap;
+const getError = error => ({ error: error?.response?.data ?? error?.response?.message ?? error?.message ?? error?.response});
+
+export function registerCapCalls(app) {
+  app.post(Endpoint.translate, sanitizeParams, async (req, res) => {
+    try {
+      const requester = new cote.Requester({ name: 'cap_requester_translate', timeout: SERVICE_TIMEOUT});
+      const request = { type:'translate', message: req.body.message, exportJson: req.body.exportJson ?? false};
+      let r = await requester.send(request);
+      return (r.status!==undefined && r.error!==undefined) ? res.status(r.status).json({ error: r.error }) : res.send(r);
+    } catch (error) {
+      return res.status(error?.response?.status ?? HTTP_CODE.ServerError).json(getError(error));
+    }
+  });
+}

cap-gateway/gateway/index.js

@@ -0,0 +1,80 @@
+/**
+ * RESTful API Gateway Express Microservice
+ */
+/* eslint-disable no-unused-vars */
+
+// import required
+import dotenv from 'dotenv';
+import express, { json, urlencoded } from 'express';
+import compression from 'compression';
+import morgan from 'morgan';
+import helmet from 'helmet';
+import { createServer } from 'https';
+import logger from './logger.js';
+
+import { readFileSync, existsSync, mkdirSync } from 'fs';
+import { join } from 'path';
+import * as path from 'path';
+import { createStream } from 'rotating-file-stream';
+
+// import controllers for microservices
+import { registerAdminCalls } from './controllers/admin.js';
+import { registerCapCalls } from './controllers/cap.js';
+
+// read config values
+dotenv.config({ silent: process.env.NODE_ENV === 'production' });
+const { APP_URL, APP_PORT, ACCESS_LOGS_DIR } = process.env;
+
+// setup server
+const app = express();
+
+// HTTPS server settings (keys if in production and HTTPS mode)
+let httpsServer = null;
+if (process.env.NODE_ENV === 'production') {
+  const key = readFileSync('../key.pem');
+  const cert = readFileSync('../cert.pem');
+  httpsServer = createServer({ key, cert }, app);
+}
+
+// All other server modules and options
+app.use(helmet());
+app.use(compression());
+app.use(json());
+app.use(urlencoded({ extended: true, parameterLimit: 10 }));
+
+// web server access-logs - create logs in the log directory if config setting exists in .env
+if (ACCESS_LOGS_DIR) {
+  if (!existsSync(ACCESS_LOGS_DIR)) {
+    mkdirSync(ACCESS_LOGS_DIR, { recursive: true });
+  }
+  const dirname = path.resolve();
+  const accessLogStream = createStream('access.log', { path: join(dirname, ACCESS_LOGS_DIR)});
+  app.use( 
+    morgan(
+      ':remote-addr - :remote-user [:date[clf]] ":method :url HTTP/:http-version" :status :res[content-length] ":referrer" ":user-agent" - :response-time ms - :total-time ms',
+      { stream: accessLogStream }
+    )
+  );
+}
+
+
+// register microservices endpoints
+registerAdminCalls(app);
+registerCapCalls(app);
+
+// redirect all unknown endpoints to /
+function redirectUnmatched(req, res) {
+  res.redirect(APP_URL);
+}
+app.use(redirectUnmatched);
+
+// run HTTP(S) server local/production
+if (process.env.NODE_ENV === 'production') {
+  httpsServer.listen(APP_PORT, () => {
+    logger.info(`API Gateway listening HTTPS at ${APP_URL}:${APP_PORT}`);
+  });
+} else {
+  app.listen(APP_PORT, () => {
+    logger.info(`API Gateway listening at ${APP_URL}:${APP_PORT}`);
+  });
+}

cap-gateway/gateway/logger.js

@@ -0,0 +1,16 @@
+import { createLogger, format, transports } from 'winston';
+
+// Configure the Winston logger. For the complete documentation see https://github.com/winstonjs/winston
+const logger = createLogger({
+  // To see more detailed errors, change this to 'debug'
+  level: 'info',
+  format: format.combine(
+    format.splat(),
+    format.simple()
+  ),
+  transports: [
+    new transports.Console()
+  ],
+});
+
+export default logger;

cap-gateway/gateway/middlewares/sanitize.js

@@ -0,0 +1,52 @@
+import validator from 'validator';
+import dotenv from 'dotenv';
+dotenv.config({ silent: process.env.NODE_ENV === 'production' });
+
+import { HTTP_CODE } from '../constants.js';
+const getError = error => ({ error: error?.response?.data ?? error?.response?.message ?? error?.message ?? error?.response});
+
+const MAX_PARAM_NAME_LENGTH = 50;
+const MAX_POST_PARAM_VALUES_LENGTH = 100000;
+
+export async function sanitizeParams(req, res, next) {
+  // middleware for basic sanitization of all requests parameters
+  try {
+    let valType = undefined;
+    let newValue = null;
+
+    if (req.body) {
+      let strValue = '';
+      // sanitize POST values
+      for (const [key, value] of Object.entries(req.body)) {
+        valType = typeof value;
+
+        if (valType == 'object' && value == null) {
+          req.body[key] = null;
+        } else {
+          if (valType == 'object') {
+            strValue = JSON.stringify(value);
+          } else {
+            strValue = value + '';
+          }
+
+          if (valType == 'symbol' || valType == 'function') {
+            return res.status(HTTP_CODE.BadRequest).json({ error: 'Malformed Request' });
+          }
+          if (key.length > MAX_PARAM_NAME_LENGTH || strValue.length > MAX_POST_PARAM_VALUES_LENGTH) {
+            return res.status(HTTP_CODE.BadRequest).json({ error: 'Malformed Request' });
+          }
+
+          if (valType == 'string') {
+            newValue = validator.trim(value);
+            newValue = validator.stripLow(newValue, true);
+            req.body[key] = newValue;
+          }
+        }
+      }
+    }
+
+    next();
+  } catch (error) {
+    return res.status(error.response?.status ?? error.status ?? HTTP_CODE.ServerError).json(getError(error));
+  }
+}