feat: Prometheus and HealthCheck (#1780)

backend/package.json

@@ -29,18 +29,21 @@
     "@nestjs/platform-express": "^10.0.0",
     "@nestjs/schematics": "^10.0.0",
     "@nestjs/swagger": "^7.0.3",
+    "@nestjs/terminus": "^10.2.1",
     "@nestjs/testing": "^10.0.0",
     "@prisma/client": "^5.7.0",
     "dotenv": "^16.0.1",
+    "express-prom-bundle": "^7.0.0",
+    "helmet": "^7.0.0",
+    "nest-winston": "^1.9.4",
     "nestjs-prisma": "^0.22.0",
     "pg": "^8.11.3",
+    "prom-client": "^15.1.0",
     "reflect-metadata": "^0.2.0",
     "rimraf": "^5.0.0",
     "rxjs": "^7.8.0",
     "swagger-ui-express": "^5.0.0",
-    "winston": "^3.11.0",
-    "nest-winston": "^1.9.4",
-    "helmet": "^7.0.0"
+    "winston": "^3.11.0"
   },
   "devDependencies": {
     "@types/express": "^4.17.15",

backend/prisma/schema.prisma

@@ -1,5 +1,6 @@
 generator client {
   provider = "prisma-client-js"
+  previewFeatures = ["metrics"]
 }
 
 datasource db {

backend/src/app.module.ts

@@ -1,11 +1,14 @@
 import "dotenv/config";
-import { Logger, MiddlewareConsumer, Module } from "@nestjs/common";
+import { Logger, MiddlewareConsumer, Module, RequestMethod } from "@nestjs/common";
 import { HTTPLoggerMiddleware } from "./middleware/req.res.logger";
 import { loggingMiddleware, PrismaModule } from "nestjs-prisma";
 import { ConfigModule } from "@nestjs/config";
 import { UsersModule } from "./users/users.module";
 import { AppService } from "./app.service";
 import { AppController } from "./app.controller";
+import { MetricsController } from "./metrics.controller";
+import { TerminusModule } from '@nestjs/terminus';
+import { HealthController } from "./health.controller";
 
 const DB_HOST = process.env.POSTGRES_HOST || "localhost";
 const DB_USER = process.env.POSTGRES_USER || "postgres";
@@ -14,33 +17,41 @@ const DB_PORT = process.env.POSTGRES_PORT || 5432;
 const DB_NAME = process.env.POSTGRES_DATABASE || "postgres";
 const DB_SCHEMA = process.env.DB_SCHEMA || "users";
 
+function getMiddlewares() {
+  if (process.env.PRISMA_LOGGING) {
+    return [
+      // configure your prisma middleware
+      loggingMiddleware({
+        logger: new Logger("PrismaMiddleware"),
+        logLevel: "debug"
+      })
+    ];
+  }
+  return [];
+}
+
 @Module({
   imports: [
     ConfigModule.forRoot(),
+    TerminusModule,
     PrismaModule.forRoot({
       isGlobal: true,
       prismaServiceOptions:{
         prismaOptions:{
-          log: ["query", "info", "error", "warn"],
+          log: ["error", "warn"],
           errorFormat: "pretty",
           datasourceUrl: `postgresql://${DB_USER}:${DB_PWD}@${DB_HOST}:${DB_PORT}/${DB_NAME}?schema=${DB_SCHEMA}&connection_limit=5`,
         },
-        middlewares: [
-          // configure your prisma middleware
-          loggingMiddleware({
-            logger: new Logger("PrismaMiddleware"),
-            logLevel: "log"
-          })
-        ]
+        middlewares: getMiddlewares(),
       },
     }),
     UsersModule
   ],
-  controllers: [AppController],
+  controllers: [AppController,MetricsController, HealthController],
   providers: [AppService]
 })
 export class AppModule { // let's add a middleware on all routes
   configure(consumer: MiddlewareConsumer) {
-    consumer.apply(HTTPLoggerMiddleware).forRoutes("*");
+    consumer.apply(HTTPLoggerMiddleware).exclude({ path: 'metrics', method: RequestMethod.ALL }, { path: 'health', method: RequestMethod.ALL }).forRoutes('*');
   }
 }

backend/src/app.spec.ts

@@ -1,6 +1,18 @@
 import {NestExpressApplication} from '@nestjs/platform-express';
 import {bootstrap} from "./app";
-
+jest.mock('prom-client', () => ({
+  Registry: jest.fn().mockImplementation(() => ({
+  })),
+  collectDefaultMetrics: jest.fn().mockImplementation(() => ({
+  })),
+}));
+jest.mock('express-prom-bundle', () => ({
+  default: jest.fn().mockImplementation(() => ({
+  })),
+}));
+jest.mock('./prom', () => ({
+  metricsMiddleware: jest.fn().mockImplementation((req, res, next) => next()),
+}));
 describe('main', () => {
   let app: NestExpressApplication;
 

backend/src/app.ts

@@ -5,6 +5,7 @@ import { customLogger } from './common/logger.config';
 import { NestExpressApplication } from '@nestjs/platform-express';
 import helmet from 'helmet';
 import { VersioningType } from '@nestjs/common';
+import { metricsMiddleware } from "./prom";
 
 /**
  *
@@ -17,6 +18,7 @@ export async function bootstrap() {
   app.use(helmet());
   app.enableCors();
   app.set("trust proxy", 1);
+  app.use(metricsMiddleware);
   app.enableShutdownHooks();
   app.setGlobalPrefix("api");
   app.enableVersioning({

backend/src/health.controller.ts

@@ -0,0 +1,19 @@
+import { Controller, Get } from "@nestjs/common";
+import { HealthCheckService, HealthCheck, PrismaHealthIndicator } from "@nestjs/terminus";
+import { PrismaService } from "nestjs-prisma";
+@Controller("health")
+export class HealthController {
+  constructor(
+    private health: HealthCheckService,
+    private prisma: PrismaHealthIndicator,
+    private readonly prismaService: PrismaService,
+  ) {}
+
+  @Get()
+  @HealthCheck()
+  check() {
+    return this.health.check([
+      () => this.prisma.pingCheck('prisma', this.prismaService),
+    ]);
+  }
+}

backend/src/metrics.controller.ts

@@ -0,0 +1,15 @@
+import { Controller, Get, Res } from "@nestjs/common";
+import { Response } from "express";
+import { register } from "./prom";
+import { PrismaService } from "nestjs-prisma";
+@Controller("metrics")
+export class MetricsController {
+  constructor(private prisma: PrismaService) {}
+
+  @Get()
+  async getMetrics(@Res() res: Response) {
+    const prismaMetrics = await this.prisma.$metrics.prometheus();
+    const appMetrics = await register.metrics();
+    res.end(prismaMetrics + appMetrics);
+  }
+}

backend/src/middleware/req.res.logger.ts

@@ -10,7 +10,7 @@ export class HTTPLoggerMiddleware implements NestMiddleware {
 
     response.on("finish", () => {
       const { statusCode } = response;
-      const contentLength = response.get("content-length");
+      const contentLength = response.get("content-length") || '-';
       const hostedHttpLogFormat = `${method} ${originalUrl} ${statusCode} ${contentLength} - ${request.get(
         "user-agent"
       )}`;

backend/src/prom.ts

@@ -0,0 +1,11 @@
+import * as prom from 'prom-client';
+import promBundle from 'express-prom-bundle';
+const register = new prom.Registry();
+prom.collectDefaultMetrics({ register });
+const metricsMiddleware = promBundle({
+  includeMethod: true,
+  includePath: true,
+  metricsPath: '/prom-metrics',
+  promRegistry: register,
+});
+export { metricsMiddleware, register };

charts/quickstart-openshift/values.yaml

@@ -79,7 +79,7 @@ backend:
       #-- the readiness probe for the container. it is optional and is an object. for default values check this link: https://github.com/bcgov/helm-service/blob/main/charts/component/templates/deployment.yaml#L312-L316
       readinessProbe:
         httpGet:
-          path: /api
+          path: /api/health
           port: 3000
           scheme: HTTP
         initialDelaySeconds: 5
@@ -92,7 +92,7 @@ backend:
         successThreshold: 1
         failureThreshold: 3
         httpGet:
-          path: /api
+          path: /api/health
           port: 3000
           scheme: HTTP
         initialDelaySeconds: 15
@@ -208,7 +208,10 @@ backend:
   nodeSelector: { }
   tolerations: [ ]
   affinity: { }
-
+  podAnnotations: |
+    prometheus.io/scrape: 'true'
+    prometheus.io/port: '3000'
+    prometheus.io/path: '/api/metrics'
 frontend:
   # -- enable or disable a component deployment.
   enabled: true
@@ -276,6 +279,9 @@ frontend:
         - name: http2
           containerPort: 3001
           protocol: TCP
+        - name: metrics
+          containerPort: 3003
+          protocol: TCP
       #-- the resources for the container. it is optional and is an object. for default values check this link: https://github.com/bcgov/helm-service/blob/main/charts/component/templates/deployment.yaml#L298-L304
       resources:
         limits:
@@ -306,6 +312,10 @@ frontend:
         initialDelaySeconds: 15
         periodSeconds: 30
         timeoutSeconds: 5
+  podAnnotations: |
+    prometheus.io/scrape: 'true'
+    prometheus.io/port: '3003'
+    prometheus.io/path: '/metrics'
   #-- autoscaling for the component. it is optional and is an object.
   autoscaling:
     #-- enable or disable autoscaling.
@@ -353,6 +363,10 @@ frontend:
         targetPort: 3000
         #-- the protocol for the port. it can be TCP or UDP. TCP is the default and is recommended.
         protocol: TCP
+      - port: 3003
+        targetPort: 3003
+        protocol: TCP
+        name: metrics
   #-- the route for the component. it is optional and is an object. To make the application accessible from internet, use the route.
   route:
     #-- enable or disable the route.

frontend/Caddyfile

@@ -1,6 +1,9 @@
 {
 	auto_https off
-	admin off
+	admin 0.0.0.0:3003
+    servers {
+        metrics
+    }
 }
 :3000 {
 	log {