feat: add global metrics (#10)

CHANGELOG

@@ -1,5 +1,16 @@
 # Changelog
 
+## 0.3.0 | 2023-08-25
+
+#### Features
+
+- Add global metrics (you can get the count of all files, whatever their prefix are)
+
+#### Chores
+
+- Improve files architectures (prefixed/global metrics)
+- Upgrade dependencies
+
 ## 0.2.0 | 2023-08-19
 
 #### Features

README.md

@@ -14,3 +14,18 @@ docker pull tchoupinax/s3-prometheus-exporter
 - `PORT`: The port on which the server listens
 - `S3_PREFIXES`: The prefix of files to watch. You can provide a list separataed by a comma. (e.g `prefix1,prefix2`)
 - `S3_SECRET_KEY`: 
+
+## Exemples
+
+When you provide prefixes like `pg,git` you will receive metrics built like this
+
+```
+s3_files_count_git 12
+s3_files_count_pg 13
+```
+
+Also, there is a global metrics that will take care about all files, whatever their prefix are
+
+```
+s3_files_count_global 25
+```

package.json

@@ -1,6 +1,6 @@
 {
   "name": "s3-prometheus-exporter",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "description": "S3 data exporter for prometheus",
   "author": "Tchoupinax <[email protected]> (https://corentinfiloche.xyz)",
   "license": "MIT",
@@ -21,7 +21,7 @@
     "watch": "nodemon --exec vite-node --options.deps.inline=\"@aws-sdk\" src/index.ts"
   },
   "dependencies": {
-    "@aws-sdk/client-s3": "3.395.0",
+    "@aws-sdk/client-s3": "3.398.0",
     "config": "3.3.9",
     "express": "4.18.2",
     "fastify": "4.21.0",
@@ -30,13 +30,13 @@
     "pino-pretty": "10.2.0",
     "pretty-bytes": "6.1.1",
     "prom-client": "14.2.0",
-    "vite-node": "0.34.2",
-    "vitest": "0.34.2"
+    "vite-node": "0.34.3",
+    "vitest": "0.34.3"
   },
   "devDependencies": {
     "@types/config": "3.3.0",
     "@types/express": "4.17.17",
-    "@types/node": "20.5.1",
+    "@types/node": "20.5.6",
     "esbuild-plugin-tsc": "0.4.0",
     "eslint-config-tchoupinax": "1.0.3",
     "nodemon": "3.0.1",

src/index.ts

@@ -5,38 +5,55 @@ import config from "config";
 import Fastify, { FastifyReply } from "fastify";
 import { Registry } from "prom-client";
 
-import Metric from "./metrics/metric";
+import { Metric } from "./metrics/metric";
 import queryS3 from "./queryS3";
 import logger from "./utils/logger";
 
-let plugins: InstanceType<typeof Metric>[] = [];
+let globalPlugins: InstanceType<typeof Metric>[] = [];
+let prefixedPlugins: InstanceType<typeof Metric>[] = [];
 const register = new Registry();
 const prefixes = (config.get("prefixes") as string).split(",") ?? ["default"];
 
-const pluginsFileNames = readdirSync(path.join(__dirname, "/metrics"))
+const prefixedPluginsFileNames = readdirSync(path.join(__dirname, "/metrics/prefixed"))
   .filter(name => !name.includes("metric."))
+  .filter(name => !name.includes("spec"))
+  .filter(name => name.endsWith(".js") || name.endsWith(".ts"));
+
+const globalPluginsFileNames = readdirSync(path.join(__dirname, "/metrics/global"))
+  .filter(name => !name.includes("metric."))
+  .filter(name => !name.includes("spec"))
   .filter(name => name.endsWith(".js") || name.endsWith(".ts"));
 
 async function main (): Promise<void> {
   for (let j = 0; j < prefixes.length; j++) {
-    plugins = [
-      ...plugins,
+    prefixedPlugins = [
+      ...prefixedPlugins,
       ...(await Promise.all(
-        pluginsFileNames.map(async filename => {
-          const { default: LocalClass } = await import(path.join(__dirname, "metrics", filename));
+        prefixedPluginsFileNames.map(async filename => {
+          const { default: LocalClass } = await import(path.join(__dirname, "metrics/prefixed", filename));
           return (new LocalClass(prefixes[j])) as InstanceType<typeof Metric>;
         }))),
     ];
 
-    for (let i = 0; i < pluginsFileNames.length; i++) {
-      plugins[i + (j * pluginsFileNames.length)].saveMesure(plugins[i + (j * pluginsFileNames.length)].declarePrometheusMesure(register));
+    for (let i = 0; i < prefixedPluginsFileNames.length; i++) {
+      prefixedPlugins[i + (j * prefixedPluginsFileNames.length)].saveMesure(prefixedPlugins[i + (j * prefixedPluginsFileNames.length)].declarePrometheusMesure(register));
     }
   }
 
+  globalPlugins = await Promise.all(
+    globalPluginsFileNames.map(async filename => {
+      const { default: LocalClass } = await import(path.join(__dirname, "metrics/global", filename));
+      return (new LocalClass()) as InstanceType<typeof Metric>;
+    }));
+
+  for (let i = 0; i < globalPluginsFileNames.length; i++) {
+    globalPlugins[i].saveMesure(globalPlugins[i].declarePrometheusMesure(register));
+  }
+
   const app = Fastify();
   app.get("/metrics", async (_, reply: FastifyReply) => {
     reply.header("Content-Type", register.contentType);
-    await queryS3(plugins);
+    await queryS3(prefixedPlugins, globalPlugins);
     return reply.send(await register.metrics());
   });
 

src/metrics/global/metric-biggest-file-size.ts

@@ -0,0 +1,29 @@
+import { _Object } from "@aws-sdk/client-s3";
+import { Gauge, Registry } from "prom-client";
+
+import { Metric } from "../metric";
+
+export default class extends Metric {
+  constructor () {
+    super("biggest_file_size", "global");
+  }
+
+  declarePrometheusMesure (register: Registry): Gauge<any> {
+    return new Gauge({
+      name: this.name(),
+      help: "Biggest file size",
+      labelNames: ["name"],
+      registers: [register],
+    });
+  }
+
+  process (files: _Object[]): number {
+    return files.reduce((acc: number, cur: _Object) => {
+      if (cur.Size && acc < cur.Size) {
+        return cur.Size;
+      } else {
+        return acc;
+      }
+    }, 0);
+  }
+}

src/metrics/global/metric-files-count.ts

@@ -0,0 +1,23 @@
+import { _Object } from "@aws-sdk/client-s3";
+import { Gauge, Registry } from "prom-client";
+
+import { Metric } from "./../metric";
+
+export default class extends Metric {
+  constructor () {
+    super("files_count", "global");
+  }
+
+  declarePrometheusMesure (register: Registry): Gauge<any> {
+    return new Gauge({
+      name: this.name(),
+      help: "Count of files in the bucket",
+      labelNames: ["name"],
+      registers: [register],
+    });
+  }
+
+  process (files: _Object[]): number {
+    return files.length;
+  }
+}

src/metrics/global/metric-smallest-file-size.ts

@@ -0,0 +1,29 @@
+import { _Object } from "@aws-sdk/client-s3";
+import { Gauge, Registry } from "prom-client";
+
+import { Metric } from "../metric";
+
+export default class extends Metric {
+  constructor () {
+    super("smallest_file_size", "global");
+  }
+
+  declarePrometheusMesure (register: Registry): Gauge<any> {
+    return new Gauge({
+      name: this.name(),
+      help: "Smallest file size",
+      labelNames: ["name"],
+      registers: [register],
+    });
+  }
+
+  process (files: Array<_Object>): number {
+    return files.reduce((acc: number, cur: _Object) => {
+      if (cur.Size && acc < cur.Size) {
+        return cur.Size ?? 0;
+      } else {
+        return acc;
+      }
+    }, 0);
+  }
+}

src/metrics/metric.spec.ts

@@ -1,7 +1,7 @@
 import { Gauge } from "prom-client";
 import { beforeAll, describe, expect, it } from "vitest";
 
-import Metric from "./metric";
+import { Metric } from "./metric";
 
 class TestMetric extends Metric {
   constructor (prefix: string) {

src/metrics/metric.ts

@@ -1,7 +1,7 @@
 import { _Object } from "@aws-sdk/client-s3";
 import { Gauge, Registry } from "prom-client";
 
-export default abstract class Metric {
+export abstract class Metric {
   public metricName: string;
   private mesure?: Gauge<"mesure"> = undefined;
   protected prefix: string;

src/metrics/metric-biggest-file-size.ts → src/metrics/prefixed/metric-biggest-file-size.ts

@@ -1,7 +1,7 @@
 import { _Object } from "@aws-sdk/client-s3";
 import { Gauge, Registry } from "prom-client";
 
-import Metric from "./metric";
+import { Metric } from "../metric";
 
 export default class extends Metric {
   constructor (prefix: string) {

src/metrics/metric-files-count.ts → src/metrics/prefixed/metric-files-count.ts

@@ -1,7 +1,7 @@
 import { _Object } from "@aws-sdk/client-s3";
 import { Gauge, Registry } from "prom-client";
 
-import Metric from "./metric";
+import { Metric } from "../metric";
 
 export default class extends Metric {
   constructor (prefix: string) {
@@ -18,10 +18,6 @@ export default class extends Metric {
   }
 
   process (files: _Object[]): number {
-    if (this.prefix !== undefined) {
-      return files.filter(file => file.Key?.includes(this.prefix!)).length;
-    }
-
     return files.length;
   }
 }

src/metrics/metric-latest-file-size.ts → src/metrics/prefixed/metric-latest-file-size.ts

@@ -1,7 +1,7 @@
 import { _Object } from "@aws-sdk/client-s3";
 import { Gauge, Registry } from "prom-client";
 
-import Metric from "./metric";
+import { Metric } from "../metric";
 
 export default class extends Metric {
   constructor (prefix: string) {

src/metrics/metric-latest-file-timestamp.ts → src/metrics/prefixed/metric-latest-file-timestamp.ts

@@ -1,7 +1,7 @@
 import { _Object } from "@aws-sdk/client-s3";
 import { Gauge, Registry } from "prom-client";
 
-import Metric from "./metric";
+import { Metric } from "../metric";
 
 export default class extends Metric {
   constructor (prefix: string) {

src/metrics/metric-oldest-file-timestamp.ts → src/metrics/prefixed/metric-oldest-file-timestamp.ts

@@ -1,7 +1,7 @@
 import { _Object } from "@aws-sdk/client-s3";
 import { Gauge, Registry } from "prom-client";
 
-import Metric from "./metric";
+import { Metric } from "../metric";
 
 export default class extends Metric {
   constructor (prefix: string) {

src/metrics/metric-smallest-file-size.ts → src/metrics/prefixed/metric-smallest-file-size.ts

@@ -1,7 +1,7 @@
 import { _Object } from "@aws-sdk/client-s3";
 import { Gauge, Registry } from "prom-client";
 
-import Metric from "./metric";
+import { Metric } from "../metric";
 
 export default class extends Metric {
   constructor (prefix: string) {

src/queryS3.ts

@@ -1,7 +1,7 @@
 import { _Object, ListObjectsCommand, ListObjectsCommandInput, S3Client } from "@aws-sdk/client-s3";
 import config from "config";
 
-import Metric from "./metrics/metric";
+import { Metric } from "./metrics/metric";
 
 const s3Client = new S3Client({
   credentials: {
@@ -13,16 +13,23 @@ const s3Client = new S3Client({
 });
 
 export default async function (
-  plugins: InstanceType<typeof Metric>[],
+  prefixedPlugins: InstanceType<typeof Metric>[],
+  globalPlugins: InstanceType<typeof Metric>[],
 ): Promise<void> {
   const files = await listAllContents({ Bucket: config.get("bucket") });
 
-  for (let i = 0; i < plugins.length; i++) {
-    const prefix = plugins[i].getPrefix();
-    plugins[i].getMesure().set(plugins[i].process(
+  for (let i = 0; i < prefixedPlugins.length; i++) {
+    const prefix = prefixedPlugins[i].getPrefix();
+    prefixedPlugins[i].getMesure().set(prefixedPlugins[i].process(
       files.filter(file => file.Key?.includes(prefix)),
     ));
   }
+
+  for (let i = 0; i < globalPlugins.length; i++) {
+    globalPlugins[i].getMesure().set(globalPlugins[i].process(
+      files,
+    ));
+  }
 }
 
 async function listAllContents (