Add hash generator (#22)

* integrate hasset * add hasset * add back refs * fix name * remove arrayIsArray * readme * simplify * move fastq and glob to dependencies * update example * save
gurgunday · Jul 5, 2024 · 37f8cf3 · 37f8cf3
1 parent 8b9cc42
commit 37f8cf3
Show file tree

Hide file tree

Showing 11 changed files with 355 additions and 26 deletions.
diff --git a/README.md b/README.md
@@ -1,4 +1,4 @@
-Replace your template engine with fast JavaScript by leveraging the power of [tagged templates](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Template_literals#tagged_templates).
+ta**ghtml** lets you replace your template engine with fast JavaScript by leveraging the power of [tagged templates](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Template_literals#tagged_templates).
 
 Inspired by [html-template-tag](https://github.com/AntonioVdlC/html-template-tag).
 

diff --git a/bin/README.md b/bin/README.md
@@ -0,0 +1,66 @@
+Append unique hashes to assets referenced in your views to aggressively cache them while guaranteeing that clients receive the most recent versions.
+
+## Usage
+
+Running the following command will scan asset files found in the `roots` path(s) and replace their references with hashed versions in the `refs` path(s):
+
+```sh
+npx ghtml --roots="path/to/scan/assets1/,path/to/scan/assets2/" --refs="views/path/to/append/hashes1/,views/path/to/append/hashes2/"
+```
+
+## Example (Fastify)
+
+Register `@fastify/static`:
+
+```js
+await fastify.register(import("@fastify/static"), {
+  root: new URL("assets/", import.meta.url).pathname,
+  prefix: "/p/assets/",
+  wildcard: false,
+  index: false,
+  immutable: true,
+  maxAge: process.env.NODE_ENV === "production" ? 31536000 * 1000 : 0,
+});
+```
+
+Add the `ghtml` command to the build script:
+
+```json
+"scripts": {
+  "build": "npx ghtml --roots=assets/ --refs=views/,routes/",
+},
+```
+
+Make sure to `npm run build` in `Dockerfile`:
+
+```dockerfile
+FROM node:latest
+
+WORKDIR /app
+
+COPY package*.json ./
+
+RUN npm ci --include=dev
+
+COPY . .
+
+RUN npm run build
+
+RUN npm prune --omit=dev
+
+CMD ["npm", "start"]
+```
+
+## Demo
+
+A full project that uses the `ghtml` executable can be found in the `example` folder:
+
+```sh
+cd example
+
+npm i
+
+npm run build
+
+node .
+```
diff --git a/bin/example/assets/cat.jpeg b/bin/example/assets/cat.jpeg
diff --git a/bin/example/assets/style.css b/bin/example/assets/style.css
@@ -0,0 +1,19 @@
+body {
+  display: flex;
+  flex-direction: column;
+  align-items: center;
+  justify-content: center;
+  height: 100vh;
+  margin: 0;
+  padding: 0;
+}
+
+img {
+  max-width: 100%;
+  height: auto;
+}
+
+.caption {
+  text-align: center;
+  margin-top: 10px;
+}
diff --git a/bin/example/package.json b/bin/example/package.json
@@ -0,0 +1,13 @@
+{
+  "type": "module",
+  "main": "./server.js",
+  "scripts": {
+    "start": "node server.js",
+    "build": "node ../src/index.js --roots=assets/ --refs=routes/"
+  },
+  "dependencies": {
+    "@fastify/static": "^7.0.1",
+    "fastify": "^4.26.1",
+    "fastify-html": "^0.3.3"
+  }
+}
diff --git a/bin/example/routes/index.js b/bin/example/routes/index.js
@@ -0,0 +1,28 @@
+export default async (fastify) => {
+  const { html } = fastify;
+
+  fastify.addLayout((inner) => {
+    return html`<!doctype html>
+      <html lang="en">
+        <head>
+          <meta charset="UTF-8" />
+          <meta
+            name="viewport"
+            content="width=device-width, initial-scale=1.0"
+          />
+          <title>Document</title>
+          <link rel="stylesheet" href="/p/assets/style.css" />
+        </head>
+        <body>
+          !${inner}
+        </body>
+      </html>`;
+  });
+
+  fastify.get("/", async (request, reply) => {
+    return reply.html`
+      <h1 class="caption">Hello, world!</h1>
+      <img width="500" src="/p/assets/cat.jpeg" alt="Picture of a cat" />
+    `;
+  });
+};
diff --git a/bin/example/server.js b/bin/example/server.js
@@ -0,0 +1,22 @@
+/* eslint n/no-missing-import: "off" */
+
+import Fastify from "fastify";
+
+const fastify = Fastify();
+
+// Plugins
+await fastify.register(import("@fastify/static"), {
+  root: new URL("assets/", import.meta.url).pathname,
+  prefix: "/p/assets/",
+  wildcard: false,
+  index: false,
+  immutable: true,
+  maxAge: 31536000 * 1000,
+});
+await fastify.register(import("fastify-html"));
+
+// Routes
+fastify.register(import("./routes/index.js"));
+
+await fastify.listen({ port: 5050 });
+console.warn("Server listening at http://localhost:5050");
diff --git a/bin/src/index.js b/bin/src/index.js
@@ -0,0 +1,48 @@
+#!/usr/bin/env node
+
+import { generateHashesAndReplace } from "./utils.js";
+import process from "node:process";
+
+const parseArguments = (args) => {
+  let roots = null;
+  let refs = null;
+
+  for (const arg of args) {
+    if (arg.startsWith("--roots=")) {
+      roots = arg.split("=", 2)[1].split(",");
+    } else if (arg.startsWith("--refs=")) {
+      refs = arg.split("=", 2)[1].split(",");
+    }
+  }
+
+  if (!roots || !refs) {
+    console.error(
+      'Usage: npx ghtml --roots="path/to/scan/assets1/,path/to/scan/assets2/" --refs="views/path/to/append/hashes1/,views/path/to/append/hashes2/"',
+    );
+    process.exit(1);
+  }
+
+  return { roots, refs };
+};
+
+const main = async () => {
+  const { roots, refs } = parseArguments(process.argv.slice(2));
+
+  try {
+    console.warn(`Generating hashes and updating file paths...`);
+    console.warn(`Scanning files in: ${roots}`);
+    console.warn(`Updating files in: ${refs}`);
+
+    await generateHashesAndReplace({
+      roots,
+      refs,
+    });
+
+    console.warn("Hash generation and file updates completed successfully.");
+  } catch (error) {
+    console.error(`Error occurred: ${error.message}`);
+    process.exit(1);
+  }
+};
+
+main();
diff --git a/bin/src/utils.js b/bin/src/utils.js
@@ -0,0 +1,129 @@
+import { createHash } from "node:crypto";
+import { readFile, writeFile } from "node:fs/promises";
+import { win32, posix } from "node:path";
+import { cpus } from "node:os";
+import { Glob } from "glob";
+import { promise as fastq } from "fastq";
+const fastqConcurrency = Math.max(1, cpus().length - 1);
+
+const generateFileHash = async (filePath) => {
+  try {
+    const fileBuffer = await readFile(filePath);
+    return createHash("md5").update(fileBuffer).digest("hex").slice(0, 16);
+  } catch (err) {
+    if (err.code !== "ENOENT") {
+      throw err;
+    }
+    return "";
+  }
+};
+
+const updateFilePathsWithHashes = async (
+  fileHashes,
+  refs,
+  includeDotFiles,
+  skipPatterns,
+) => {
+  for (let ref of refs) {
+    ref = ref.split(win32.sep).join(posix.sep);
+    if (!ref.endsWith("/")) {
+      ref += "/";
+    }
+
+    const filesIterable = new Glob("**/**", {
+      nodir: true,
+      follow: true,
+      absolute: true,
+      cwd: ref,
+      dot: includeDotFiles,
+      ignore: skipPatterns,
+    });
+
+    for await (const file of filesIterable) {
+      let content = await readFile(file, "utf8");
+      let found = false;
+
+      for (const [originalPath, hash] of fileHashes) {
+        const escapedPath = originalPath.replace(
+          /[$()*+.?[\\\]^{|}]/gu,
+          "\\$&",
+        );
+        const regex = new RegExp(
+          `(?<path>${escapedPath})(\\?(?<queryString>[^#"'\`]*))?`,
+          "gu",
+        );
+
+        content = content.replace(
+          regex,
+          (match, p1, p2, p3, offset, string, groups) => {
+            found = true;
+            const { path, queryString } = groups;
+
+            return !queryString
+              ? `${path}?hash=${hash}`
+              : queryString.includes("hash=")
+                ? `${path}?${queryString.replace(/(?<hash>hash=)[\dA-Fa-f]*/u, `$1${hash}`)}`
+                : `${path}?hash=${hash}&${queryString}`;
+          },
+        );
+      }
+
+      if (found) {
+        await writeFile(file, content);
+      }
+    }
+  }
+};
+
+const generateHashesAndReplace = async ({
+  roots,
+  refs,
+  includeDotFiles = false,
+  skipPatterns = ["**/node_modules/**"],
+}) => {
+  const fileHashes = new Map();
+  roots = Array.isArray(roots) ? roots : [roots];
+  refs = Array.isArray(refs) ? refs : [refs];
+
+  for (let rootPath of roots) {
+    rootPath = rootPath.split(win32.sep).join(posix.sep);
+    if (!rootPath.endsWith("/")) {
+      rootPath += "/";
+    }
+
+    const queue = fastq(generateFileHash, fastqConcurrency);
+    const queuePromises = [];
+    const files = [];
+
+    const filesIterable = new Glob("**/**", {
+      nodir: true,
+      follow: true,
+      absolute: true,
+      cwd: rootPath,
+      dot: includeDotFiles,
+      ignore: skipPatterns,
+    });
+
+    for await (let file of filesIterable) {
+      file = file.split(win32.sep).join(posix.sep);
+      files.push(file);
+      queuePromises.push(queue.push(file));
+    }
+
+    const hashes = await Promise.all(queuePromises);
+
+    for (let i = 0; i < files.length; i++) {
+      const fileRelativePath = posix.relative(rootPath, files[i]);
+      fileHashes.set(fileRelativePath, hashes[i]);
+    }
+  }
+
+  await updateFilePathsWithHashes(
+    fileHashes,
+    refs,
+    includeDotFiles,
+    skipPatterns,
+  );
+};
+
+export { generateFileHash, generateHashesAndReplace };
diff --git a/package.json b/package.json
@@ -5,6 +5,7 @@
   "license": "MIT",
   "version": "2.0.4",
   "type": "module",
+  "bin": "./bin/src/index.js",
   "main": "./src/index.js",
   "exports": {
     ".": "./src/index.js",
@@ -19,9 +20,13 @@
     "lint": "eslint . && prettier --check .",
     "lint:fix": "eslint --fix . && prettier --write ."
   },
+  "dependencies": {
+    "fastq": "^1.17.1",
+    "glob": "^10.4.2"
+  },
   "devDependencies": {
     "@fastify/pre-commit": "^2.1.0",
-    "c8": "^10.0.0",
+    "c8": "^10.1.2",
     "grules": "^0.17.2",
     "tinybench": "^2.8.0"
   },