Fix index page format and cache

.devcontainer/devcontainer.json

@@ -1,5 +1,12 @@
 {
     "dockerComposeFile": "../docker-compose.yml",
     "service": "spider",
-    "workspaceFolder": "/usr/local/bin/node"
+    "workspaceFolder": "/usr/local/bin/node",
+    "customizations": {
+        "vscode": {
+            "extensions": [
+                "esbenp.prettier-vscode"
+            ]
+        }
+    }
 }

conf/node/controllers/__snapshots__/generate-indexes.test.js.snap

@@ -28,7 +28,8 @@ exports[`generateAgencyIndex should return the snapshots as a string 1`] = `
           <h1>Ministry of Justice Intranet Archive</h1>
           <h2 class="pb-2 border-bottom">test.generate.indexes - hmcts</h2>
           <ul class="list-group">
-            <li class="list-group-item">
+
+                <li class="list-group-item">
                   <a href="/test.generate.indexes/hmcts/2024-01-01/test.generate.indexes/index.html" target="_blank">2024-01-01</a>
                 </li>
           </ul>
@@ -66,9 +67,12 @@ exports[`generateRootIndex should return the agencies as a string 1`] = `
           <h1>Ministry of Justice Intranet Archive</h1>
           <h2 class="pb-2 border-bottom">test.generate.indexes</h2>
           <ul class="list-group">
-            <li class="list-group-item">
+
+                <li class="list-group-item">
                   <a href="/test.generate.indexes/hmcts/index.html" target="_blank">hmcts</a>
-                </li>,<li class="list-group-item">
+                </li>
+
+                <li class="list-group-item">
                   <a href="/test.generate.indexes/hq/index.html" target="_blank">hq</a>
                 </li>
           </ul>

conf/node/controllers/generate-indexes.js

@@ -26,11 +26,14 @@ export const generateRootIndex = async (bucket = s3BucketName, host) => {
           <h1>Ministry of Justice Intranet Archive</h1>
           <h2 class="pb-2 border-bottom">${host}</h2>
           <ul class="list-group">
-            ${agencies.map(
-              (agency) => `<li class="list-group-item">
+            ${agencies
+              .map(
+                (agency) => `
+                <li class="list-group-item">
                   <a href="/${host}/${agency}/index.html" target="_blank">${agency}</a>
                 </li>`,
-            )}
+              )
+              .join("\n")}
           </ul>
         </div>
       </main>
@@ -74,11 +77,14 @@ export const generateAgencyIndex = async (
           <h1>Ministry of Justice Intranet Archive</h1>
           <h2 class="pb-2 border-bottom">${host} - ${agency}</h2>
           <ul class="list-group">
-            ${snapshots.map(
-              (snapshot) => `<li class="list-group-item">
+            ${snapshots
+              .map(
+                (snapshot) => `
+                <li class="list-group-item">
                   <a href="/${host}/${agency}/${snapshot}/${host}/index.html" target="_blank">${snapshot}</a>
                 </li>`,
-            )}
+              )
+              .join("\n")}
           </ul>
         </div>
       </main>

conf/node/controllers/main.js

@@ -59,12 +59,23 @@ export const main = async ({ url, agency, depth }) => {
   await fs.rm(paths.fs, { recursive: true, force: true });
 
   // Generate and write content for the agency index file.
-  const agencyIndexHtml = await generateAgencyIndex(s3BucketName, url.host, agency);
-  await writeToS3(s3BucketName, `${url.host}/${agency}/index.html`, agencyIndexHtml);
+  const agencyIndexHtml = await generateAgencyIndex(
+    s3BucketName,
+    url.host,
+    agency,
+  );
+  await writeToS3(
+    s3BucketName,
+    `${url.host}/${agency}/index.html`,
+    agencyIndexHtml,
+    { cacheMaxAge: 600 },
+  );
 
   // Generate and write content for the root index file.
   const rootIndexHtml = await generateRootIndex(s3BucketName, url.host);
-  await writeToS3(s3BucketName, `index.html`, rootIndexHtml);
+  await writeToS3(s3BucketName, `index.html`, rootIndexHtml, {
+    cacheMaxAge: 600,
+  });
 
   console.log("Snapshot complete", { url: url.href, agency, depth });
 };

conf/node/controllers/s3.js

@@ -1,13 +1,17 @@
 import fs from "fs/promises";
-import { S3Client, ListObjectsV2Command, PutObjectCommand } from "@aws-sdk/client-s3";
+import {
+  S3Client,
+  ListObjectsV2Command,
+  PutObjectCommand,
+} from "@aws-sdk/client-s3";
 import { S3SyncClient } from "s3-sync-client";
 import mime from "mime-types";
 
 import {
   s3BucketName,
   s3Credentials as credentials,
   s3Region,
-  heartbeatEndpoint
+  heartbeatEndpoint,
 } from "../constants.js";
 
 /**
@@ -37,29 +41,32 @@ export const client = new S3Client(s3Options);
 
 /**
  * Create dummy /auth/heartbeat at bucket root, if it doesn't exist.
- * 
+ *
  * @param {string} bucket - The bucket name, defaults to the s3BucketName constant
  * @returns {Promise<void>}
- * 
+ *
  * @throws {Error}
  */
 
-export const createHeartbeat = async (bucket = s3BucketName, file = heartbeatEndpoint) => {
+export const createHeartbeat = async (
+  bucket = s3BucketName,
+  file = heartbeatEndpoint,
+) => {
   const objects = await client.send(
     new ListObjectsV2Command({
       Bucket: bucket,
       Prefix: file,
     }),
-  )
+  );
 
   if (!objects.Contents?.length) {
     const response = await client.send(
       new PutObjectCommand({
         Bucket: bucket,
         Key: file,
         Body: "OK",
-      })
-    )
+      }),
+    );
   }
 
   return;
@@ -140,7 +147,7 @@ export const checkAccess = async (bucket = s3BucketName) => {
  * @param {string} bucket - The bucket name, defaults to the s3BucketName constant
  * @param {string} host - The host to the intranet e.g. intranet.justice.gov.uk or dev.intranet.justice.gov.uk
  * @returns {Promise<string[]>}
- * 
+ *
  * @throws {Error}
  */
 
@@ -164,12 +171,12 @@ export const getAgenciesFromS3 = async (bucket = s3BucketName, host) => {
 
 /**
  * Get an agencies snapshots from S3
- * 
+ *
  * @param {string} bucket - The bucket name, defaults to the s3BucketName constant
  * @param {string} host - The host to the intranet e.g. intranet.justice.gov.uk or dev.intranet.justice.gov.uk
  * @param {string} agency - The agency to get snapshots for e.g. hq, hmcts etc.
  * @returns {Promise<string[]>}
- * 
+ *
  * @throws {Error}
  */
 
@@ -194,28 +201,40 @@ export const getSnapshotsFromS3 = async (
 
   const { CommonPrefixes } = await client.send(command);
 
-  return CommonPrefixes.map((folder) =>
-    folder.Prefix.replace(`${host}/${agency}/`, "").replace("/", ""),
+  return CommonPrefixes.map(({ Prefix }) =>
+    // Remove the host and agency from the Prefix
+    Prefix.replace(`${host}/${agency}/`, "").replace("/", ""),
+  ).filter((folder) =>
+    // Filter out any folders that are not in the format YYYY-MM-DD
+    /^\d{4}-\d{2}-\d{2}$/.test(folder),
   );
 };
 
 /**
  * Write string to an S3 file.
- * 
+ *
  * @param {string} bucket - The bucket name, defaults to the s3BucketName constant
  * @param {string} path - The path to write to
  * @param {string} content - The content to write
- * 
+ * @param {Object} [options] - The options object
+ * @param {number} [options.cacheMaxAge] - The cache max age in seconds - a simpler alternative to invalidating CloudFront cache
+ *
  * @returns {Promise<void>}
  */
 
-export const writeToS3 = async (bucket = s3BucketName, path, content) => {
+export const writeToS3 = async (
+  bucket = s3BucketName,
+  path,
+  content,
+  { cacheMaxAge } = {},
+) => {
   const command = new PutObjectCommand({
     Bucket: bucket,
     Key: path,
     Body: content,
     ContentType: mime.lookup(path) || "text/html",
+    ...(cacheMaxAge && { CacheControl: `max-age=${cacheMaxAge}` }),
   });
 
   await client.send(command);
-}
+};

conf/node/controllers/s3.test.js

@@ -230,7 +230,9 @@ describe("writeToS3", () => {
 
     await writeToS3(undefined, keyPlain, fileContent);
 
-    const resPlain = await client.send(new GetObjectCommand({...commandArgs, Key: keyPlain}));
+    const resPlain = await client.send(
+      new GetObjectCommand({ ...commandArgs, Key: keyPlain }),
+    );
 
     expect(resPlain.ContentType).toBe("text/plain");
 
@@ -239,10 +241,21 @@ describe("writeToS3", () => {
 
     await writeToS3(undefined, keyHtml, fileContent);
 
-    const resHtml = await client.send(new GetObjectCommand({...commandArgs, Key: keyHtml}));
+    const resHtml = await client.send(
+      new GetObjectCommand({ ...commandArgs, Key: keyHtml }),
+    );
 
     expect(resHtml.ContentType).toBe("text/html");
+  });
+
+  it("should write the correct cache control", async () => {
+    await writeToS3(undefined, commandArgs.Key, fileContent, {
+      cacheMaxAge: 60,
+    });
 
+    const res = await client.send(new GetObjectCommand(commandArgs));
+
+    expect(res.CacheControl).toBe("max-age=60");
   });
 });
 
@@ -312,6 +325,14 @@ describe("getSnapshotsFromS3", () => {
         Body: "test",
       }),
     );
+
+    await client.send(
+      new PutObjectCommand({
+        Bucket: s3BucketName,
+        Key: "test.get.snapshots/hq/non-date-folder/index.html",
+        Body: "test",
+      }),
+    );
   });
 
   afterAll(() => {