From 1a3d74a592b4037bcdb1378a54d42c3752fe8d11 Mon Sep 17 00:00:00 2001 From: EarthlingDavey <15802017+EarthlingDavey@users.noreply.github.com> Date: Fri, 6 Dec 2024 16:30:23 +0000 Subject: [PATCH] Add content types during S3 sync. --- conf/node/controllers/main.js | 2 +- conf/node/controllers/s3.js | 28 ++++++++++++++++++++++------ conf/node/controllers/s3.test.js | 32 ++++++++++++++++++++++++++++++-- conf/node/package-lock.json | 1 + conf/node/package.json | 1 + 5 files changed, 55 insertions(+), 9 deletions(-) diff --git a/conf/node/controllers/main.js b/conf/node/controllers/main.js index 36d910d..d93896a 100644 --- a/conf/node/controllers/main.js +++ b/conf/node/controllers/main.js @@ -21,7 +21,7 @@ import { sync } from "./s3.js"; export const main = async ({ url, agency, depth }) => { const paths = getSnapshotPaths({ host: url.host, agency }); - const { complete } = getHttrackProgress(paths.fs); + const { complete } = await getHttrackProgress(paths.fs); // If the snapshot is already complete, skip httrack if (!complete) { diff --git a/conf/node/controllers/s3.js b/conf/node/controllers/s3.js index 221b34e..8ddb550 100644 --- a/conf/node/controllers/s3.js +++ b/conf/node/controllers/s3.js @@ -1,6 +1,7 @@ import fs from "fs/promises"; import { S3Client, ListObjectsV2Command } from "@aws-sdk/client-s3"; import { S3SyncClient } from "s3-sync-client"; +import mime from "mime-types"; import { s3BucketName, @@ -36,18 +37,33 @@ export const client = new S3Client(s3Options); /** * S3 sync client * - * @type {S3SyncClient} - * * @see https://github.com/jeanbmar/s3-sync-client */ -const { sync } = new S3SyncClient({ client }); +const { sync: originalSync } = new S3SyncClient({ client }); -export { sync }; +/** + * Sync a local directory to an S3 bucket + * + * @param {string} source - The source directory + * @param {string} destination - The destination directory + * @param {?import('s3-sync-client/dist/commands/SyncCommand').SyncOptions} options - The sync options + * + * @returns {Promise} + */ + +export const sync = async (source, destination, options = {}) => { + // Set the content type for each file + options.commandInput = (input) => ({ + ContentType: mime.lookup(input.Key) || "text/html", + }); + + return originalSync(source, destination, options); +}; /** * Empty an S3 folder by using sync and deleting all files - * + * * @param {string} path - The path to empty */ @@ -63,7 +79,7 @@ export const s3EmptyDir = async (path) => { // Remove the empty directory await fs.rm(emptyDir, { recursive: true }); -} +}; /** * Check if the bucket is accessible by listing the root of the bucket diff --git a/conf/node/controllers/s3.test.js b/conf/node/controllers/s3.test.js index 7f597bc..0bf96c6 100644 --- a/conf/node/controllers/s3.test.js +++ b/conf/node/controllers/s3.test.js @@ -7,8 +7,8 @@ import { ListObjectsV2Command, } from "@aws-sdk/client-s3"; -import { s3Options, checkAccess, sync, s3EmptyDir } from "./s3"; import { s3BucketName } from "../constants.js"; +import { s3Options, checkAccess, sync, s3EmptyDir } from "./s3"; describe("checkAccess", () => { let client; @@ -55,11 +55,17 @@ describe("sync", () => { // Create a test file in /tmp/s3-test await fs.promises.writeFile("/tmp/s3-test/test.txt", fileContent); + + await fs.promises.writeFile( + "/tmp/s3-test/test.html", + "

Hello, World!

", + ); }); afterAll(async () => { - // Remove the test file + // Remove the test files await fs.promises.unlink("/tmp/s3-test/test.txt"); + await fs.promises.unlink("/tmp/s3-test/test.html"); await client.destroy(); }); @@ -75,6 +81,28 @@ describe("sync", () => { expect(bodyString).toBe(fileContent); }); + + it("should add content type to the destination files", async () => { + await sync("/tmp/s3-test", `s3://${s3BucketName}/test-types`); + + const object = await client.send( + new GetObjectCommand({ + Bucket: s3BucketName, + Key: "test-types/test.txt", + }), + ); + + expect(object.ContentType).toBe("text/plain"); + + const htmlObject = await client.send( + new GetObjectCommand({ + Bucket: s3BucketName, + Key: "test-types/test.html", + }), + ); + + expect(htmlObject.ContentType).toBe("text/html"); + }); }); describe("S3EmptyDir", () => { diff --git a/conf/node/package-lock.json b/conf/node/package-lock.json index 172d1a5..79ad551 100644 --- a/conf/node/package-lock.json +++ b/conf/node/package-lock.json @@ -13,6 +13,7 @@ "@aws-sdk/cloudfront-signer": "^3.696.0", "cors": "^2.8.5", "express": "^4.21.1", + "mime-types": "^2.1.35", "s3-sync-client": "^4.3.1" }, "devDependencies": { diff --git a/conf/node/package.json b/conf/node/package.json index 68a0fc3..a2fb450 100644 --- a/conf/node/package.json +++ b/conf/node/package.json @@ -24,6 +24,7 @@ "@aws-sdk/cloudfront-signer": "^3.696.0", "cors": "^2.8.5", "express": "^4.21.1", + "mime-types": "^2.1.35", "s3-sync-client": "^4.3.1" }, "devDependencies": {