Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add content types during S3 sync. #38

Merged
merged 1 commit into from
Dec 6, 2024
Merged

Add content types during S3 sync. #38

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion conf/node/controllers/main.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ import { sync } from "./s3.js";
export const main = async ({ url, agency, depth }) => {
const paths = getSnapshotPaths({ host: url.host, agency });

const { complete } = getHttrackProgress(paths.fs);
const { complete } = await getHttrackProgress(paths.fs);

// If the snapshot is already complete, skip httrack
if (!complete) {
Expand Down
28 changes: 22 additions & 6 deletions conf/node/controllers/s3.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import fs from "fs/promises";
import { S3Client, ListObjectsV2Command } from "@aws-sdk/client-s3";
import { S3SyncClient } from "s3-sync-client";
import mime from "mime-types";

import {
s3BucketName,
Expand Down Expand Up @@ -36,18 +37,33 @@ export const client = new S3Client(s3Options);
/**
* S3 sync client
*
* @type {S3SyncClient}
*
* @see https://github.com/jeanbmar/s3-sync-client
*/

const { sync } = new S3SyncClient({ client });
const { sync: originalSync } = new S3SyncClient({ client });

export { sync };
/**
* Sync a local directory to an S3 bucket
*
* @param {string} source - The source directory
* @param {string} destination - The destination directory
* @param {?import('s3-sync-client/dist/commands/SyncCommand').SyncOptions} options - The sync options
*
* @returns {Promise<import('s3-sync-client/dist/commands/SyncCommand').SyncCommandOutput>}
*/

export const sync = async (source, destination, options = {}) => {
// Set the content type for each file
options.commandInput = (input) => ({
ContentType: mime.lookup(input.Key) || "text/html",
});

return originalSync(source, destination, options);
};

/**
* Empty an S3 folder by using sync and deleting all files
*
*
* @param {string} path - The path to empty
*/

Expand All @@ -63,7 +79,7 @@ export const s3EmptyDir = async (path) => {

// Remove the empty directory
await fs.rm(emptyDir, { recursive: true });
}
};

/**
* Check if the bucket is accessible by listing the root of the bucket
Expand Down
32 changes: 30 additions & 2 deletions conf/node/controllers/s3.test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@ import {
ListObjectsV2Command,
} from "@aws-sdk/client-s3";

import { s3Options, checkAccess, sync, s3EmptyDir } from "./s3";
import { s3BucketName } from "../constants.js";
import { s3Options, checkAccess, sync, s3EmptyDir } from "./s3";

describe("checkAccess", () => {
let client;
Expand Down Expand Up @@ -55,11 +55,17 @@ describe("sync", () => {

// Create a test file in /tmp/s3-test
await fs.promises.writeFile("/tmp/s3-test/test.txt", fileContent);

await fs.promises.writeFile(
"/tmp/s3-test/test.html",
"<html><body><h1>Hello, World!</h1></body></html>",
);
});

afterAll(async () => {
// Remove the test file
// Remove the test files
await fs.promises.unlink("/tmp/s3-test/test.txt");
await fs.promises.unlink("/tmp/s3-test/test.html");

await client.destroy();
});
Expand All @@ -75,6 +81,28 @@ describe("sync", () => {

expect(bodyString).toBe(fileContent);
});

it("should add content type to the destination files", async () => {
await sync("/tmp/s3-test", `s3://${s3BucketName}/test-types`);

const object = await client.send(
new GetObjectCommand({
Bucket: s3BucketName,
Key: "test-types/test.txt",
}),
);

expect(object.ContentType).toBe("text/plain");

const htmlObject = await client.send(
new GetObjectCommand({
Bucket: s3BucketName,
Key: "test-types/test.html",
}),
);

expect(htmlObject.ContentType).toBe("text/html");
});
});

describe("S3EmptyDir", () => {
Expand Down
1 change: 1 addition & 0 deletions conf/node/package-lock.json
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions conf/node/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
"@aws-sdk/cloudfront-signer": "^3.696.0",
"cors": "^2.8.5",
"express": "^4.21.1",
"mime-types": "^2.1.35",
"s3-sync-client": "^4.3.1"
},
"devDependencies": {
Expand Down
Loading