feat: Follow redirects when checking if a layer exists in the remote …

…registry (eoftedal#28)

Also adding an option for optimistic registry checking where redirects
are treated as the layer existing.

CHANGELOG.md

@@ -1,5 +1,16 @@
 # Changelog
 
+## [2.6.1] - 2023-11-12
+
+### Added
+
+- Option for optimistic remote registry cache checking. i.e. treat redirects as layer existing
+
+### Fixed
+
+- Follow redirects when checking for existing layers in remote registry
+
+
 ## [2.6.0] - 2023-10-11
 
 ### Added

README.md

@@ -43,6 +43,7 @@ Options:
   --fromRegistry <registry url>  Optional: URL of registry to pull base image from - Default: https://registry-1.docker.io/v2/
   --fromToken <token>            Optional: Authentication token for from registry
   --toRegistry <registry url>    Optional: URL of registry to push base image to - Default: https://registry-1.docker.io/v2/
+  --optimisticToRegistryCheck    Optional: Treat redirects as layer existing in remote registry. Potentially unsafe, but could save bandwidth.
   --toToken <token>              Optional: Authentication token for target registry
   --toTar <path>                 Optional: Export to tar file
   --registry <path>              Optional: Convenience argument for setting both from and to registry

package.json

@@ -1,6 +1,6 @@
 {
 	"name": "containerify",
-	"version": "2.6.0",
+	"version": "2.6.1",
 	"description": "Build node.js docker images without docker",
 	"main": "./lib/cli.js",
 	"scripts": {

src/cli.ts

@@ -27,6 +27,7 @@ const possibleArgs = {
 	"--fromToken <token>": "Optional: Authentication token for from registry",
 	"--toRegistry <registry url>":
 		"Optional: URL of registry to push base image to - Default: https://registry-1.docker.io/v2/",
+	"--optimisticToRegistryCheck": "Treat redirects as layer existing in remote registry. Potentially unsafe, but can save bandwidth.",
 	"--toToken <token>": "Optional: Authentication token for target registry",
 	"--toTar <path>": "Optional: Export to tar file",
 	"--toDocker": "Optional: Export to local docker registry",
@@ -278,7 +279,7 @@ async function run(options: Options) {
 		await tarExporter.saveToTar(todir, tmpdir, options.toTar, [options.toImage], options);
 	}
 	if (options.toRegistry) {
-		const toRegistry = createRegistry(options.toRegistry, options.toToken ?? "");
+		const toRegistry = createRegistry(options.toRegistry, options.toToken ?? "", options.optimisticToRegistryCheck);
 		await toRegistry.upload(options.toImage, todir);
 	}
 	logger.debug("Deleting " + tmpdir + " ...");

src/registry.ts

@@ -1,20 +1,20 @@
 import * as https from "https";
 import * as http from "http";
 import * as URL from "url";
+import * as fss from "fs";
 import { promises as fs } from "fs";
 import * as path from "path";
 import * as fse from "fs-extra";
-import * as fss from "fs";
 
 import * as fileutil from "./fileutil";
 import logger from "./logger";
-import { Config, Image, Index, Layer, Manifest, IndexManifest, PartialManifestConfig, Platform } from "./types";
+import { Config, Image, Index, IndexManifest, Layer, Manifest, PartialManifestConfig, Platform } from "./types";
 import { DockerV2, OCI } from "./MIMETypes";
 import { getLayerTypeFileEnding } from "./utils";
 
 type Headers = Record<string, string>;
 
-const redirectCodes = [307, 303, 302];
+const redirectCodes = [308, 307, 303, 302, 301];
 
 function request(options: https.RequestOptions, callback: (res: http.IncomingMessage) => void) {
 	return (options.protocol == "https:" ? https : http).request(options, (res) => {
@@ -122,16 +122,32 @@ function buildHeaders(accept: string, auth: string) {
 	return headers;
 }
 
-function headOk(url: string, headers: Headers): Promise<boolean> {
+function headOk(url: string, headers: Headers, optimisticCheck = false, depth =0): Promise<boolean> {
+	if (depth >= 5) {
+		logger.info("Followed five redirects, assuming layer does not exist");
+		return new Promise((resolve) => resolve(false));
+	}
 	return new Promise((resolve, reject) => {
 		logger.debug(`HEAD ${url}`);
 		const options: http.RequestOptions = URL.parse(url);
 		options.headers = headers;
 		options.method = "HEAD";
 		request(options, (res) => {
 			logger.debug(`HEAD ${url}`, res.statusCode);
+			// Not found
 			if (res.statusCode == 404) return resolve(false);
+			// OK
 			if (res.statusCode == 200) return resolve(true);
+			// Redirected
+			if (redirectCodes.includes(res.statusCode ?? 0)  && res.headers.location) {
+				if (optimisticCheck) return resolve(true)
+				return resolve(headOk(res.headers.location, headers, optimisticCheck, ++depth));
+			}
+			// Unauthorized
+			// Possibly related to https://gitlab.com/gitlab-org/gitlab/-/issues/23132
+			if (res.statusCode == 401) {
+				return resolve(false);
+			}
 			reject(toError(res));
 		}).end();
 	});
@@ -171,12 +187,12 @@ function uploadContent(
 	});
 }
 
-export function createRegistry(registryBaseUrl: string, token: string) {
+export function createRegistry(registryBaseUrl: string, token: string, optimisticToRegistryCheck = false) {
 	const auth = token.startsWith("Basic ") ? token : "Bearer " + token;
 
 	async function exists(image: Image, layer: Layer) {
 		const url = `${registryBaseUrl}${image.path}/blobs/${layer.digest}`;
-		return await headOk(url, buildHeaders(layer.mediaType, auth));
+		return await headOk(url, buildHeaders(layer.mediaType, auth), optimisticToRegistryCheck, 0);
 	}
 
 	async function uploadLayerContent(uploadUrl: string, layer: Layer, dir: string) {
@@ -360,6 +376,7 @@ export function createDockerRegistry(auth?: string) {
 		);
 		return resp.token;
 	}
+
 	async function download(imageStr: string, folder: string, platform: Platform, cacheFolder?: string) {
 		const image = parseImage(imageStr);
 		if (!auth) auth = await getToken(image);

src/types.ts

@@ -73,6 +73,7 @@ export type Options = {
 	fromRegistry?: string;
 	fromToken?: string;
 	toRegistry?: string;
+	optimisticToRegistryCheck?: boolean;
 	toToken?: string;
 	toTar?: string;
 	toDocker?: boolean;

src/version.ts

@@ -1 +1 @@
-export const VERSION = "2.6.0";
+export const VERSION = "2.6.1";