diff --git a/CHANGELOG.md b/CHANGELOG.md index 58146b5..c4d1765 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog +## [2.6.1] - 2023-11-12 + +### Added + +- Option for optimistic remote registry cache checking. i.e. treat redirects as layer existing + +### Fixed + +- Follow redirects when checking for existing layers in remote registry + + ## [2.6.0] - 2023-10-11 ### Added diff --git a/README.md b/README.md index e72e631..e38bda2 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,7 @@ Options: --fromRegistry Optional: URL of registry to pull base image from - Default: https://registry-1.docker.io/v2/ --fromToken Optional: Authentication token for from registry --toRegistry Optional: URL of registry to push base image to - Default: https://registry-1.docker.io/v2/ + --optimisticToRegistryCheck Optional: Treat redirects as layer existing in remote registry. Potentially unsafe, but could save bandwidth. --toToken Optional: Authentication token for target registry --toTar Optional: Export to tar file --registry Optional: Convenience argument for setting both from and to registry diff --git a/package-lock.json b/package-lock.json index b3d5a65..b75ada7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "containerify", - "version": "2.6.0", + "version": "2.6.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "containerify", - "version": "2.6.0", + "version": "2.6.1", "license": "Apache-2.0", "dependencies": { "commander": "^11.0.0", diff --git a/package.json b/package.json index 805f394..95b4974 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "containerify", - "version": "2.6.0", + "version": "2.6.1", "description": "Build node.js docker images without docker", "main": "./lib/cli.js", "scripts": { diff --git a/src/cli.ts b/src/cli.ts index d070b97..7559f8b 100755 --- a/src/cli.ts +++ b/src/cli.ts @@ -27,6 +27,7 @@ const possibleArgs = { "--fromToken ": "Optional: Authentication token for from registry", "--toRegistry ": "Optional: URL of registry to push base image to - Default: https://registry-1.docker.io/v2/", + "--optimisticToRegistryCheck": "Treat redirects as layer existing in remote registry. Potentially unsafe, but can save bandwidth.", "--toToken ": "Optional: Authentication token for target registry", "--toTar ": "Optional: Export to tar file", "--toDocker": "Optional: Export to local docker registry", @@ -278,7 +279,7 @@ async function run(options: Options) { await tarExporter.saveToTar(todir, tmpdir, options.toTar, [options.toImage], options); } if (options.toRegistry) { - const toRegistry = createRegistry(options.toRegistry, options.toToken ?? ""); + const toRegistry = createRegistry(options.toRegistry, options.toToken ?? "", options.optimisticToRegistryCheck); await toRegistry.upload(options.toImage, todir); } logger.debug("Deleting " + tmpdir + " ..."); diff --git a/src/registry.ts b/src/registry.ts index 8b276a4..fa8bc3f 100644 --- a/src/registry.ts +++ b/src/registry.ts @@ -1,20 +1,20 @@ import * as https from "https"; import * as http from "http"; import * as URL from "url"; +import * as fss from "fs"; import { promises as fs } from "fs"; import * as path from "path"; import * as fse from "fs-extra"; -import * as fss from "fs"; import * as fileutil from "./fileutil"; import logger from "./logger"; -import { Config, Image, Index, Layer, Manifest, IndexManifest, PartialManifestConfig, Platform } from "./types"; +import { Config, Image, Index, IndexManifest, Layer, Manifest, PartialManifestConfig, Platform } from "./types"; import { DockerV2, OCI } from "./MIMETypes"; import { getLayerTypeFileEnding } from "./utils"; type Headers = Record; -const redirectCodes = [307, 303, 302]; +const redirectCodes = [308, 307, 303, 302, 301]; function request(options: https.RequestOptions, callback: (res: http.IncomingMessage) => void) { return (options.protocol == "https:" ? https : http).request(options, (res) => { @@ -122,7 +122,11 @@ function buildHeaders(accept: string, auth: string) { return headers; } -function headOk(url: string, headers: Headers): Promise { +function headOk(url: string, headers: Headers, optimisticCheck = false, depth =0): Promise { + if (depth >= 5) { + logger.info("Followed five redirects, assuming layer does not exist"); + return new Promise((resolve) => resolve(false)); + } return new Promise((resolve, reject) => { logger.debug(`HEAD ${url}`); const options: http.RequestOptions = URL.parse(url); @@ -130,8 +134,20 @@ function headOk(url: string, headers: Headers): Promise { options.method = "HEAD"; request(options, (res) => { logger.debug(`HEAD ${url}`, res.statusCode); + // Not found if (res.statusCode == 404) return resolve(false); + // OK if (res.statusCode == 200) return resolve(true); + // Redirected + if (redirectCodes.includes(res.statusCode ?? 0) && res.headers.location) { + if (optimisticCheck) return resolve(true) + return resolve(headOk(res.headers.location, headers, optimisticCheck, ++depth)); + } + // Unauthorized + // Possibly related to https://gitlab.com/gitlab-org/gitlab/-/issues/23132 + if (res.statusCode == 401) { + return resolve(false); + } reject(toError(res)); }).end(); }); @@ -171,12 +187,12 @@ function uploadContent( }); } -export function createRegistry(registryBaseUrl: string, token: string) { +export function createRegistry(registryBaseUrl: string, token: string, optimisticToRegistryCheck = false) { const auth = token.startsWith("Basic ") ? token : "Bearer " + token; async function exists(image: Image, layer: Layer) { const url = `${registryBaseUrl}${image.path}/blobs/${layer.digest}`; - return await headOk(url, buildHeaders(layer.mediaType, auth)); + return await headOk(url, buildHeaders(layer.mediaType, auth), optimisticToRegistryCheck, 0); } async function uploadLayerContent(uploadUrl: string, layer: Layer, dir: string) { @@ -360,6 +376,7 @@ export function createDockerRegistry(auth?: string) { ); return resp.token; } + async function download(imageStr: string, folder: string, platform: Platform, cacheFolder?: string) { const image = parseImage(imageStr); if (!auth) auth = await getToken(image); diff --git a/src/types.ts b/src/types.ts index 8829e20..ad186d3 100644 --- a/src/types.ts +++ b/src/types.ts @@ -73,6 +73,7 @@ export type Options = { fromRegistry?: string; fromToken?: string; toRegistry?: string; + optimisticToRegistryCheck?: boolean; toToken?: string; toTar?: string; toDocker?: boolean; diff --git a/src/version.ts b/src/version.ts index 181428f..31972bc 100644 --- a/src/version.ts +++ b/src/version.ts @@ -1 +1 @@ -export const VERSION = "2.6.0"; +export const VERSION = "2.6.1";