Docker ratelimit ci

.gitignore

@@ -74,3 +74,6 @@ website/i18n/*
 docs/packages/*/api
 
 .ts-test-config
+
+# CI test files
+images/*

package.json

@@ -20,6 +20,9 @@
         "build:pkg": "./scripts/build-pkg.sh",
         "build:watch": "yarn run build --watch",
         "bump": "ts-scripts bump",
+        "docker:limit": "./scripts/docker-limit-check.sh",
+        "docker:listImages": "ts-scripts images list",
+        "docker:saveImages": "ts-scripts images save",
         "docs": "ts-scripts docs",
         "k8s": "TEST_ELASTICSEARCH=true ELASTICSEARCH_PORT=9200 ts-scripts k8s-env",
         "k8s:kafka": "TEST_ELASTICSEARCH=true ELASTICSEARCH_PORT=9200 TEST_KAFKA=true KAFKA_PORT=9092 ts-scripts k8s-env",

packages/scripts/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@terascope/scripts",
     "displayName": "Scripts",
-    "version": "0.80.0",
+    "version": "0.81.0",
     "description": "A collection of terascope monorepo scripts",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/scripts#readme",
     "bugs": {

packages/scripts/src/cmds/images.ts

@@ -0,0 +1,33 @@
+import { CommandModule } from 'yargs';
+import { ImagesAction } from '../helpers/images/interfaces';
+import { images } from '../helpers/images';
+import { GlobalCMDOptions } from '../helpers/interfaces';
+
+interface Options {
+    action?: ImagesAction;
+}
+
+const cmd: CommandModule<GlobalCMDOptions, Options> = {
+    command: 'images <action>',
+    describe: 'Helper function related to docker images.',
+    builder(yargs) {
+        return yargs
+            .example('$0 images list', 'Get the list of docker images needed for a test.')
+            .example('$0 images save', 'Save the docker images needed for a test.')
+            .positional('action', {
+                description: 'The action to take',
+                choices: Object.values(ImagesAction),
+                coerce(arg): ImagesAction {
+                    return arg;
+                },
+            })
+            .requiresArg('action');
+    },
+    async handler(argv) {
+        if (argv.action) {
+            await images(argv.action);
+        }
+    },
+};
+
+export = cmd;

packages/scripts/src/helpers/config.ts

@@ -183,3 +183,7 @@ export const {
     K8S_VERSION = undefined,
     TERASLICE_IMAGE = undefined
 } = process.env;
+
+export const DOCKER_CACHE_PATH = '/tmp/docker_cache';
+export const DOCKER_IMAGES_PATH = './images';
+export const DOCKER_LIST_FILE_NAME = 'image-list.txt';

packages/scripts/src/helpers/images/index.ts

@@ -0,0 +1,79 @@
+import fse from 'fs-extra';
+import execa from 'execa';
+import path from 'node:path';
+import * as config from '../config';
+import { ImagesAction } from './interfaces';
+import signale from '../signale';
+
+export async function images(action: ImagesAction): Promise<void> {
+    if (action === ImagesAction.List) {
+        return createImageList(config.DOCKER_IMAGES_PATH);
+    }
+
+    if (action === ImagesAction.Save) {
+        return saveImages(config.DOCKER_CACHE_PATH, config.DOCKER_IMAGES_PATH);
+    }
+}
+
+/**
+ * Builds a list of all docker images needed for the teraslice CI pipeline
+ * @returns Record<string, string>
+ */
+async function createImageList(imagesTxtPath: string): Promise<void> {
+    const filePath = path.join(imagesTxtPath, `${config.DOCKER_LIST_FILE_NAME}`);
+
+    signale.info(`Creating Docker image list at ${filePath}`);
+
+    const list = 'terascope/node-base:18.19.1\n'
+               + 'terascope/node-base:20.11.1\n'
+               + 'terascope/node-base:22.2.0\n'
+               + `${config.ELASTICSEARCH_DOCKER_IMAGE}:6.8.6\n`
+               + `${config.ELASTICSEARCH_DOCKER_IMAGE}:7.9.3\n`
+               + `${config.OPENSEARCH_DOCKER_IMAGE}:1.3.10\n`
+               + `${config.OPENSEARCH_DOCKER_IMAGE}:2.8.0\n`
+               + `${config.KAFKA_DOCKER_IMAGE}:7.1.9\n`
+               + `${config.ZOOKEEPER_DOCKER_IMAGE}:7.1.9\n`
+               + `${config.MINIO_DOCKER_IMAGE}:RELEASE.2020-02-07T23-28-16Z\n`
+               + 'kindest/node:v1.30.0';
+
+    if (!fse.existsSync(imagesTxtPath)) {
+        await fse.emptyDir(imagesTxtPath);
+    }
+    fse.writeFileSync(filePath, list);
+}
+
+async function saveAndZip(imageName:string, imageSavePath: string) {
+    const fileName = imageName.replace(/[/:]/g, '_');
+    const filePath = path.join(imageSavePath, `${fileName}.tar`);
+    const command = `docker save ${imageName} | gzip > ${filePath}.gz`;
+    await execa.command(command, { shell: true });
+}
+
+async function saveImages(imageSavePath: string, imageTxtPath: string): Promise<void> {
+    try {
+        if (fse.existsSync(imageSavePath)) {
+            fse.rmSync(imageSavePath, { recursive: true, force: true });
+        }
+        fse.mkdirSync(imageSavePath);
+        const imagesString = fse.readFileSync(path.join(imageTxtPath, config.DOCKER_LIST_FILE_NAME), 'utf-8');
+        const imagesArray = imagesString.split('\n');
+        const pullPromises = imagesArray.map(async (imageName) => {
+            signale.info(`Pulling Docker image ${imageName}`);
+            await execa.command(`docker pull ${imageName}`);
+        });
+        await Promise.all(pullPromises);
+
+        for (let i = 0; i < imagesArray.length; i += 2) {
+            if (typeof imagesArray[i + 1] === 'string') {
+                await Promise.all([
+                    saveAndZip(imagesArray[i], imageSavePath),
+                    saveAndZip(imagesArray[i + 1], imageSavePath)
+                ]);
+            } else {
+                await saveAndZip(imagesArray[i], imageSavePath);
+            }
+        }
+    } catch (err) {
+        throw new Error(`Unable to pull and save images due to error: ${err}`);
+    }
+}

packages/scripts/src/helpers/images/interfaces.ts

@@ -0,0 +1,4 @@
+export enum ImagesAction {
+    List = 'list',
+    Save = 'save'
+}

packages/scripts/src/helpers/kind.ts

@@ -3,12 +3,12 @@ import os from 'os';
 import path from 'path';
 import execa from 'execa';
 import yaml from 'js-yaml';
-import { Logger, debugLogger } from '@terascope/utils';
+import { Logger, debugLogger, isCI } from '@terascope/utils';
 import type { V1Volume, V1VolumeMount } from '@kubernetes/client-node';
 import signale from './signale';
 import { getE2eK8sDir } from '../helpers/packages';
 import { KindCluster, TsVolumeSet } from './interfaces';
-import { TERASLICE_PORT } from './config';
+import { DOCKER_CACHE_PATH, TERASLICE_PORT } from './config';
 
 export class Kind {
     clusterName: string;
@@ -83,7 +83,19 @@ export class Kind {
         serviceName: string, serviceImage: string, version: string
     ): Promise<void> {
         try {
-            const subprocess = await execa.command(`kind load docker-image ${serviceImage}:${version} --name ${this.clusterName}`);
+            let subprocess;
+            if (isCI) {
+                // In CI we load images directly from the github docker image cache
+                // Without this we run out of disk space
+                const fileName = `${serviceImage}_${version}`.replace(/[/:]/g, '_');
+                const filePath = path.join(DOCKER_CACHE_PATH, `${fileName}.tar.gz`);
+                if (!fs.existsSync(filePath)) {
+                    throw new Error(`No file found at ${filePath}. Have you restored the cache?`);
+                }
+                subprocess = await execa.command(`kind --name ${this.clusterName} load image-archive <(gunzip -c ${filePath})`);
+            } else {
+                subprocess = await execa.command(`kind --name ${this.clusterName} load docker-image ${serviceImage}:${version}`);
+            }
             this.logger.debug(subprocess.stderr);
         } catch (err) {
             this.logger.debug(`The ${serviceName} docker image ${serviceImage}:${version} could not be loaded. It may not be present locally.`);

packages/scripts/src/helpers/scripts.ts

@@ -425,6 +425,30 @@ export async function dockerPush(image: string): Promise<void> {
     }
 }
 
+export async function loadThenDeleteImageFromCache(imageName: string): Promise<boolean> {
+    signale.time(`unzip and load ${imageName}`);
+    const fileName = imageName.trim().replace(/[/:]/g, '_');
+    const filePath = path.join(config.DOCKER_CACHE_PATH, `${fileName}.tar.gz`);
+    if (!fs.existsSync(filePath)) {
+        signale.error(`No file found at ${filePath}. Have you restored the cache?`);
+        return false;
+    }
+    const result = await execa.command(`gunzip -c ${filePath} | docker load`, { shell: true });
+    signale.info('Result: ', result);
+    if (result.exitCode !== 0) {
+        signale.error(`Error loading ${filePath} to docker`);
+        return false;
+    }
+    fs.rmSync(filePath);
+    signale.timeEnd(`unzip and load ${imageName}`);
+    return true;
+}
+
+export async function deleteDockerImageCache() {
+    signale.info(`Deleting Docker image cache at ${config.DOCKER_CACHE_PATH}`);
+    fse.removeSync(config.DOCKER_CACHE_PATH);
+}
+
 export async function pgrep(name: string): Promise<string> {
     const result = await exec({ cmd: 'ps', args: ['aux'] }, false);
     if (!result) {

packages/scripts/src/helpers/test-runner/index.ts

@@ -6,14 +6,16 @@ import {
     writePkgHeader, writeHeader, getRootDir,
     getRootInfo, getAvailableTestSuites, getDevDockerImage,
 } from '../misc';
-import { ensureServices, pullServices } from './services';
+import { ensureServices, loadOrPullServiceImages } from './services';
 import { PackageInfo } from '../interfaces';
 import { TestOptions } from './interfaces';
 import {
     runJest,
     dockerTag,
     isKindInstalled,
-    isKubectlInstalled
+    isKubectlInstalled,
+    loadThenDeleteImageFromCache,
+    deleteDockerImageCache
 } from '../scripts';
 import { Kind } from '../kind';
 import {
@@ -104,6 +106,11 @@ async function runTestSuite(
 ): Promise<void> {
     if (suite === 'e2e') return;
 
+    if (isCI) {
+        // load the services from cache in CI
+        await loadOrPullServiceImages(suite, options);
+    }
+
     const CHUNK_SIZE = options.debug ? 1 : MAX_PROJECTS_PER_BATCH;
 
     if (options.watch && pkgInfos.length > MAX_PROJECTS_PER_BATCH) {
@@ -229,9 +236,22 @@ async function runE2ETest(
     const rootInfo = getRootInfo();
     const e2eImage = `${rootInfo.name}:e2e-nodev${options.nodeVersion}`;
 
-    if (isCI && options.testPlatform === 'native') {
-        // pull the services first in CI
-        await pullServices(suite, options);
+    if (isCI) {
+        const promises = [];
+
+        // load the services from cache or pull if not found
+        promises.push(loadOrPullServiceImages(suite, options));
+
+        // load the base docker image
+        promises.push(loadThenDeleteImageFromCache(`terascope/node-base:${options.nodeVersion}`));
+
+        // load kind if using k8s
+        if (options.testPlatform === 'kubernetes' || options.testPlatform === 'kubernetesV2') {
+            promises.push(loadThenDeleteImageFromCache('kindest/node:v1.30.0'));
+        }
+
+        await Promise.all([...promises]);
+        await deleteDockerImageCache();
     }
 
     try {

packages/scripts/src/helpers/test-runner/services.ts

@@ -12,9 +12,10 @@ import {
     DockerRunOptions,
     getContainerInfo,
     dockerStop,
-    dockerPull,
     k8sStartService,
-    k8sStopService
+    k8sStopService,
+    loadThenDeleteImageFromCache,
+    dockerPull
 } from '../scripts';
 import { Kind } from '../kind';
 import { TestOptions } from './interfaces';
@@ -160,11 +161,12 @@ const services: Readonly<Record<Service, Readonly<DockerRunOptions>>> = {
     }
 };
 
-export async function pullServices(suite: string, options: TestOptions): Promise<void> {
+export async function loadOrPullServiceImages(suite: string, options: TestOptions): Promise<void> {
     const launchServices = getServicesForSuite(suite);
 
     try {
         const images: string[] = [];
+        const loadFailedList: string[] = [];
 
         if (launchServices.includes(Service.Elasticsearch)) {
             const image = `${config.ELASTICSEARCH_DOCKER_IMAGE}:${options.elasticsearchVersion}`;
@@ -206,12 +208,25 @@ export async function pullServices(suite: string, options: TestOptions): Promise
             images.push(image);
         }
 
-        await Promise.all(images.map(async (image) => {
-            const label = `docker pull ${image}`;
-            signale.time(label);
-            await dockerPull(image);
-            signale.timeEnd(label);
-        }));
+        if (fs.existsSync(config.DOCKER_CACHE_PATH)) {
+            await Promise.all(images.map(async (imageName) => {
+                const success = await loadThenDeleteImageFromCache(imageName);
+                if (!success) {
+                    loadFailedList.push(imageName);
+                }
+            }));
+        } else {
+            loadFailedList.push(...images);
+        }
+
+        if (loadFailedList.length > 0) {
+            await Promise.all(loadFailedList.map(async (image) => {
+                const label = `docker pull ${image}`;
+                signale.time(label);
+                await dockerPull(image);
+                signale.timeEnd(label);
+            }));
+        }
     } catch (err) {
         throw new ts.TSError(err, {
             message: `Failed to pull services for test suite "${suite}", ${err.message}`

packages/scripts/test/service-spec.ts

@@ -27,13 +27,6 @@ describe('services', () => {
         kindClusterName: 'default'
     };
 
-    describe('pullServices', () => {
-        it('should throw error if service image is invalid', async () => {
-            await expect(services.pullServices('_for_testing_', options))
-                .rejects.toThrowWithMessage(TSError, /w*Failed to pull services for test suite*\w/);
-        });
-    });
-
     describe('ensureServices', () => {
         it('should throw if service has an incorrect setting', async () => {
             await expect(services.ensureServices('_for_testing_', options))

scripts/docker-limit-check.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+###
+## This checks the docker rate limit
+###
+TOKEN=$(curl -Ss --user "$USER:$PASS" "https://auth.docker.io/token?service=registry.docker.io&scope=repository:ratelimitpreview/test:pull" | jq -r .token)
+result=$(curl -Ss --head -H "Authorization: Bearer $TOKEN" https://registry-1.docker.io/v2/ratelimitpreview/test/manifests/latest)
+
+echo "$result"