From 7b085147d47cc77a5f73637bee0cb028f586bdf2 Mon Sep 17 00:00:00 2001 From: Peter Luitjens <43619525+busma13@users.noreply.github.com> Date: Thu, 11 Jul 2024 15:16:36 -0700 Subject: [PATCH] Docker ratelimit ci (#3679) This PR makes the following changes: - Add `images` cmd to scripts - `images list` creates a list of images that CI will need - `images save` will use the list to pull and save those images - Update `test.yml` workflow to check for a docker image cache, create one if needed, then use the cache to load images on all jobs. - Update `test-runner` to load images from cache if in CI - Update `scripts` from 0.80.0 to 0.81.0 ref: #3676 --------- Co-authored-by: Joseph Soto Co-authored-by: sotojn --- .github/workflows/test.yml | 241 +++++++++++++++++- .gitignore | 3 + package.json | 3 + packages/scripts/package.json | 2 +- packages/scripts/src/cmds/images.ts | 33 +++ packages/scripts/src/helpers/config.ts | 4 + packages/scripts/src/helpers/images/index.ts | 79 ++++++ .../scripts/src/helpers/images/interfaces.ts | 4 + packages/scripts/src/helpers/kind.ts | 18 +- packages/scripts/src/helpers/scripts.ts | 24 ++ .../scripts/src/helpers/test-runner/index.ts | 30 ++- .../src/helpers/test-runner/services.ts | 33 ++- packages/scripts/test/service-spec.ts | 7 - scripts/docker-limit-check.sh | 8 + 14 files changed, 459 insertions(+), 30 deletions(-) create mode 100644 packages/scripts/src/cmds/images.ts create mode 100644 packages/scripts/src/helpers/images/index.ts create mode 100644 packages/scripts/src/helpers/images/interfaces.ts create mode 100755 scripts/docker-limit-check.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4b703e827d1..76e5a721364 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -28,6 +28,67 @@ jobs: - name: Sync codebase run: yarn sync --verify + cache-docker-images: + runs-on: ubuntu-latest + needs: verify-build + steps: + - name: Check out code + uses: actions/checkout@v4 + + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: 18.19.1 + cache: 'yarn' + + # we login to docker to avoid docker pull limit rates + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Check docker.hub limit start of job + env: + USER: ${{ secrets.DOCKER_USERNAME }} + PASS: ${{ secrets.DOCKER_PASSWORD }} + run: yarn docker:limit + + - name: Install and build packages + run: yarn setup + env: + YARN_SETUP_ARGS: "--prod=false --silent" + + - name: Create Docker Image List + run: | + yarn docker:listImages + cat ./images/image-list.txt + + - name: Restore Docker image cache + id: docker-cache + uses: actions/cache@v4 + with: + path: /tmp/docker_cache + key: docker-images-${{ hashFiles('./images/image-list.txt') }} + lookup-only: true + + - name: Pull and save Docker images + if: ${{steps.docker-cache.outputs.cache-hit != 'true'}} + run: yarn docker:saveImages + + - name: Update Docker image cache + if: ${{steps.docker-cache.outputs.cache-hit != 'true'}} + uses: actions/cache@v4 + with: + path: /tmp/docker_cache + key: docker-images-${{ hashFiles('./images/image-list.txt') }} + + - name: Check docker.hub limit end of job + env: + USER: ${{ secrets.DOCKER_USERNAME }} + PASS: ${{ secrets.DOCKER_PASSWORD }} + run: npm run docker:limit + linux-unit-tests: runs-on: ubuntu-latest strategy: @@ -76,9 +137,9 @@ jobs: env: NODE_OPTIONS: '--experimental-vm-modules' - teraslice-elasticsearch-tests: runs-on: ubuntu-latest + needs: cache-docker-images strategy: # opensearch is finiky, keep testing others if it fails fail-fast: false @@ -102,17 +163,42 @@ jobs: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} + - name: Check docker.hub limit start + env: + USER: ${{ secrets.DOCKER_USERNAME }} + PASS: ${{ secrets.DOCKER_PASSWORD }} + run: npm run docker:limit + - name: Install and build packages run: yarn setup env: YARN_SETUP_ARGS: "--prod=false --silent" + - name: Create Docker Image List + run: | + yarn docker:listImages + cat ./images/image-list.txt + + - name: Restore Docker image cache + id: docker-cache + uses: actions/cache@v4 + with: + path: /tmp/docker_cache + key: docker-images-${{ hashFiles('./images/image-list.txt') }} + - name: Test ${{ matrix.search-version }} run: yarn --silent test:${{ matrix.search-version }} working-directory: ./packages/teraslice + - name: Check docker.hub limit after Test + env: + USER: ${{ secrets.DOCKER_USERNAME }} + PASS: ${{ secrets.DOCKER_PASSWORD }} + run: npm run docker:limit + elasticsearch-store-tests: runs-on: ubuntu-latest + needs: cache-docker-images strategy: # opensearch is finiky, keep testing others if it fails fail-fast: false @@ -136,17 +222,42 @@ jobs: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} + - name: Check docker.hub limit start + env: + USER: ${{ secrets.DOCKER_USERNAME }} + PASS: ${{ secrets.DOCKER_PASSWORD }} + run: npm run docker:limit + - name: Install and build packages run: yarn setup env: YARN_SETUP_ARGS: "--prod=false --silent" + - name: Create Docker Image List + run: | + yarn docker:listImages + cat ./images/image-list.txt + + - name: Restore Docker image cache + id: docker-cache + uses: actions/cache@v4 + with: + path: /tmp/docker_cache + key: docker-images-${{ hashFiles('./images/image-list.txt') }} + - name: Test ${{ matrix.search-version }} run: yarn --silent test:${{ matrix.search-version }} working-directory: ./packages/elasticsearch-store + - name: Check docker.hub limit end of job + env: + USER: ${{ secrets.DOCKER_USERNAME }} + PASS: ${{ secrets.DOCKER_PASSWORD }} + run: npm run docker:limit + elasticsearch-api-tests: runs-on: ubuntu-latest + needs: cache-docker-images strategy: # opensearch is finiky, keep testing others if it fails fail-fast: false @@ -170,17 +281,42 @@ jobs: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} + - name: Check docker.hub limit start + env: + USER: ${{ secrets.DOCKER_USERNAME }} + PASS: ${{ secrets.DOCKER_PASSWORD }} + run: npm run docker:limit + - name: Install and build packages run: yarn setup env: YARN_SETUP_ARGS: "--prod=false --silent" + - name: Create Docker Image List + run: | + yarn docker:listImages + cat ./images/image-list.txt + + - name: Restore Docker image cache + id: docker-cache + uses: actions/cache@v4 + with: + path: /tmp/docker_cache + key: docker-images-${{ hashFiles('./images/image-list.txt') }} + - name: Test ${{ matrix.search-version }} run: yarn --silent test:${{ matrix.search-version }} working-directory: ./packages/elasticsearch-api + - name: Check docker.hub limit end of job + env: + USER: ${{ secrets.DOCKER_USERNAME }} + PASS: ${{ secrets.DOCKER_PASSWORD }} + run: npm run docker:limit + e2e-tests: runs-on: ubuntu-latest + needs: cache-docker-images strategy: # opensearch is finiky, keep testing others if it fails fail-fast: false @@ -209,6 +345,18 @@ jobs: env: YARN_SETUP_ARGS: "--prod=false --silent" + - name: Create Docker Image List + run: | + yarn docker:listImages + cat ./images/image-list.txt + + - name: Restore Docker image cache + id: docker-cache + uses: actions/cache@v4 + with: + path: /tmp/docker_cache + key: docker-images-${{ hashFiles('./images/image-list.txt') }} + - name: Compile e2e code run: yarn build working-directory: ./e2e @@ -217,8 +365,15 @@ jobs: run: yarn --silent test:${{ matrix.search-version }} --node-version ${{ matrix.node-version }} working-directory: ./e2e + - name: Check docker.hub limit end of job + env: + USER: ${{ secrets.DOCKER_USERNAME }} + PASS: ${{ secrets.DOCKER_PASSWORD }} + run: npm run docker:limit + e2e-k8s-tests: runs-on: ubuntu-latest + needs: cache-docker-images strategy: # opensearch is finiky, keep testing others if it fails fail-fast: false @@ -246,12 +401,24 @@ jobs: env: YARN_SETUP_ARGS: "--prod=false --silent" + - name: Create Docker Image List + run: | + yarn docker:listImages + cat ./images/image-list.txt + + - name: Restore Docker image cache + id: docker-cache + uses: actions/cache@v4 + with: + path: /tmp/docker_cache + key: docker-images-${{ hashFiles('./images/image-list.txt') }} + - name: Compile e2e code run: yarn build working-directory: ./e2e - name: Install Kind and Kubectl - uses: helm/kind-action@v1.8.0 + uses: helm/kind-action@v1.10.0 with: install_only: "true" @@ -259,8 +426,15 @@ jobs: run: yarn --silent test:k8s --node-version ${{ matrix.node-version }} working-directory: ./e2e + - name: Check docker.hub limit end of job + env: + USER: ${{ secrets.DOCKER_USERNAME }} + PASS: ${{ secrets.DOCKER_PASSWORD }} + run: npm run docker:limit + e2e-k8s-v2-tests: runs-on: ubuntu-latest + needs: cache-docker-images strategy: # opensearch is finiky, keep testing others if it fails fail-fast: false @@ -268,17 +442,17 @@ jobs: node-version: [18.19.1, 20.11.1] steps: - name: Check out code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Node ${{ matrix.node-version }} - uses: actions/setup-node@v3 + uses: actions/setup-node@v4 with: node-version: ${{ matrix.node-version }} cache: 'yarn' # we login to docker to avoid docker pull limit rates - name: Login to Docker Hub - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} @@ -288,6 +462,18 @@ jobs: env: YARN_SETUP_ARGS: "--prod=false --silent" + - name: Create Docker Image List + run: | + yarn docker:listImages + cat ./images/image-list.txt + + - name: Restore Docker image cache + id: docker-cache + uses: actions/cache@v4 + with: + path: /tmp/docker_cache + key: docker-images-${{ hashFiles('./images/image-list.txt') }} + - name: Compile e2e code run: yarn build working-directory: ./e2e @@ -301,8 +487,15 @@ jobs: run: yarn --silent test:k8sV2 --node-version ${{ matrix.node-version }} working-directory: ./e2e + - name: Check docker.hub limit end of job + env: + USER: ${{ secrets.DOCKER_USERNAME }} + PASS: ${{ secrets.DOCKER_PASSWORD }} + run: npm run docker:limit + e2e-external-storage-tests: runs-on: ubuntu-latest + needs: cache-docker-images strategy: # opensearch is finiky, keep testing others if it fails fail-fast: false @@ -330,6 +523,18 @@ jobs: env: YARN_SETUP_ARGS: "--prod=false --silent" + - name: Create Docker Image List + run: | + yarn docker:listImages + cat ./images/image-list.txt + + - name: Restore Docker image cache + id: docker-cache + uses: actions/cache@v4 + with: + path: /tmp/docker_cache + key: docker-images-${{ hashFiles('./images/image-list.txt') }} + - name: Compile e2e code run: yarn build working-directory: ./e2e @@ -337,8 +542,16 @@ jobs: - name: Test external Asset Storage opensearch1 run: yarn --silent test:s3AssetStorage --node-version ${{ matrix.node-version }} working-directory: ./e2e + + - name: Check docker.hub limit end of job + env: + USER: ${{ secrets.DOCKER_USERNAME }} + PASS: ${{ secrets.DOCKER_PASSWORD }} + run: npm run docker:limit + e2e-external-storage-tests-encrypted: runs-on: ubuntu-latest + needs: cache-docker-images strategy: # opensearch is finiky, keep testing others if it fails fail-fast: false @@ -366,6 +579,18 @@ jobs: env: YARN_SETUP_ARGS: "--prod=false --silent" + - name: Create Docker Image List + run: | + yarn docker:listImages + cat ./images/image-list.txt + + - name: Restore Docker image cache + id: docker-cache + uses: actions/cache@v4 + with: + path: /tmp/docker_cache + key: docker-images-${{ hashFiles('./images/image-list.txt') }} + - name: Install mkcert run: curl -JLO "https://dl.filippo.io/mkcert/latest?for=linux/amd64" && sudo chmod 777 mkcert-v*-linux-amd64 && sudo cp mkcert-v*-linux-amd64 /usr/local/bin/mkcert @@ -385,3 +610,9 @@ jobs: - name: Test external Asset Storage opensearch1 run: ENCRYPT_MINIO=true yarn --silent test:s3AssetStorage --node-version ${{ matrix.node-version }} working-directory: ./e2e + + - name: Check docker.hub limit end of job + env: + USER: ${{ secrets.DOCKER_USERNAME }} + PASS: ${{ secrets.DOCKER_PASSWORD }} + run: npm run docker:limit diff --git a/.gitignore b/.gitignore index 070401289c6..465c0269200 100644 --- a/.gitignore +++ b/.gitignore @@ -74,3 +74,6 @@ website/i18n/* docs/packages/*/api .ts-test-config + +# CI test files +images/* diff --git a/package.json b/package.json index bc6e808ccbe..464b27d7e87 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,9 @@ "build:pkg": "./scripts/build-pkg.sh", "build:watch": "yarn run build --watch", "bump": "ts-scripts bump", + "docker:limit": "./scripts/docker-limit-check.sh", + "docker:listImages": "ts-scripts images list", + "docker:saveImages": "ts-scripts images save", "docs": "ts-scripts docs", "k8s": "TEST_ELASTICSEARCH=true ELASTICSEARCH_PORT=9200 ts-scripts k8s-env", "k8s:kafka": "TEST_ELASTICSEARCH=true ELASTICSEARCH_PORT=9200 TEST_KAFKA=true KAFKA_PORT=9092 ts-scripts k8s-env", diff --git a/packages/scripts/package.json b/packages/scripts/package.json index be1c5b6a15e..9d70bc69ac4 100644 --- a/packages/scripts/package.json +++ b/packages/scripts/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/scripts", "displayName": "Scripts", - "version": "0.80.0", + "version": "0.81.0", "description": "A collection of terascope monorepo scripts", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/scripts#readme", "bugs": { diff --git a/packages/scripts/src/cmds/images.ts b/packages/scripts/src/cmds/images.ts new file mode 100644 index 00000000000..c852c507910 --- /dev/null +++ b/packages/scripts/src/cmds/images.ts @@ -0,0 +1,33 @@ +import { CommandModule } from 'yargs'; +import { ImagesAction } from '../helpers/images/interfaces'; +import { images } from '../helpers/images'; +import { GlobalCMDOptions } from '../helpers/interfaces'; + +interface Options { + action?: ImagesAction; +} + +const cmd: CommandModule = { + command: 'images ', + describe: 'Helper function related to docker images.', + builder(yargs) { + return yargs + .example('$0 images list', 'Get the list of docker images needed for a test.') + .example('$0 images save', 'Save the docker images needed for a test.') + .positional('action', { + description: 'The action to take', + choices: Object.values(ImagesAction), + coerce(arg): ImagesAction { + return arg; + }, + }) + .requiresArg('action'); + }, + async handler(argv) { + if (argv.action) { + await images(argv.action); + } + }, +}; + +export = cmd; diff --git a/packages/scripts/src/helpers/config.ts b/packages/scripts/src/helpers/config.ts index 37e24562f94..b56a2afcaa0 100644 --- a/packages/scripts/src/helpers/config.ts +++ b/packages/scripts/src/helpers/config.ts @@ -183,3 +183,7 @@ export const { K8S_VERSION = undefined, TERASLICE_IMAGE = undefined } = process.env; + +export const DOCKER_CACHE_PATH = '/tmp/docker_cache'; +export const DOCKER_IMAGES_PATH = './images'; +export const DOCKER_LIST_FILE_NAME = 'image-list.txt'; diff --git a/packages/scripts/src/helpers/images/index.ts b/packages/scripts/src/helpers/images/index.ts new file mode 100644 index 00000000000..177d60246e2 --- /dev/null +++ b/packages/scripts/src/helpers/images/index.ts @@ -0,0 +1,79 @@ +import fse from 'fs-extra'; +import execa from 'execa'; +import path from 'node:path'; +import * as config from '../config'; +import { ImagesAction } from './interfaces'; +import signale from '../signale'; + +export async function images(action: ImagesAction): Promise { + if (action === ImagesAction.List) { + return createImageList(config.DOCKER_IMAGES_PATH); + } + + if (action === ImagesAction.Save) { + return saveImages(config.DOCKER_CACHE_PATH, config.DOCKER_IMAGES_PATH); + } +} + +/** + * Builds a list of all docker images needed for the teraslice CI pipeline + * @returns Record + */ +async function createImageList(imagesTxtPath: string): Promise { + const filePath = path.join(imagesTxtPath, `${config.DOCKER_LIST_FILE_NAME}`); + + signale.info(`Creating Docker image list at ${filePath}`); + + const list = 'terascope/node-base:18.19.1\n' + + 'terascope/node-base:20.11.1\n' + + 'terascope/node-base:22.2.0\n' + + `${config.ELASTICSEARCH_DOCKER_IMAGE}:6.8.6\n` + + `${config.ELASTICSEARCH_DOCKER_IMAGE}:7.9.3\n` + + `${config.OPENSEARCH_DOCKER_IMAGE}:1.3.10\n` + + `${config.OPENSEARCH_DOCKER_IMAGE}:2.8.0\n` + + `${config.KAFKA_DOCKER_IMAGE}:7.1.9\n` + + `${config.ZOOKEEPER_DOCKER_IMAGE}:7.1.9\n` + + `${config.MINIO_DOCKER_IMAGE}:RELEASE.2020-02-07T23-28-16Z\n` + + 'kindest/node:v1.30.0'; + + if (!fse.existsSync(imagesTxtPath)) { + await fse.emptyDir(imagesTxtPath); + } + fse.writeFileSync(filePath, list); +} + +async function saveAndZip(imageName:string, imageSavePath: string) { + const fileName = imageName.replace(/[/:]/g, '_'); + const filePath = path.join(imageSavePath, `${fileName}.tar`); + const command = `docker save ${imageName} | gzip > ${filePath}.gz`; + await execa.command(command, { shell: true }); +} + +async function saveImages(imageSavePath: string, imageTxtPath: string): Promise { + try { + if (fse.existsSync(imageSavePath)) { + fse.rmSync(imageSavePath, { recursive: true, force: true }); + } + fse.mkdirSync(imageSavePath); + const imagesString = fse.readFileSync(path.join(imageTxtPath, config.DOCKER_LIST_FILE_NAME), 'utf-8'); + const imagesArray = imagesString.split('\n'); + const pullPromises = imagesArray.map(async (imageName) => { + signale.info(`Pulling Docker image ${imageName}`); + await execa.command(`docker pull ${imageName}`); + }); + await Promise.all(pullPromises); + + for (let i = 0; i < imagesArray.length; i += 2) { + if (typeof imagesArray[i + 1] === 'string') { + await Promise.all([ + saveAndZip(imagesArray[i], imageSavePath), + saveAndZip(imagesArray[i + 1], imageSavePath) + ]); + } else { + await saveAndZip(imagesArray[i], imageSavePath); + } + } + } catch (err) { + throw new Error(`Unable to pull and save images due to error: ${err}`); + } +} diff --git a/packages/scripts/src/helpers/images/interfaces.ts b/packages/scripts/src/helpers/images/interfaces.ts new file mode 100644 index 00000000000..1a38a6654ed --- /dev/null +++ b/packages/scripts/src/helpers/images/interfaces.ts @@ -0,0 +1,4 @@ +export enum ImagesAction { + List = 'list', + Save = 'save' +} diff --git a/packages/scripts/src/helpers/kind.ts b/packages/scripts/src/helpers/kind.ts index 37733e1fe32..e8c8936f631 100644 --- a/packages/scripts/src/helpers/kind.ts +++ b/packages/scripts/src/helpers/kind.ts @@ -3,12 +3,12 @@ import os from 'os'; import path from 'path'; import execa from 'execa'; import yaml from 'js-yaml'; -import { Logger, debugLogger } from '@terascope/utils'; +import { Logger, debugLogger, isCI } from '@terascope/utils'; import type { V1Volume, V1VolumeMount } from '@kubernetes/client-node'; import signale from './signale'; import { getE2eK8sDir } from '../helpers/packages'; import { KindCluster, TsVolumeSet } from './interfaces'; -import { TERASLICE_PORT } from './config'; +import { DOCKER_CACHE_PATH, TERASLICE_PORT } from './config'; export class Kind { clusterName: string; @@ -83,7 +83,19 @@ export class Kind { serviceName: string, serviceImage: string, version: string ): Promise { try { - const subprocess = await execa.command(`kind load docker-image ${serviceImage}:${version} --name ${this.clusterName}`); + let subprocess; + if (isCI) { + // In CI we load images directly from the github docker image cache + // Without this we run out of disk space + const fileName = `${serviceImage}_${version}`.replace(/[/:]/g, '_'); + const filePath = path.join(DOCKER_CACHE_PATH, `${fileName}.tar.gz`); + if (!fs.existsSync(filePath)) { + throw new Error(`No file found at ${filePath}. Have you restored the cache?`); + } + subprocess = await execa.command(`kind --name ${this.clusterName} load image-archive <(gunzip -c ${filePath})`); + } else { + subprocess = await execa.command(`kind --name ${this.clusterName} load docker-image ${serviceImage}:${version}`); + } this.logger.debug(subprocess.stderr); } catch (err) { this.logger.debug(`The ${serviceName} docker image ${serviceImage}:${version} could not be loaded. It may not be present locally.`); diff --git a/packages/scripts/src/helpers/scripts.ts b/packages/scripts/src/helpers/scripts.ts index bbc577156f6..89d1763e015 100644 --- a/packages/scripts/src/helpers/scripts.ts +++ b/packages/scripts/src/helpers/scripts.ts @@ -425,6 +425,30 @@ export async function dockerPush(image: string): Promise { } } +export async function loadThenDeleteImageFromCache(imageName: string): Promise { + signale.time(`unzip and load ${imageName}`); + const fileName = imageName.trim().replace(/[/:]/g, '_'); + const filePath = path.join(config.DOCKER_CACHE_PATH, `${fileName}.tar.gz`); + if (!fs.existsSync(filePath)) { + signale.error(`No file found at ${filePath}. Have you restored the cache?`); + return false; + } + const result = await execa.command(`gunzip -c ${filePath} | docker load`, { shell: true }); + signale.info('Result: ', result); + if (result.exitCode !== 0) { + signale.error(`Error loading ${filePath} to docker`); + return false; + } + fs.rmSync(filePath); + signale.timeEnd(`unzip and load ${imageName}`); + return true; +} + +export async function deleteDockerImageCache() { + signale.info(`Deleting Docker image cache at ${config.DOCKER_CACHE_PATH}`); + fse.removeSync(config.DOCKER_CACHE_PATH); +} + export async function pgrep(name: string): Promise { const result = await exec({ cmd: 'ps', args: ['aux'] }, false); if (!result) { diff --git a/packages/scripts/src/helpers/test-runner/index.ts b/packages/scripts/src/helpers/test-runner/index.ts index 8fd45c18587..fd20c48a203 100644 --- a/packages/scripts/src/helpers/test-runner/index.ts +++ b/packages/scripts/src/helpers/test-runner/index.ts @@ -6,14 +6,16 @@ import { writePkgHeader, writeHeader, getRootDir, getRootInfo, getAvailableTestSuites, getDevDockerImage, } from '../misc'; -import { ensureServices, pullServices } from './services'; +import { ensureServices, loadOrPullServiceImages } from './services'; import { PackageInfo } from '../interfaces'; import { TestOptions } from './interfaces'; import { runJest, dockerTag, isKindInstalled, - isKubectlInstalled + isKubectlInstalled, + loadThenDeleteImageFromCache, + deleteDockerImageCache } from '../scripts'; import { Kind } from '../kind'; import { @@ -104,6 +106,11 @@ async function runTestSuite( ): Promise { if (suite === 'e2e') return; + if (isCI) { + // load the services from cache in CI + await loadOrPullServiceImages(suite, options); + } + const CHUNK_SIZE = options.debug ? 1 : MAX_PROJECTS_PER_BATCH; if (options.watch && pkgInfos.length > MAX_PROJECTS_PER_BATCH) { @@ -229,9 +236,22 @@ async function runE2ETest( const rootInfo = getRootInfo(); const e2eImage = `${rootInfo.name}:e2e-nodev${options.nodeVersion}`; - if (isCI && options.testPlatform === 'native') { - // pull the services first in CI - await pullServices(suite, options); + if (isCI) { + const promises = []; + + // load the services from cache or pull if not found + promises.push(loadOrPullServiceImages(suite, options)); + + // load the base docker image + promises.push(loadThenDeleteImageFromCache(`terascope/node-base:${options.nodeVersion}`)); + + // load kind if using k8s + if (options.testPlatform === 'kubernetes' || options.testPlatform === 'kubernetesV2') { + promises.push(loadThenDeleteImageFromCache('kindest/node:v1.30.0')); + } + + await Promise.all([...promises]); + await deleteDockerImageCache(); } try { diff --git a/packages/scripts/src/helpers/test-runner/services.ts b/packages/scripts/src/helpers/test-runner/services.ts index 32d8763ecb0..bebfb8b8aff 100644 --- a/packages/scripts/src/helpers/test-runner/services.ts +++ b/packages/scripts/src/helpers/test-runner/services.ts @@ -12,9 +12,10 @@ import { DockerRunOptions, getContainerInfo, dockerStop, - dockerPull, k8sStartService, - k8sStopService + k8sStopService, + loadThenDeleteImageFromCache, + dockerPull } from '../scripts'; import { Kind } from '../kind'; import { TestOptions } from './interfaces'; @@ -160,11 +161,12 @@ const services: Readonly>> = { } }; -export async function pullServices(suite: string, options: TestOptions): Promise { +export async function loadOrPullServiceImages(suite: string, options: TestOptions): Promise { const launchServices = getServicesForSuite(suite); try { const images: string[] = []; + const loadFailedList: string[] = []; if (launchServices.includes(Service.Elasticsearch)) { const image = `${config.ELASTICSEARCH_DOCKER_IMAGE}:${options.elasticsearchVersion}`; @@ -206,12 +208,25 @@ export async function pullServices(suite: string, options: TestOptions): Promise images.push(image); } - await Promise.all(images.map(async (image) => { - const label = `docker pull ${image}`; - signale.time(label); - await dockerPull(image); - signale.timeEnd(label); - })); + if (fs.existsSync(config.DOCKER_CACHE_PATH)) { + await Promise.all(images.map(async (imageName) => { + const success = await loadThenDeleteImageFromCache(imageName); + if (!success) { + loadFailedList.push(imageName); + } + })); + } else { + loadFailedList.push(...images); + } + + if (loadFailedList.length > 0) { + await Promise.all(loadFailedList.map(async (image) => { + const label = `docker pull ${image}`; + signale.time(label); + await dockerPull(image); + signale.timeEnd(label); + })); + } } catch (err) { throw new ts.TSError(err, { message: `Failed to pull services for test suite "${suite}", ${err.message}` diff --git a/packages/scripts/test/service-spec.ts b/packages/scripts/test/service-spec.ts index 7365d034d2a..3cb500bde65 100644 --- a/packages/scripts/test/service-spec.ts +++ b/packages/scripts/test/service-spec.ts @@ -27,13 +27,6 @@ describe('services', () => { kindClusterName: 'default' }; - describe('pullServices', () => { - it('should throw error if service image is invalid', async () => { - await expect(services.pullServices('_for_testing_', options)) - .rejects.toThrowWithMessage(TSError, /w*Failed to pull services for test suite*\w/); - }); - }); - describe('ensureServices', () => { it('should throw if service has an incorrect setting', async () => { await expect(services.ensureServices('_for_testing_', options)) diff --git a/scripts/docker-limit-check.sh b/scripts/docker-limit-check.sh new file mode 100755 index 00000000000..7d92a2bdf2c --- /dev/null +++ b/scripts/docker-limit-check.sh @@ -0,0 +1,8 @@ +#!/bin/bash +### +## This checks the docker rate limit +### +TOKEN=$(curl -Ss --user "$USER:$PASS" "https://auth.docker.io/token?service=registry.docker.io&scope=repository:ratelimitpreview/test:pull" | jq -r .token) +result=$(curl -Ss --head -H "Authorization: Bearer $TOKEN" https://registry-1.docker.io/v2/ratelimitpreview/test/manifests/latest) + +echo "$result"