diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9b4f6455713..a0bec208fa5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -75,14 +75,7 @@ jobs: - name: Pull and save Docker images if: ${{steps.docker-cache.outputs.cache-hit != 'true'}} - # run: yarn docker:pullAndSaveImages - run: | - mkdir -p /tmp/docker_cache - while IFS= read -r image; do - echo "pulling image: $image" - docker pull "$image" - docker save "$image" | gzip > "/tmp/docker_cache/$(echo $image | tr '/:' '__').tar.gz" - done < ./images/image-list.txt + run: yarn docker:saveImages - name: Update Docker image cache if: ${{steps.docker-cache.outputs.cache-hit != 'true'}} @@ -208,13 +201,6 @@ jobs: gzip -dc "$tar_gz" | docker load done - - name: Pull Docker images if cache is not hit - if: ${{steps.docker-cache.outputs.cache-hit != 'true'}} - run: | - while IFS= read -r image; do - docker pull "$image" - done < image-list.txt - - name: Test ${{ matrix.search-version }} run: yarn --silent test:${{ matrix.search-version }} working-directory: ./packages/teraslice diff --git a/.gitignore b/.gitignore index 070401289c6..fb161f1b3d9 100644 --- a/.gitignore +++ b/.gitignore @@ -74,3 +74,6 @@ website/i18n/* docs/packages/*/api .ts-test-config + +# CI test files +images diff --git a/package.json b/package.json index 837cdb6c3a2..02f5e29debd 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,7 @@ "docker:limit": "./scripts/docker-limit-check.sh", "docker:listImages": "ts-scripts images list", "docker:loadImages": "ts-scripts images load", + "docker:saveImages": "ts-scripts images save", "docs": "ts-scripts docs", "k8s": "TEST_ELASTICSEARCH=true ELASTICSEARCH_PORT=9200 ts-scripts k8s-env", "k8s:kafka": "TEST_ELASTICSEARCH=true ELASTICSEARCH_PORT=9200 TEST_KAFKA=true KAFKA_PORT=9092 ts-scripts k8s-env", diff --git a/packages/scripts/src/helpers/images/index.ts b/packages/scripts/src/helpers/images/index.ts index e768540374f..a35d7014c97 100644 --- a/packages/scripts/src/helpers/images/index.ts +++ b/packages/scripts/src/helpers/images/index.ts @@ -1,6 +1,5 @@ import fse from 'fs-extra'; -import execa = require('execa'); -// import { Gzip } from 'node:zlib'; +import execa from 'execa'; import path from 'node:path'; import * as config from '../config'; import { ImagesAction } from './interfaces'; @@ -15,7 +14,7 @@ export async function images(action: ImagesAction): Promise { } if (action === ImagesAction.Save) { - // return saveImages('/tmp/docker_cache/'); + return saveImages('/tmp/docker_cache/', './images'); } } @@ -33,7 +32,7 @@ async function createImageList(imagesPath: string): Promise { + `${config.OPENSEARCH_DOCKER_IMAGE}:2.8.0\n` + `${config.KAFKA_DOCKER_IMAGE}:7.1.9\n` + `${config.ZOOKEEPER_DOCKER_IMAGE}:7.1.9\n` - + `${config.MINIO_DOCKER_IMAGE}:RELEASE.2020-02-07T23-28-16Z\n`; + + `${config.MINIO_DOCKER_IMAGE}:RELEASE.2020-02-07T23-28-16Z`; if (!fse.existsSync(imagesPath)) { await fse.emptyDir(imagesPath); } @@ -53,19 +52,37 @@ async function loadImages(imagesPath: string): Promise { } } -// async function saveImages(imageSavePath: string): Promise { +async function saveAndZip(imageName:string, imageSavePath: string) { + const fileName = imageName.replace(/[/:]/g, '_'); + const filePath = path.join(imageSavePath, `${fileName}.tar`); + const command = `docker save ${imageName} | gzip > ${filePath}.gz`; + await execa.command(command, { shell: true }); +} -// } +async function saveImages(imageSavePath: string, imageTxtPath: string): Promise { + try { + if (fse.existsSync(imageSavePath)) { + fse.rmSync(imageSavePath, { recursive: true, force: true }); + } + fse.mkdirSync(imageSavePath); + const imagesString = fse.readFileSync(path.join(imageTxtPath, 'image-list.txt'), 'utf-8'); + const imagesArray = imagesString.split('\n'); + const pullPromises = imagesArray.map(async (imageName) => { + await execa.command(`docker pull ${imageName}`); + }); + await Promise.all(pullPromises); -// const list = { -// baseDockerImageNode18: 'terascope/node-base:18.19.1', -// baseDockerImageNode20: 'terascope/node-base:20.11.1', -// baseDockerImageNode22: 'terascope/node-base:22.2.0', -// elasticsearch6: `${config.ELASTICSEARCH_DOCKER_IMAGE}:6.8.6`, -// elasticsearch7: `${config.ELASTICSEARCH_DOCKER_IMAGE}:7.9.3`, -// opensearch: `${config.OPENSEARCH_DOCKER_IMAGE}:1.3.10`, -// opensearch2: `${config.OPENSEARCH_DOCKER_IMAGE}:2.8.0`, -// kafka: `${config.KAFKA_DOCKER_IMAGE}:3.1`, -// zookeeper: `${config.ZOOKEEPER_DOCKER_IMAGE}:3.1`, -// minio: `${config.MINIO_DOCKER_IMAGE}:RELEASE.2020-02-07T23-28-16Z`, -// }; + for (let i = 0; i < imagesArray.length; i += 2) { + if (typeof imagesArray[i + 1] === 'string') { + await Promise.all([ + saveAndZip(imagesArray[i], imageSavePath), + saveAndZip(imagesArray[i + 1], imageSavePath) + ]); + } else { + await saveAndZip(imagesArray[i], imageSavePath); + } + } + } catch (err) { + throw new Error(`Unable to pull and save images due to error: ${err}`); + } +}