Further improvements to CI test pipeline (#3682)

This PR makes the following changes: - Run `verify-build` and `cache-docker-images` jobs in parallel in `test.yml` workflow. E2e tests now await both. - Remove node 20 from e2e k8s jobs matrices - Move `saveAndZip` function from `images` to `scripts` - Add JSDoc style comments to functions - Add tests for `images` command and `loadOrPullServiceImages` function - Load kind image at proper time and load service images directly into kind correctly. - Change error handling on `kindLoadImages` so errors aren't swallowed.
terascope · Jul 16, 2024 · 060b336 · 060b336
1 parent 3084b64
commit 060b336
Show file tree

Hide file tree

Showing 10 changed files with 157 additions and 64 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -30,7 +30,6 @@ jobs:
 
   cache-docker-images:
     runs-on: ubuntu-latest
-    needs: verify-build
     steps:
     - name: Check out code
       uses: actions/checkout@v4
@@ -78,7 +77,7 @@ jobs:
 
     - name: Update Docker image cache
       if: ${{steps.docker-cache.outputs.cache-hit != 'true'}}
-      uses: actions/cache@v4
+      uses: actions/cache/save@v4
       with:
         path: /tmp/docker_cache
         key: docker-images-${{ hashFiles('./images/image-list.txt') }}
@@ -139,7 +138,7 @@ jobs:
 
   teraslice-elasticsearch-tests:
     runs-on: ubuntu-latest
-    needs: cache-docker-images
+    needs: [verify-build, cache-docker-images]
     strategy:
       # opensearch is finiky, keep testing others if it fails
       fail-fast: false
@@ -198,7 +197,7 @@ jobs:
 
   elasticsearch-store-tests:
     runs-on: ubuntu-latest
-    needs: cache-docker-images
+    needs: [verify-build, cache-docker-images]
     strategy:
       # opensearch is finiky, keep testing others if it fails
       fail-fast: false
@@ -257,7 +256,7 @@ jobs:
 
   elasticsearch-api-tests:
     runs-on: ubuntu-latest
-    needs: cache-docker-images
+    needs: [verify-build, cache-docker-images]
     strategy:
       # opensearch is finiky, keep testing others if it fails
       fail-fast: false
@@ -316,7 +315,7 @@ jobs:
 
   e2e-tests:
     runs-on: ubuntu-latest
-    needs: cache-docker-images
+    needs: [verify-build, cache-docker-images]
     strategy:
       # opensearch is finiky, keep testing others if it fails
       fail-fast: false
@@ -340,6 +339,12 @@ jobs:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
+      - name: Check docker.hub limit start
+        env:
+          USER: ${{ secrets.DOCKER_USERNAME }}
+          PASS: ${{ secrets.DOCKER_PASSWORD }}
+        run: npm run docker:limit
+
       - name: Install and build packages
         run: yarn setup
         env:
@@ -373,12 +378,12 @@ jobs:
 
   e2e-k8s-tests:
     runs-on: ubuntu-latest
-    needs: cache-docker-images
+    needs: [verify-build, cache-docker-images]
     strategy:
       # opensearch is finiky, keep testing others if it fails
       fail-fast: false
       matrix:
-        node-version: [18.19.1, 20.11.1, 22.2.0]
+        node-version: [18.19.1, 22.2.0]
     steps:
       - name: Check out code
         uses: actions/checkout@v4
@@ -396,6 +401,12 @@ jobs:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
+      - name: Check docker.hub limit start
+        env:
+          USER: ${{ secrets.DOCKER_USERNAME }}
+          PASS: ${{ secrets.DOCKER_PASSWORD }}
+        run: npm run docker:limit
+
       - name: Install and build packages
         run: yarn setup
         env:
@@ -434,12 +445,12 @@ jobs:
 
   e2e-k8s-v2-tests:
     runs-on: ubuntu-latest
-    needs: cache-docker-images
+    needs: [verify-build, cache-docker-images]
     strategy:
       # opensearch is finiky, keep testing others if it fails
       fail-fast: false
       matrix:
-        node-version: [18.19.1, 20.11.1]
+        node-version: [18.19.1, 22.2.0]
     steps:
       - name: Check out code
         uses: actions/checkout@v4
@@ -457,6 +468,12 @@ jobs:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
+      - name: Check docker.hub limit start
+        env:
+          USER: ${{ secrets.DOCKER_USERNAME }}
+          PASS: ${{ secrets.DOCKER_PASSWORD }}
+        run: npm run docker:limit
+
       - name: Install and build packages
         run: yarn setup
         env:
@@ -479,7 +496,7 @@ jobs:
         working-directory: ./e2e
 
       - name: Install Kind and Kubectl
-        uses: helm/kind-action@v1.8.0
+        uses: helm/kind-action@v1.10.0
         with:
           install_only: "true"
 
@@ -495,7 +512,7 @@ jobs:
 
   e2e-external-storage-tests:
     runs-on: ubuntu-latest
-    needs: cache-docker-images
+    needs: [verify-build, cache-docker-images]
     strategy:
       # opensearch is finiky, keep testing others if it fails
       fail-fast: false
@@ -518,6 +535,12 @@ jobs:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
+      - name: Check docker.hub limit start
+        env:
+          USER: ${{ secrets.DOCKER_USERNAME }}
+          PASS: ${{ secrets.DOCKER_PASSWORD }}
+        run: npm run docker:limit
+
       - name: Install and build packages
         run: yarn setup
         env:
@@ -551,7 +574,7 @@ jobs:
 
   e2e-external-storage-tests-encrypted:
     runs-on: ubuntu-latest
-    needs: cache-docker-images
+    needs: [verify-build, cache-docker-images]
     strategy:
       # opensearch is finiky, keep testing others if it fails
       fail-fast: false
@@ -574,6 +597,12 @@ jobs:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
+      - name: Check docker.hub limit start
+        env:
+          USER: ${{ secrets.DOCKER_USERNAME }}
+          PASS: ${{ secrets.DOCKER_PASSWORD }}
+        run: npm run docker:limit
+
       - name: Install and build packages
         run: yarn setup
         env:

diff --git a/packages/scripts/package.json b/packages/scripts/package.json
@@ -1,7 +1,7 @@
 {
     "name": "@terascope/scripts",
     "displayName": "Scripts",
-    "version": "0.81.0",
+    "version": "0.81.1",
     "description": "A collection of terascope monorepo scripts",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/scripts#readme",
     "bugs": {

diff --git a/packages/scripts/src/helpers/config.ts b/packages/scripts/src/helpers/config.ts
@@ -184,6 +184,6 @@ export const {
     TERASLICE_IMAGE = undefined
 } = process.env;
 
-export const DOCKER_CACHE_PATH = '/tmp/docker_cache';
 export const DOCKER_IMAGES_PATH = './images';
-export const DOCKER_LIST_FILE_NAME = 'image-list.txt';
+export const DOCKER_IMAGE_LIST_PATH = `${DOCKER_IMAGES_PATH}/image-list.txt`;
+export const DOCKER_CACHE_PATH = '/tmp/docker_cache';
diff --git a/packages/scripts/src/helpers/images/index.ts b/packages/scripts/src/helpers/images/index.ts
@@ -1,28 +1,25 @@
 import fse from 'fs-extra';
-import execa from 'execa';
-import path from 'node:path';
 import * as config from '../config';
 import { ImagesAction } from './interfaces';
 import signale from '../signale';
+import { dockerPull, saveAndZip } from '../scripts';
 
 export async function images(action: ImagesAction): Promise<void> {
     if (action === ImagesAction.List) {
-        return createImageList(config.DOCKER_IMAGES_PATH);
+        return createImageList();
     }
 
     if (action === ImagesAction.Save) {
-        return saveImages(config.DOCKER_CACHE_PATH, config.DOCKER_IMAGES_PATH);
+        return saveImages();
     }
 }
 
 /**
  * Builds a list of all docker images needed for the teraslice CI pipeline
- * @returns Record<string, string>
+ * @returns Promise<void>
  */
-async function createImageList(imagesTxtPath: string): Promise<void> {
-    const filePath = path.join(imagesTxtPath, `${config.DOCKER_LIST_FILE_NAME}`);
-
-    signale.info(`Creating Docker image list at ${filePath}`);
+export async function createImageList(): Promise<void> {
+    signale.info(`Creating Docker image list at ${config.DOCKER_IMAGE_LIST_PATH}`);
 
     const list = 'terascope/node-base:18.19.1\n'
                + 'terascope/node-base:20.11.1\n'
@@ -36,41 +33,39 @@ async function createImageList(imagesTxtPath: string): Promise<void> {
                + `${config.MINIO_DOCKER_IMAGE}:RELEASE.2020-02-07T23-28-16Z\n`
                + 'kindest/node:v1.30.0';
 
-    if (!fse.existsSync(imagesTxtPath)) {
-        await fse.emptyDir(imagesTxtPath);
+    if (!fse.existsSync(config.DOCKER_IMAGES_PATH)) {
+        await fse.emptyDir(config.DOCKER_IMAGES_PATH);
     }
-    fse.writeFileSync(filePath, list);
-}
-
-async function saveAndZip(imageName:string, imageSavePath: string) {
-    const fileName = imageName.replace(/[/:]/g, '_');
-    const filePath = path.join(imageSavePath, `${fileName}.tar`);
-    const command = `docker save ${imageName} | gzip > ${filePath}.gz`;
-    await execa.command(command, { shell: true });
+    fse.writeFileSync(config.DOCKER_IMAGE_LIST_PATH, list);
 }
 
-async function saveImages(imageSavePath: string, imageTxtPath: string): Promise<void> {
+/**
+ * Pulls all docker images from the list at config.DOCKER_IMAGE_LIST_PATH
+ * then saves and zips them to config.DOCKER_CACHE_PATH in batches of 2.
+ * @returns Promise<void>
+ */
+export async function saveImages(): Promise<void> {
     try {
-        if (fse.existsSync(imageSavePath)) {
-            fse.rmSync(imageSavePath, { recursive: true, force: true });
+        if (fse.existsSync(config.DOCKER_CACHE_PATH)) {
+            fse.rmSync(config.DOCKER_CACHE_PATH, { recursive: true, force: true });
         }
-        fse.mkdirSync(imageSavePath);
-        const imagesString = fse.readFileSync(path.join(imageTxtPath, config.DOCKER_LIST_FILE_NAME), 'utf-8');
+        fse.mkdirSync(config.DOCKER_CACHE_PATH);
+        const imagesString = fse.readFileSync(config.DOCKER_IMAGE_LIST_PATH, 'utf-8');
         const imagesArray = imagesString.split('\n');
         const pullPromises = imagesArray.map(async (imageName) => {
-            signale.info(`Pulling Docker image ${imageName}`);
-            await execa.command(`docker pull ${imageName}`);
+            signale.info(`Pulling Docker image: ${imageName}`);
+            await dockerPull(imageName);
         });
         await Promise.all(pullPromises);
 
         for (let i = 0; i < imagesArray.length; i += 2) {
             if (typeof imagesArray[i + 1] === 'string') {
                 await Promise.all([
-                    saveAndZip(imagesArray[i], imageSavePath),
-                    saveAndZip(imagesArray[i + 1], imageSavePath)
+                    saveAndZip(imagesArray[i], config.DOCKER_CACHE_PATH),
+                    saveAndZip(imagesArray[i + 1], config.DOCKER_CACHE_PATH)
                 ]);
             } else {
-                await saveAndZip(imagesArray[i], imageSavePath);
+                await saveAndZip(imagesArray[i], config.DOCKER_CACHE_PATH);
             }
         }
     } catch (err) {

diff --git a/packages/scripts/src/helpers/kind.ts b/packages/scripts/src/helpers/kind.ts
@@ -82,23 +82,27 @@ export class Kind {
     async loadServiceImage(
         serviceName: string, serviceImage: string, version: string
     ): Promise<void> {
+        let subprocess;
         try {
-            let subprocess;
             if (isCI) {
                 // In CI we load images directly from the github docker image cache
-                // Without this we run out of disk space
                 const fileName = `${serviceImage}_${version}`.replace(/[/:]/g, '_');
                 const filePath = path.join(DOCKER_CACHE_PATH, `${fileName}.tar.gz`);
+                const tarPath = path.join(DOCKER_CACHE_PATH, `${fileName}.tar`);
                 if (!fs.existsSync(filePath)) {
                     throw new Error(`No file found at ${filePath}. Have you restored the cache?`);
                 }
-                subprocess = await execa.command(`kind --name ${this.clusterName} load image-archive <(gunzip -c ${filePath})`);
+                subprocess = await execa.command(`gunzip -d ${filePath}`);
+                signale.info(`${subprocess.command}: successful`);
+                subprocess = await execa.command(`kind load --name ${this.clusterName} image-archive ${tarPath}`);
+                fs.rmSync(tarPath);
             } else {
-                subprocess = await execa.command(`kind --name ${this.clusterName} load docker-image ${serviceImage}:${version}`);
+                subprocess = await execa.command(`kind load --name ${this.clusterName} docker-image ${serviceImage}:${version}`);
             }
-            this.logger.debug(subprocess.stderr);
+            signale.info(`${subprocess.command}: successful`);
         } catch (err) {
-            this.logger.debug(`The ${serviceName} docker image ${serviceImage}:${version} could not be loaded. It may not be present locally.`);
+            signale.info(`The ${serviceName} docker image ${serviceImage}:${version} could not be loaded. It may not be present locally.`);
+            signale.info('Error: ', err);
         }
     }
 

diff --git a/packages/scripts/src/helpers/scripts.ts b/packages/scripts/src/helpers/scripts.ts
@@ -425,6 +425,12 @@ export async function dockerPush(image: string): Promise<void> {
     }
 }
 
+/**
+ * Unzips and loads a Docker image from a Docker cache
+ * If successful the image will be deleted from the cache
+ * @param {string} imageName Name of the image to load
+ * @returns {Promise<boolean>} Whether or not the image loaded successfully
+ */
 export async function loadThenDeleteImageFromCache(imageName: string): Promise<boolean> {
     signale.time(`unzip and load ${imageName}`);
     const fileName = imageName.trim().replace(/[/:]/g, '_');
@@ -465,6 +471,20 @@ export async function pgrep(name: string): Promise<string> {
     return '';
 }
 
+/**
+ * Save a docker image as a tar.gz to a local directory
+ * @param {string} imageName Name of image to pull and save
+ * @param {string} imageSavePath Location where image will be saved and compressed.
+ * @returns void
+ */
+export async function saveAndZip(imageName:string, imageSavePath: string) {
+    signale.info(`Saving Docker image: ${imageName}`);
+    const fileName = imageName.replace(/[/:]/g, '_');
+    const filePath = path.join(imageSavePath, `${fileName}.tar`);
+    const command = `docker save ${imageName} | gzip > ${filePath}.gz`;
+    await execa.command(command, { shell: true });
+}
+
 export async function getCommitHash(): Promise<string> {
     if (process.env.GIT_COMMIT_HASH) return process.env.GIT_COMMIT_HASH;