Skip to content
Teraslice CI Tests

Docker ratelimit ci #871

Sign in to view logs

Docker ratelimit ci

Docker ratelimit ci #871

Workflow file for this run

.github/workflows/test.yml at 5f08d07
name: Teraslice CI Tests
# this will technically run again on merge to master, should limit it
on:
pull_request:
branches: [ master ]
jobs:
verify-build:
runs-on: ubuntu-latest
steps:
- name: Check out code
uses: actions/checkout@v4
- name: Setup Node
uses: actions/setup-node@v4
with:
node-version: 18.19.1
cache: 'yarn'
- name: Install and build packages
run: yarn setup
env:
YARN_SETUP_ARGS: "--prod=false --silent"
- name: Lint codebase
run: yarn lint
- name: Sync codebase
run: yarn sync --verify
cache-docker-images:
runs-on: ubuntu-latest
needs: verify-build
steps:
- name: Check out code
uses: actions/checkout@v3
- name: Setup Node
uses: actions/setup-node@v3
with:
node-version: 18.19.1
cache: 'yarn'
# we login to docker to avoid docker pull limit rates
- name: Login to Docker Hub
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Check docker.hub limit start of job
env:
USER: ${{ secrets.DOCKER_USERNAME }}
PASS: ${{ secrets.DOCKER_PASSWORD }}
run: yarn docker:limit
- name: Install and build packages
run: yarn setup
env:
YARN_SETUP_ARGS: "--prod=false --silent"
- name: Create Docker Image List
run: yarn dockerImageList
- name: Restore Docker image cache
id: docker-cache
uses: actions/cache@v3
with:
path: /tmp/docker_cache
key: docker-images-${{ hashFiles('./images/image-list.txt') }}
# restore-keys: |
# docker-images-
- name: Pull and save Docker images
if: ${{steps.docker-cache.outputs.cache-hit != 'true'}}
# run: yarn docker:pullAndSaveImages
run: |
mkdir -p /tmp/docker_cache
while IFS= read -r image; do
docker pull "$image"
docker save "$image" | gzip > "/tmp/docker_cache/$(echo $image | tr '/:' '__').tar.gz"
done < ./images/image-list.txt
- name: Update Docker image cache
if: ${{steps.docker-cache.outputs.cache-hit != 'true'}}
uses: actions/cache@v3
with:
path: /tmp/docker_cache
key: docker-images-${{ hashFiles('./images/image-list.txt') }}
- name: Check docker.hub limit end of job
env:
USER: ${{ secrets.DOCKER_USERNAME }}
PASS: ${{ secrets.DOCKER_PASSWORD }}
run: npm run docker:limit
# linux-unit-tests:
# runs-on: ubuntu-latest
# strategy:
# matrix:
# node-version: [18.19.1, 20.11.1, 22.2.0]
# steps:
# - name: Check out code
# uses: actions/checkout@v4
# - name: Setup Node ${{ matrix.node-version }}
# uses: actions/setup-node@v4
# with:
# node-version: ${{ matrix.node-version }}
# cache: 'yarn'
# - name: Install and build packages
# run: yarn setup
# env:
# YARN_SETUP_ARGS: "--prod=false --silent"
# - name: Test
# run: yarn --silent test -- --suite unit --
# linux-unit-esm-tests:
# runs-on: ubuntu-latest
# strategy:
# matrix:
# node-version: [18.19.1, 20.11.1, 22.2.0]
# steps:
# - name: Check out code
# uses: actions/checkout@v4
# - name: Setup Node ${{ matrix.node-version }}
# uses: actions/setup-node@v4
# with:
# node-version: ${{ matrix.node-version }}
# cache: 'yarn'
# - name: Install and build packages
# run: yarn setup
# env:
# YARN_SETUP_ARGS: "--prod=false --silent"
# - name: Test
# run: yarn --silent test -- --suite unit-esm --
# env:
# NODE_OPTIONS: '--experimental-vm-modules'
teraslice-elasticsearch-tests:
runs-on: ubuntu-latest
needs: cache-docker-images
strategy:
# opensearch is finiky, keep testing others if it fails
fail-fast: false
matrix:
node-version: [18.19.1] #, 20.11.1, 22.2.0]
search-version: [elasticsearch6] #, elasticsearch7, opensearch1, opensearch2]
steps:
- name: Check out code
uses: actions/checkout@v4
- name: Setup Node ${{ matrix.node-version }}
uses: actions/setup-node@v4
with:
node-version: ${{ matrix.node-version }}
cache: 'yarn'
# we login to docker to avoid docker pull limit rates
- name: Login to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Check docker.hub limit start
env:
USER: ${{ secrets.DOCKER_USERNAME }}
PASS: ${{ secrets.DOCKER_PASSWORD }}
run: npm run docker:limit
- name: Install and build packages
run: yarn setup
env:
YARN_SETUP_ARGS: "--prod=false --silent"
- name: Create Docker Image List
run: yarn dockerImageList
- name: Restore Docker image cache
id: docker-cache
uses: actions/cache@v3
with:
path: /tmp/docker_cache
key: docker-images-${{ hashFiles('./images/image-list.txt') }}
# restore-keys: |
# docker-images-
- name: Decompress and load Docker images from cache
if: ${{steps.docker-cache.outputs.cache-hit == 'true'}}
# run: yarn docker:loadImagesFromCache
run: |
for tar_gz in /tmp/docker_cache/*.tar.gz; do
if
gzip -dc "$tar_gz" | docker load
done
# - name: Pull Docker images if cache is not hit
# if: ${{steps.docker-cache.outputs.cache-hit != 'true'}}
# run: |
# while IFS= read -r image; do
# docker pull "$image"
# done < image-list.txt
- name: Test ${{ matrix.search-version }}
run: yarn --silent test:${{ matrix.search-version }}
working-directory: ./packages/teraslice
- name: Check docker.hub limit after Test
env:
USER: ${{ secrets.DOCKER_USERNAME }}
PASS: ${{ secrets.DOCKER_PASSWORD }}
run: npm run docker:limit
# elasticsearch-store-tests:
# runs-on: ubuntu-latest
# strategy:
# # opensearch is finiky, keep testing others if it fails
# fail-fast: false
# matrix:
# node-version: [18.19.1, 20.11.1, 22.2.0]
# search-version: [elasticsearch6, elasticsearch7, opensearch1, opensearch2]
# steps:
# - name: Check out code
# uses: actions/checkout@v4
# - name: Setup Node ${{ matrix.node-version }}
# uses: actions/setup-node@v4
# with:
# node-version: ${{ matrix.node-version }}
# cache: 'yarn'
# # we login to docker to avoid docker pull limit rates
# - name: Login to Docker Hub
# uses: docker/login-action@v3
# with:
# username: ${{ secrets.DOCKER_USERNAME }}
# password: ${{ secrets.DOCKER_PASSWORD }}
# - name: Install and build packages
# run: yarn setup
# env:
# YARN_SETUP_ARGS: "--prod=false --silent"
# - name: Test ${{ matrix.search-version }}
# run: yarn --silent test:${{ matrix.search-version }}
# working-directory: ./packages/elasticsearch-store
# elasticsearch-api-tests:
# runs-on: ubuntu-latest
# strategy:
# # opensearch is finiky, keep testing others if it fails
# fail-fast: false
# matrix:
# node-version: [18.19.1, 20.11.1, 22.2.0]
# search-version: [elasticsearch6, elasticsearch7, opensearch1, opensearch2]
# steps:
# - name: Check out code
# uses: actions/checkout@v4
# - name: Setup Node ${{ matrix.node-version }}
# uses: actions/setup-node@v4
# with:
# node-version: ${{ matrix.node-version }}
# cache: 'yarn'
# # we login to docker to avoid docker pull limit rates
# - name: Login to Docker Hub
# uses: docker/login-action@v3
# with:
# username: ${{ secrets.DOCKER_USERNAME }}
# password: ${{ secrets.DOCKER_PASSWORD }}
# - name: Install and build packages
# run: yarn setup
# env:
# YARN_SETUP_ARGS: "--prod=false --silent"
# - name: Test ${{ matrix.search-version }}
# run: yarn --silent test:${{ matrix.search-version }}
# working-directory: ./packages/elasticsearch-api
# e2e-tests:
# runs-on: ubuntu-latest
# strategy:
# # opensearch is finiky, keep testing others if it fails
# fail-fast: false
# matrix:
# node-version: [18.19.1, 20.11.1, 22.2.0]
# search-version: [elasticsearch6, elasticsearch7, opensearch1, opensearch2]
# steps:
# - name: Check out code
# uses: actions/checkout@v4
# - name: Setup Node ${{ matrix.node-version }}
# uses: actions/setup-node@v4
# with:
# node-version: ${{ matrix.node-version }}
# cache: 'yarn'
# # we login to docker to avoid docker pull limit rates
# - name: Login to Docker Hub
# uses: docker/login-action@v3
# with:
# username: ${{ secrets.DOCKER_USERNAME }}
# password: ${{ secrets.DOCKER_PASSWORD }}
# - name: Install and build packages
# run: yarn setup
# env:
# YARN_SETUP_ARGS: "--prod=false --silent"
# - name: Compile e2e code
# run: yarn build
# working-directory: ./e2e
# - name: Test ${{ matrix.search-version }}
# run: yarn --silent test:${{ matrix.search-version }} --node-version ${{ matrix.node-version }}
# working-directory: ./e2e
# e2e-k8s-tests:
# runs-on: ubuntu-latest
# strategy:
# # opensearch is finiky, keep testing others if it fails
# fail-fast: false
# matrix:
# node-version: [18.19.1, 20.11.1, 22.2.0]
# steps:
# - name: Check out code
# uses: actions/checkout@v4
# - name: Setup Node ${{ matrix.node-version }}
# uses: actions/setup-node@v4
# with:
# node-version: ${{ matrix.node-version }}
# cache: 'yarn'
# # we login to docker to avoid docker pull limit rates
# - name: Login to Docker Hub
# uses: docker/login-action@v3
# with:
# username: ${{ secrets.DOCKER_USERNAME }}
# password: ${{ secrets.DOCKER_PASSWORD }}
# - name: Install and build packages
# run: yarn setup
# env:
# YARN_SETUP_ARGS: "--prod=false --silent"
# - name: Compile e2e code
# run: yarn build
# working-directory: ./e2e
# - name: Install Kind and Kubectl
# uses: helm/[email protected]
# with:
# install_only: "true"
# - name: Test k8s elasticsearch7
# run: yarn --silent test:k8s --node-version ${{ matrix.node-version }}
# working-directory: ./e2e
# e2e-external-storage-tests:
# runs-on: ubuntu-latest
# strategy:
# # opensearch is finiky, keep testing others if it fails
# fail-fast: false
# matrix:
# node-version: [18.19.1]
# steps:
# - name: Check out code
# uses: actions/checkout@v4
# - name: Setup Node ${{ matrix.node-version }}
# uses: actions/setup-node@v4
# with:
# node-version: ${{ matrix.node-version }}
# cache: 'yarn'
# # we login to docker to avoid docker pull limit rates
# - name: Login to Docker Hub
# uses: docker/login-action@v3
# with:
# username: ${{ secrets.DOCKER_USERNAME }}
# password: ${{ secrets.DOCKER_PASSWORD }}
# - name: Install and build packages
# run: yarn setup
# env:
# YARN_SETUP_ARGS: "--prod=false --silent"
# - name: Compile e2e code
# run: yarn build
# working-directory: ./e2e
# - name: Test external Asset Storage opensearch1
# run: yarn --silent test:s3AssetStorage --node-version ${{ matrix.node-version }}
# working-directory: ./e2e
# e2e-external-storage-tests-encrypted:
# runs-on: ubuntu-latest
# strategy:
# # opensearch is finiky, keep testing others if it fails
# fail-fast: false
# matrix:
# node-version: [18.19.1]
# steps:
# - name: Check out code
# uses: actions/checkout@v4
# - name: Setup Node ${{ matrix.node-version }}
# uses: actions/setup-node@v4
# with:
# node-version: ${{ matrix.node-version }}
# cache: 'yarn'
# # we login to docker to avoid docker pull limit rates
# - name: Login to Docker Hub
# uses: docker/login-action@v3
# with:
# username: ${{ secrets.DOCKER_USERNAME }}
# password: ${{ secrets.DOCKER_PASSWORD }}
# - name: Install and build packages
# run: yarn setup
# env:
# YARN_SETUP_ARGS: "--prod=false --silent"
# - name: Install mkcert
# run: curl -JLO "https://dl.filippo.io/mkcert/latest?for=linux/amd64" && sudo chmod 777 mkcert-v*-linux-amd64 && sudo cp mkcert-v*-linux-amd64 /usr/local/bin/mkcert
# - name: Install grep
# run: sudo apt update && sudo apt install grep
# - name: Check mkcert
# run: command -v mkcert
# - name: Check grep
# run: command -v grep
# - name: Compile e2e code
# run: yarn build
# working-directory: ./e2e
# - name: Test external Asset Storage opensearch1
# run: ENCRYPT_MINIO=true yarn --silent test:s3AssetStorage --node-version ${{ matrix.node-version }}
# working-directory: ./e2e