finish cd

Signed-off-by: tylertitsworth <[email protected]>

.github/workflows/composite/scan/action.yml

@@ -20,7 +20,7 @@ runs:
   args:
     - '--timeout=30m'
     - image
-    - '--format=sarif'
+    - '--format=table'
     - '--no-progress'
     - '--output=${{ inputs.output }}'
     - ${{ inputs.image-ref }}

.github/workflows/reuse-container-ci.yaml → .github/workflows/container-ci.yaml

@@ -1,29 +1,26 @@
 # Copyright (C) 2024 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
-name: Container CI
+name: Example Container CI
 permissions: read-all
 on:
-  workflow_dispatch:
-    inputs:
-      group_dir:
-        required: true
-        description: 'Enter Container Group Directory:'
-        type: string
-      env_overrides:
-        description: 'Enter Bash Env Variable Overrides in `KEY=VAL KEY2=VAL2` format:'
-        required: false
-        type: string
-      runner_label:
-        description: 'Enter Validation Runner Label:'
-        default: test-runner
-        required: true
-        type: string
   workflow_call:
     inputs:
-      group_dir:
+      example_dir:
         required: true
         type: string
+      scan:
+        default: true
+        required: false
+        type: boolean
+      test:
+        default: true
+        required: false
+        type: boolean
+      publish:
+        default: false
+        required: false
+        type: boolean
 jobs:
 ####################################################################################################
 # Compose Build
@@ -45,15 +42,15 @@ jobs:
       id: build-group
       uses: ./.github/workflows/composite/docker-build
       with:
-        group_dir: ${{ inputs.group_dir }}
+        example_dir: ${{ inputs.example_dir }}
         env_overrides: ${{ inputs.env_overrides || env.env_overrides || '' }}
         registry: ${{ secrets.REGISTRY }}
 ####################################################################################################
 # Trivy Scan
 ####################################################################################################
   setup-scan:
     needs: [build-containers]
-    if: ${{ github.event_name == 'pull_request' }}
+    if: ${{ fromJSON(inputs.scan) }}
     runs-on: ubuntu-latest
     outputs:
       matrix: ${{ steps.scan-matrix.outputs.matrix }}
@@ -70,10 +67,10 @@ jobs:
       run: echo "matrix=$(cat matrix/*-${{ needs.build-containers.outputs.group }}/*.txt | jq -R '.' | jq -sc '. | unique')" >> $GITHUB_OUTPUT
   scan-containers:
     needs: [setup-scan]
-    if: ${{ !inputs.no_build && github.event_name == 'pull_request' }}
+    if: ${{ fromJSON(inputs.scan) }}
     runs-on: ubuntu-latest # local registry label
     env:
-      GROUP_DIR: ${{ inputs.group_dir }}
+      example_dir: ${{ inputs.example_dir }}
     strategy:
       matrix:
         container: ${{ fromJSON(needs.setup-scan.outputs.matrix) }}
@@ -90,26 +87,36 @@ jobs:
         username: ${{ secrets.REGISTRY_USER }}
         password: ${{ secrets.REGISTRY_TOKEN }}
     - name: Downcase Group Dir
-      run: echo "group_dir=${GROUP_DIR,,}" >> ${GITHUB_ENV}
+      run: echo "example_dir=${example_dir,,}" >> ${GITHUB_ENV}
     - name: Pull Image
-      run: docker pull ${{ secrets.REGISTRY }}/${{ env.group_dir }}:${{ matrix.container }}
+      run: docker pull ${{ secrets.REGISTRY }}/${{ env.example_dir }}:${{ matrix.container }}
       env:
-        GROUP_DIR: ${{ inputs.group_dir }}
+        example_dir: ${{ inputs.example_dir }}
     - name: Scan Container
       uses: ./.github/workflows/composite/scan
       with:
-        image-ref: ${{ secrets.REGISTRY }}/${{ env.group_dir }}:${{ matrix.container }}
-        output: ${{ matrix.container }}-${{ env.group_dir }}-scan.sarif
+        image-ref: ${{ secrets.REGISTRY }}/${{ env.example_dir }}:${{ matrix.container }}
+        output: ${{ matrix.container }}-${{ env.example_dir }}-scan.txt
     - name: Cleanup
       if: always()
-      run: docker rmi -f ${{ secrets.REGISTRY }}/${{ env.group_dir }}:${{ matrix.container }}
-    - uses: github/codeql-action/upload-sarif@b611370bb5703a7efb587f9d136a52ea24c5c38c # v3.25.11
+      run: docker rmi -f ${{ secrets.REGISTRY }}/${{ env.example_dir }}:${{ matrix.container }}
+    # Requires GitHub Advanced Security
+    # - uses: github/codeql-action/upload-sarif@b611370bb5703a7efb587f9d136a52ea24c5c38c # v3.25.11
+    #   with:
+    #     sarif_file: '${{ matrix.container }}-${{ env.example_dir }}-scan.sarif'
+    #     category: '${{ matrix.container }}'
+    #   continue-on-error: true
+    - uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4.3.4
       with:
-        sarif_file: '${{ matrix.container }}-${{ env.group_dir }}-scan.sarif'
-        category: '${{ matrix.container }}'
-      continue-on-error: true
+        name: ${{ matrix.container }}-${{ env.example_dir }}-scan
+        path: ${{ matrix.container }}-${{ env.example_dir }}-scan.txt
+        overwrite: true
+####################################################################################################
+# Validation
+####################################################################################################
   setup-test:
     needs: [build-containers]
+    if: ${{ fromJSON(inputs.test) }}
     runs-on: ubuntu-latest
     outputs:
       matrix: ${{ steps.test-matrix.outputs.matrix }}
@@ -121,10 +128,10 @@ jobs:
     - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: Get Tests
       id: test-matrix
-      run: echo "matrix=$(find ${{ inputs.group_dir }}/tests -type f -name 'test_*.sh' -print | jq -R . | jq -sc .)" >> $GITHUB_OUTPUT
+      run: echo "matrix=$(find ${{ inputs.example_dir }}/tests -type f -name 'test_*.sh' -print | jq -R . | jq -sc .)" >> $GITHUB_OUTPUT
   test-containers:
     needs: [setup-test]
-    if: ${{ needs.setup-test.outputs.matrix != '[]' }}
+    if: ${{ needs.setup-test.outputs.matrix != '[]' && fromJSON(inputs.test) }}
     runs-on: ubuntu-latest # validation label
     strategy:
       matrix:
@@ -140,3 +147,33 @@ jobs:
       run: bash ${{ matrix.tests }}
       env:
         REGISTRY: ${{ secrets.REGISTRY }}
+####################################################################################################
+# Publish
+####################################################################################################
+  publish:
+    needs: [build-containers, scan-containers, test-containers]
+    if: ${{ fromJSON(inputs.publish) }}
+    runs-on: ubuntu-latest # local registry label
+    steps:
+    - name: Harden Runner
+      uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1
+      with:
+        egress-policy: audit
+    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+    - uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0
+      with:
+        username: ${{ secrets.DOCKERHUB_USER }}
+        password: ${{ secrets.DOCKERHUB_TOKEN }}
+    - name: Pull Images
+      run: docker compose pull
+      working-directory: ${{ inputs.example_dir }}
+    - name: Push Images
+      run: docker compose push
+      working-directory: ${{ inputs.example_dir }}
+    - name: Cleanup
+      if: always()
+      run: |
+        docker logout
+        docker compose down
+        echo 'y' | docker system prune
+      working-directory: ${{ inputs.example_dir }}

.github/workflows/container-ci.yml → .github/workflows/example-ci.yml

@@ -1,18 +1,18 @@
 # Copyright (C) 2024 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
-name: Container Integration Tests
+name: Example Integration Tests
 on:
-  pull_request
+  pull_request: null
 permissions: read-all
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 jobs:
-  group-diff:
+  example-diff:
     runs-on: ubuntu-latest
     outputs:
-      groups: ${{ steps.group-list.outputs.FOLDERS }}
+      examples: ${{ steps.example-list.outputs.FOLDERS }}
     steps:
     - name: Harden Runner
       uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1
@@ -21,16 +21,16 @@ jobs:
     - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
       with:
         fetch-depth: 0
-    - name: Output Modified Group Directories
-      id: group-list
+    - name: Output Modified example Directories
+      id: example-list
       run: |
         # Get diff array filtered by specific filetypes
         DIFF=$(git diff --diff-filter=d \
             --name-only ${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }} \
             -- '*/*Dockerfile' '*.py' '*.yaml' '*.yml' '*.sh' '*/*requirements.txt' '*.json' '*.ts' '*.js' | \
             jq -R '.' | jq -sc '.' \
         )
-        # Search for compose files in each file to determine the container groups
+        # Search for compose files in each file to determine the container examples
         DOCKER_COMPOSE_PATHS=()
         for path in $(echo $DIFF | jq -r '.[]'); do
           while [[ "$path" != "." ]]; do
@@ -47,19 +47,22 @@ jobs:
         DOCKER_COMPOSE_PATHS_JSON=$(printf '%s\n' "${DOCKER_COMPOSE_PATHS[@]}" | jq -R '.' | jq -sc 'unique_by(.)')
         echo "FOLDERS=$DOCKER_COMPOSE_PATHS_JSON" >> $GITHUB_OUTPUT
   pipeline-ci:
-    needs: group-diff
-    if: needs.group-diff.outputs.groups != '[""]'
+    needs: example-diff
+    if: needs.example-diff.outputs.examples != '[""]'
     strategy:
       matrix:
-        group: ${{ fromJson(needs.group-diff.outputs.groups) }}
+        example: ${{ fromJson(needs.example-diff.outputs.examples) }}
         experimental: [true]
       fail-fast: false
-    uses: opea/genaiexamples/.github/workflows/reuse-container-ci.yaml@main
+    uses: opea/genaiexamples/.github/workflows/container-ci.yaml@main
     with:
-      group_dir: ${{ matrix.group }}
+      example_dir: ${{ matrix.example }}
+      scan: false
+      test: true
+      publish: false
     secrets: inherit
   status-check:
-    needs: [group-diff, pipeline-ci]
+    needs: [example-diff, pipeline-ci]
     runs-on: ubuntu-latest
     if: always()
     steps:
@@ -73,5 +76,5 @@ jobs:
              contains(needs.*.result, 'failure')
           || contains(needs.*.result, 'cancelled')
           || contains(needs.*.result, 'skipped')
-          && needs.group-diff.outputs.groups != '[""]'
+          && needs.example-diff.outputs.examples != '[""]'
         }}

.github/workflows/example-manual-ci.yml

@@ -0,0 +1,54 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+name: Build Specific Examples
+on:
+  workflow_dispatch:
+    inputs:
+      examples:
+        default: 'example1, example2'
+        description: 'List of comma-separated examples to test'
+        required: true
+        type: string
+      scan:
+        default: true
+        description: 'Scan all examples with Trivy'
+        required: false
+        type: boolean
+      test:
+        default: true
+        description: 'Test Examples'
+        required: false
+        type: boolean
+      publish:
+        default: false
+        description: 'Publish Images to Dockerhub'
+        required: false
+        type: boolean
+permissions: read-all
+jobs:
+  setup-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.get-matrix.outputs.matrix }}
+    steps:
+    - name: Create Matrix
+      id: get-matrix
+      run: |
+        EXAMPLES=($(echo ${{ github.event.inputs.examples }} | tr ',' ' '))
+        EXAMPLES_JSON=$(printf '%s\n' "${EXAMPLES[@]}" | jq -R '.' | jq -sc '.')
+        echo "matrix=$EXAMPLES_JSON" >> $GITHUB_OUTPUT
+  container-ci:
+    needs: [setup-matrix]
+    strategy:
+      matrix:
+        example: ${{ fromJson(needs.setup-matrix.outputs.matrix) }}
+        experimental: [true]
+      fail-fast: false
+    uses: opea/genaiexamples/.github/workflows/container-ci.yaml@main
+    with:
+      example_dir: ${{ matrix.example }}
+      scan: ${{ fromJSON(inputs.scan) }}
+      test: ${{ fromJSON(inputs.test) }}
+      publish: ${{ fromJSON(inputs.publish) }}
+    secrets: inherit

.github/workflows/weekly.yaml → .github/workflows/example-weekly-ci.yaml

@@ -1,18 +1,7 @@
-# Copyright (c) 2024 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
 
-name: Weekly Tests
+name: Weekly Container Tests
 on:
   schedule:
   - cron: "0 0 * * 0"
@@ -21,18 +10,18 @@ on:
     tags: ['**'] # whenever a tag is created/modified
 permissions: read-all
 jobs:
-  get-groups:
+  get-examples:
     runs-on: ubuntu-latest
     outputs:
-      groups: ${{ steps.group-list.outputs.FOLDERS }}
+      examples: ${{ steps.example-list.outputs.FOLDERS }}
     steps:
     - name: Harden Runner
       uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1
       with:
         egress-policy: audit
     - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
-    - name: Output Group Directories
-      id: group-list
+    - name: Output example Directories
+      id: example-list
       run: |
         DOCKER_COMPOSE_PATHS=()
         for path in $(find . -name 'docker-compose.yaml'); do
@@ -43,13 +32,16 @@ jobs:
         DOCKER_COMPOSE_PATHS_JSON=$(printf '%s\n' "${DOCKER_COMPOSE_PATHS[@]}" | uniq | jq -R '.' | jq -sc '.')
         echo "FOLDERS=$DOCKER_COMPOSE_PATHS_JSON" >> $GITHUB_OUTPUT
   pipeline-ci:
-    needs: [get-groups]
+    needs: [get-examples]
     strategy:
       matrix:
-        group: ${{ fromJson(needs.get-groups.outputs.groups) }}
+        example: ${{ fromJson(needs.get-examples.outputs.examples) }}
         experimental: [true]
       fail-fast: false
-    uses: opea/genaiexamples/.github/workflows/reuse-container-ci.yaml@main
+    uses: opea/genaiexamples/.github/workflows/container-ci.yaml@main
     with:
-      group_dir: ${{ matrix.group }}
+      example_dir: ${{ matrix.example }}
+      scan: false
+      test: true
+      publish: false
     secrets: inherit