fix(ci): build temporary plugins required for release-time validation

.github/compare-rule-files.sh

@@ -6,6 +6,7 @@ PLUGIN_NAME=$3
 RESULT_FILE=$4
 CHECKER_TOOL=$5
 FALCO_DOCKER_IMAGE=$6
+LATEST_TAG=$7
 
 set -e pipefail
 
@@ -17,20 +18,7 @@ echo Current branch is \"$cur_branch\"
 echo Checking version for rules file \"$RULES_FILE\"...
 cp $RULES_FILE tmp_rule_file.yaml
 
-set +e pipefail
-echo Searching tag with prefix prefix \"$PLUGIN_NAME-\"...
-latest_tag=`git describe --match="$PLUGIN_NAME-*.*.*" --exclude="$PLUGIN_NAME-*.*.*-*" --abbrev=0 --tags $(git rev-list --tags="$PLUGIN_NAME-*.*.*" --max-count=1)`
-set -e pipefail
-
-if [ -z "$latest_tag" ]
-then
-    echo Not previous tag has been found
-    exit 0
-else
-    echo Most recent tag found is \"$latest_tag\"
-fi
-
-git checkout tags/$latest_tag
+git checkout tags/$LATEST_TAG
 chmod +x $CHECKER_TOOL
 $CHECKER_TOOL \
     compare \
@@ -42,7 +30,7 @@ $CHECKER_TOOL \
 git switch --detach $cur_branch
 
 echo '##' $(basename $RULES_FILE) >> $RESULT_FILE
-echo Comparing \`$cur_branch\` with latest tag \`$latest_tag\` >> $RESULT_FILE
+echo Comparing \`$cur_branch\` with latest tag \`$LATEST_TAG\` >> $RESULT_FILE
 echo "" >> $RESULT_FILE
 if [ -s tmp_res.txt ]
 then

.github/get-latest-plugin-version.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+PLUGIN=$1
+
+set +e pipefail
+echo Searching tag with prefix prefix \"${PLUGIN}-\"...
+git fetch --tags origin
+latest_tag=`git describe --match="${PLUGIN}-*.*.*" --exclude="${PLUGIN}-*.*.*-*" --abbrev=0 --tags $(git rev-list --tags="${PLUGIN}-*.*.*" --max-count=1)`
+set -e pipefail
+
+latest_ver="0.0.0"
+if [ -z "$latest_tag" ]
+then
+  echo Not previous tag has been found
+else
+  echo Most recent tag found is \"$latest_tag\"
+  latest_ver=$(echo $latest_tag | cut -d '-' -f 2-)
+fi
+
+echo Setting plugin version for "${PLUGIN}" to $latest_ver
+echo "version=$latest_ver" >> $GITHUB_OUTPUT
+echo "ref=${PLUGIN}-$latest_ver" >> $GITHUB_OUTPUT

.github/get-config-file.sh → .github/setup-plugin-config-rules.sh

@@ -1,18 +1,22 @@
 #!/bin/bash
 
 PLUGIN=$1
-CONFIG_FILE=$2
-RULES_DIR=$3
+
+# set expected paths for plugins' config and rules files
+rules_dir="$GITHUB_WORKSPACE/plugins/${PLUGIN}/rules"
+config_file="$GITHUB_WORKSPACE/plugins/${PLUGIN}/falco.yaml"
+
+# set paths into step outputs
+echo "rules_dir=${rules_dir}" >> "$GITHUB_OUTPUT"
+echo "config_file=${config_file}" >> "$GITHUB_OUTPUT"
 
 # craft a default falco.yaml if no custom one is available
-config_file=$CONFIG_FILE
 if [ ! -f "$config_file" ]; then
   # we assume that the current plugin is always a dependency
   deps="$PLUGIN"
 
   # we collect all plugin dependencies across all plugin rulesets
   # todo(jasondellaluce): find a way to avoid ignoring alternatives
-  rules_dir=$RULES_DIR
   if [ -d "$rules_dir" ]; then
     echo Extracting plugin dependencies from rules files...
     rules_files=$(ls $rules_dir/*)

.github/workflows/ci.yaml

@@ -14,7 +14,7 @@ jobs:
   build-plugins:
     uses: ./.github/workflows/reusable_build_packages.yaml
     with:
-      makecommand: make packages
+      makecommand: make packages -j4
       suffix: ${{ github.event.number }}
     secrets: inherit
 

.github/workflows/master.yaml

@@ -14,7 +14,7 @@ jobs:
   build-plugins-dev:
     uses: falcosecurity/plugins/.github/workflows/reusable_build_packages.yaml@master
     with:
-      makecommand: make packages
+      makecommand: make packages -j4
       suffix: dev
     secrets: inherit
 

.github/workflows/release.yml

@@ -33,7 +33,7 @@ jobs:
     needs: extract-info
     uses: falcosecurity/plugins/.github/workflows/reusable_build_packages.yaml@master
     with:
-      makecommand: make release/${{ needs.extract-info.outputs.package }}
+      makecommand: make release/${{ needs.extract-info.outputs.package }} -j4
       suffix: stable
     secrets: inherit
 

.github/workflows/reusable_get_changed_plugins.yaml

@@ -18,7 +18,7 @@ jobs:
       - name: Get changed files
         id: changed-plugins
         if: github.event_name == 'pull_request'
-        uses: jitterbit/get-changed-files@v1
+        uses: Ana06/get-changed-files@v2.2.0
         with:
           format: space-delimited
           token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/reusable_validate_plugins.yaml

@@ -1,4 +1,4 @@
-# This is a reusable workflow used by master CI
+# This is a reusable workflow used by master and release CI
 on:
   workflow_call:
     inputs:
@@ -27,26 +27,21 @@ on:
         required: true
         type: string
 
-jobs:  
+jobs:
   # todo(jasondellaluce): support aarch64 too
-  validate-plugin-and-rules:
+  validate-local:
     if: inputs.arch == 'x86_64'
-    env:
-      RULES_DIR: ${{ github.workspace }}/plugins/${{ inputs.plugin }}/rules
-      CONFIG_FILE: ${{ github.workspace }}/plugins/${{ inputs.plugin }}/falco.yaml
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
         uses: actions/checkout@v3
 
       - name: Install system dependencies
-        run:
-          pip install yq
+        run: pip install yq
 
-      - name: Download falcoctl
-        run: |
-          curl --fail -LS "https://github.com/falcosecurity/falcoctl/releases/download/v${{ inputs.falcoctl-version }}/falcoctl_${{ inputs.falcoctl-version }}_linux_${{ inputs.arch == 'x86_64' && 'amd64' || 'arm64' }}.tar.gz" | tar -xz
-          sudo install -o root -g root -m 0755 falcoctl /usr/local/bin/falcoctl
+      - name: Setup plugin config and rules
+        id: get-config
+        run: ./.github/setup-plugin-config-rules.sh ${{ inputs.plugin }}
 
       - name: Download rules tool
         uses: actions/download-artifact@v3
@@ -58,57 +53,87 @@ jobs:
         with:
           name: ${{ inputs.plugins-artifact }}
           path: /tmp/plugins-${{ inputs.arch }}
-
-      - name: Get config file
-        run: |
-          ./.github/get-config-file.sh \
-            ${{ inputs.plugin }} \
-            ${{ env.CONFIG_FILE }} \
-            ${{ env.RULES_DIR }}
 
-      - name: Install plugin and rules (local artifacts)
+      - name: Install plugin and rules
         run: |
-          config_file=$CONFIG_FILE
-          loaded_plugins="$(cat $config_file | grep '\- name: ' | cut -d ':' -f 2 | xargs)"
-          
+          arch=${{ inputs.arch }}
+          loaded_plugins="$(cat ${{ steps.get-config.outputs.config_file }} | grep '\- name: ' | cut -d ':' -f 2 | xargs)"
+
           sudo mkdir -p /etc/falco/falco
           sudo mkdir -p /usr/share/falco/plugins
 
-          mkdir -p tmp
-          pushd tmp
           for plugin_name in $loaded_plugins; do
-            echo Installing locally-built plugin "$plugin_name"...
-            for archive in $(ls /tmp/plugins-${{ inputs.arch }}/${plugin_name}-*); do
-              echo Extracting archive "$archive"...
-              tar -xvf $archive
-              sudo cp -r *.so /usr/share/falco/plugins || true
-              sudo cp -r *.yaml /etc/falco/falco || true
-            done
+              echo Installing locally-built plugin "$plugin_name"...
+
+              # at release time we only build the released plugin, so it's possible
+              # that validation requires a plugin that we haven't built locally.
+              # in those cases, we build it on-the-fly perform validation with it.
+              packages=$(ls /tmp/plugins-${arch}/${plugin_name}-*)
+              if [ -z "$packages" ]; then
+                  echo Building plugin "$plugin_name" temporary packages...
+                  make package/$plugin_name -j4
+                  packages=$(ls $(pwd)/output/${plugin_name}-*)
+              fi
+
+              for archive in $packages; do
+                  echo Extracting archive "$archive"...
+                  mkdir -p tmpdir && pushd tmpdir
+                  tar -xvf $archive
+                  sudo cp -r *.yaml /etc/falco/falco || true
+                  sudo cp -r *.so /usr/share/falco/plugins || true
+                  popd && rm -fr tmpdir
+              done
           done
-          popd
-          rm -fr tmp
 
-      - name: Validate plugin and rules (local artifacts)
+      - name: Validate plugin and rules
         run: |
-          config_file=$CONFIG_FILE
-          rules_dir=$RULES_DIR
-
           # craft an empty rules file if none is available.
           # this ensures that the plugin gets still loaded even if it has no rules.
           rules_files=""
-          if [ ! -d "$rules_dir" ]; then
+          if [ ! -d "${{ steps.get-config.outputs.rules_dir }}" ]; then
             touch tmp_rules.yaml
             rules_files="./tmp_rules.yaml"
           else
-            rules_files=$(ls $rules_dir/*)
+            rules_files=$(ls ${{ steps.get-config.outputs.rules_dir }}/*)
           fi
 
           ./.github/validate-rules.sh \
             "${{ inputs.falco-image }}" \
             "${{ inputs.rules-checker }}" \
-            "$config_file" \
+            "${{ steps.get-config.outputs.config_file }}" \
             "$rules_files"
 
+  # todo(jasondellaluce): support aarch64 too
+  validate-falcoctl:
+    if: inputs.arch == 'x86_64'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Install system dependencies
+        run: pip install yq
+
+      - name: Setup plugin config and rules
+        id: get-config
+        run: ./.github/setup-plugin-config-rules.sh ${{ inputs.plugin }}
+
+      - name: Download plugins
+        uses: actions/download-artifact@v3
+        with:
+          name: ${{ inputs.plugins-artifact }}
+          path: /tmp/plugins-${{ inputs.arch }}
+
+      - name: Download rules tool
+        uses: actions/download-artifact@v3
+        with:
+          name: rules-tool.tar.gz
+
+      - name: Download falcoctl
+        run: |
+          curl --fail -LS "https://github.com/falcosecurity/falcoctl/releases/download/v${{ inputs.falcoctl-version }}/falcoctl_${{ inputs.falcoctl-version }}_linux_${{ inputs.arch == 'x86_64' && 'amd64' || 'arm64' }}.tar.gz" | tar -xz
+          sudo install -o root -g root -m 0755 falcoctl /usr/local/bin/falcoctl
+
       # note(jsondellaluce): exploring the set of all dependencies including their
       # alternatives and all the possible combinations of different versions would
       # result in a combinatorial explosion. As such, we take the simple route
@@ -132,17 +157,17 @@ jobs:
       # todo(jasondellaluce): improve this by attempting more cases
       # todo(jasondellaluce): if we skip one minor version (e.g. bump from v0.1.0
       # to v0.3.0), this algorithm would stop before finishing the exploration
-      - name: Validate plugin and rules (falcoctl artifacts)
+      - name: Validate plugin and rules
         run: |
           set -e pipefail
-          config_file=$CONFIG_FILE
-          rules_dir=$RULES_DIR
 
-          if [ ! -d "$rules_dir" ]; then
+          if [ ! -d "${{ steps.get-config.outputs.rules_dir }}" ]; then
             exit 0
           fi
 
-          rules_files=$(ls $rules_dir/*)
+          sudo mkdir -p /usr/share/falco/plugins
+
+          rules_files=$(ls ${{ steps.get-config.outputs.rules_dir }}/*)
           for rules_file in $rules_files; do
             deps=$(cat $rules_file | yq -r '.[].required_plugin_versions | select(. != null and . != "")[] | [.name + ":" + .version] | @csv')
 
@@ -166,6 +191,16 @@ jobs:
                 if [ $? -eq 0 ]; then
                   echo Installed plugin "${plugin_name}" at version "${plugin_ver}"
                   has_updates=1
+                else
+                  echo Can\'t pull plugin "${plugin_name}" at version "${plugin_ver}"
+                  echo Attempt installing locally-built plugin "${plugin_name}"...
+                  for archive in $(ls /tmp/plugins-${{ inputs.arch }}/${plugin_name}-*); do
+                    echo Extracting archive "$archive"...
+                    mkdir -p tmpdir && pushd tmpdir
+                    tar -xvf $archive
+                    sudo cp -r *.so /usr/share/falco/plugins || true
+                    popd && rm -fr tmpdir
+                  done
                 fi
                 set -e pipefail
               done
@@ -174,7 +209,7 @@ jobs:
               ./.github/validate-rules.sh \
                 "${{ inputs.falco-image }}" \
                 "${{ inputs.rules-checker }}" \
-                "$config_file" \
+                "${{ steps.get-config.outputs.config_file }}" \
                 "$rules_file"
             done 
           done

.github/workflows/rules.yaml

@@ -34,47 +34,43 @@ jobs:
       matrix:
         plugin: ${{ fromJson(needs.get-changed-plugins.outputs.changed-plugins) }}
     runs-on: ubuntu-latest
-    env:
-      RULES_DIR: ${{ github.workspace }}/plugins/${{ matrix.plugin }}/rules
-      CONFIG_FILE: ${{ github.workspace }}/plugins/${{ matrix.plugin }}/falco.yaml
     steps:
       - name: Checkout code
         uses: actions/checkout@v3
 
-      - name: Get all git tags
-        run: git fetch --tags origin
-
       - name: Install system dependencies
-        run:
-          pip install yq
+        run: pip install yq
+
+      - name: Setup plugin config and rules
+        id: get-config
+        run: ./.github/setup-plugin-config-rules.sh ${{ matrix.plugin }}
+
+      - name: Get latest tag
+        id: get-tag
+        run: ./.github/get-latest-plugin-version.sh ${{ matrix.plugin }}
 
       - name: Download rules tool
         uses: actions/download-artifact@v3
         with:
           name: rules-tool.tar.gz
-
-      - name: Get config file
-        run: |
-          ./.github/get-config-file.sh \
-            ${{ matrix.plugin }} \
-            ${{ env.CONFIG_FILE }} \
-            ${{ env.RULES_DIR }}
 
       - name: Compare changed files with previous versions
         id: compare
+        if: steps.get-tag.outputs.version != '0.0.0'
         run: |
-          rules_dir=$RULES_DIR
+          rules_dir=${{ steps.get-config.outputs.rules_dir }}
 
           if [ -d "$rules_dir" ]; then
             rules_files=$(ls $rules_dir/*)
             for rules_file in $rules_files; do
               ./.github/compare-rule-files.sh \
                   "$rules_file" \
-                  $CONFIG_FILE \
+                  ${{ steps.get-config.outputs.config_file }} \
                   ${{ matrix.plugin }} \
                   rule_result.txt \
                   ./rules-checker \
-                  "falcosecurity/falco-no-driver:$FALCO_VERSION"
+                  "falcosecurity/falco-no-driver:$FALCO_VERSION" \
+                  ${{ steps.get-tag.outputs.ref }}
 
               if [ -s rule_result.txt ]; then
                 if [ ! -s result.txt ]; then

plugins/cloudtrail/rules/aws_cloudtrail_rules.yaml

@@ -1,5 +1,5 @@
 #
-# Copyright (C) 2022 The Falco Authors.
+# Copyright (C) 2023 The Falco Authors.
 #
 #
 # Licensed under the Apache License, Version 2.0 (the "License");