Merge pull request #30 from buildkite-plugins/toote_follow_symlinks

Follow symlinks
buildkite-plugins · Dec 28, 2022 · 869273f · 869273f
2 parents 46e6127 + 6d87e61
commit 869273f
Show file tree

Hide file tree

Showing 5 changed files with 72 additions and 41 deletions.
diff --git a/README.md b/README.md
@@ -10,7 +10,7 @@ These are all the options available to configure this plugin's behaviour.
 
 #### `files` (string)
 
-Pattern of files to upload to Test Analytics, relative to the checkout path (`./` will be added to it). May contain `*` to match any number of characters of any type (unlike shell expansions, it will match `/` and `.` if necessary)
+Pattern of files to upload to Test Analytics, relative to the checkout path (`./` will be added to it). May contain `*` to match any number of characters of any type (unlike shell expansions, it will match `/` and `.` if necessary).
 
 #### `format` (string)
 
@@ -58,6 +58,10 @@ Important:
 * you may have to be careful to escape special characters like `$` during pipeline upload
 * exclusion of branches is done after the inclusion (through the [`branches` option](#branches-string))
 
+#### `follow-symlinks` (boolean)
+
+By default the plugin will not follow symlinked folders, set this option to `true` to do so. This will add the `-L` option to the `find` command used to get the files to upload.
+
 #### `timeout`(number)
 
 Maximum number of seconds to wait for each file to upload before timing out.
@@ -75,7 +79,7 @@ steps:
   - label: "🔨 Test"
     command: "make test"
     plugins:
-      - test-collector#v1.2.0:
+      - test-collector#v1.4.0:
           files: "test/junit-*.xml"
           format: "junit"
 ```
@@ -89,7 +93,7 @@ steps:
   - label: "🔨 Test"
     command: "make test"
     plugins:
-      - test-collector#v1.2.0:
+      - test-collector#v1.4.0:
           files: "test-data-*.json"
           format: "json"
 ```
@@ -110,7 +114,7 @@ steps:
   - label: "🔍 Test Analytics"
     command: buildkite-agent artifact download tests-*.xml
     plugins:
-      - test-collector#v1.2.0:
+      - test-collector#v1.4.0:
           files: "tests-*.xml"
           format: "junit"
 ```
@@ -124,7 +128,7 @@ steps:
   - label: "🔨 Test"
     command: "make test"
     plugins:
-      - test-collector#v1.2.0:
+      - test-collector#v1.4.0:
           files: "test-data-*.json"
           format: "json"
           branches: "-qa$"
@@ -137,7 +141,7 @@ steps:
   - label: "🔨 Test"
     command: "make test"
     plugins:
-      - test-collector#v1.2.0:
+      - test-collector#v1.4.0:
           files: "test-data-*.json"
           format: "json"
           exclude-branches: "^legacy$"
@@ -150,7 +154,7 @@ steps:
   - label: "🔨 Test"
     command: "make test"
     plugins:
-      - test-collector#v1.2.0:
+      - test-collector#v1.4.0:
           files: "test-data-*.json"
           format: "json"
           branches: "^stage-"

diff --git a/buildkite.yaml b/buildkite.yaml
@@ -11,5 +11,5 @@ steps:
 
   - label: ":docker: :hammer:"
     plugins:
-      docker-compose#v4.5.0:
+      docker-compose#v4.9.0:
         run: tests
diff --git a/hooks/pre-exit b/hooks/pre-exit
@@ -2,13 +2,9 @@
 set -euo pipefail
 
 TOKEN_ENV_NAME="${BUILDKITE_PLUGIN_TEST_COLLECTOR_API_TOKEN_ENV_NAME:-BUILDKITE_ANALYTICS_TOKEN}"
-
 FILES_PATTERN="${BUILDKITE_PLUGIN_TEST_COLLECTOR_FILES:-}"
-
 FORMAT="${BUILDKITE_PLUGIN_TEST_COLLECTOR_FORMAT:-}"
-
 TIMEOUT="${BUILDKITE_PLUGIN_TEST_COLLECTOR_TIMEOUT:-30}"
-
 DEBUG="false"
 
 if [[ "${BUILDKITE_PLUGIN_TEST_COLLECTOR_DEBUG:-}" =~ ^(true|on|1|always)$ ]]; then
@@ -31,24 +27,23 @@ if [[ -n "${BUILDKITE_PLUGIN_TEST_COLLECTOR_EXCLUDE_BRANCHES:-}" ]]; then
   fi
 fi
 
-TOKEN_VALUE=$(eval "echo \${$TOKEN_ENV_NAME:-}")
+TOKEN_VALUE="${!TOKEN_ENV_NAME:-}"
 PLUGIN_VERSION=$(git rev-parse --short HEAD 2>/dev/null || echo "")
 
 # Uploads files to the Test Analytics API
 #
 # Upload failures should not fail the build, and should have a sensible timeout,
 # so that Test Analytics availability doesn't affect build reliability.
-#
-# TODO: Add build annotation on failure
 upload() {
   local file="$3"
+  local format="$2"
 
   local curl_args=(
     "-X" "POST"
     "--silent"
     "--show-error"
-    "--max-time" "$TIMEOUT"
-    "--form" "format=$FORMAT"
+    "--max-time" "${TIMEOUT}"
+    "--form" "format=${format}"
     "--form" "data=@\"$file\""
     "--form" "run_env[CI]=buildkite"
     "--form" "run_env[key]=$BUILDKITE_BUILD_ID"
@@ -79,7 +74,7 @@ upload() {
 
   curl_args+=("-H" "Authorization: Token token=\"$TOKEN_VALUE\"")
 
-  curl "${curl_args[@]}" || true
+  curl "${curl_args[@]}"
 }
 
 if [[ -z "${TOKEN_VALUE}" ]]; then
@@ -97,10 +92,16 @@ if [[ -z "${FORMAT}" ]]; then
   exit 1
 fi
 
+FIND_CMD=(find)
+
+if [[ "${BUILDKITE_PLUGIN_TEST_COLLECTOR_FOLLOW_LINKS:-}" =~ ^(true|on|1|always)$ ]]; then
+  FIND_CMD+=('-L')
+fi
+
 matching_files=()
 while IFS=$'' read -r matching_file ; do
   matching_files+=("$matching_file")
-done < <(find . -path "./${FILES_PATTERN}")
+done < <("${FIND_CMD[@]}" . -path "./${FILES_PATTERN}")
 
 if [[ "${#matching_files[@]}" -eq "0" ]]; then
   echo "No files found matching '${FILES_PATTERN}'"
@@ -111,5 +112,7 @@ fi
 
 for file in "${matching_files[@]}"; do
   echo "Uploading '$file'..."
-  upload "$TOKEN_VALUE" "$FORMAT" "${file}"
+  if ! upload "$TOKEN_VALUE" "$FORMAT" "${file}"; then
+    echo "Error uploading, will continue"
+  fi
 done
diff --git a/plugin.yml b/plugin.yml
@@ -15,6 +15,8 @@ configuration:
       type: string
     files:
       type: string
+    follow-symlinks:
+      type: boolean
     format:
       enum:
         - json

diff --git a/tests/pre-exit-success.bats b/tests/pre-exit-success.bats
@@ -22,14 +22,14 @@ setup() {
   export BUILDKITE_MESSAGE="A message"
 }
 
+COMMON_CURL_OPTIONS='--form \* --form \* --form \* --form \* --form \* --form \* --form \* --form \* --form \* --form \*'
+
 @test "Uploads a file" {
-  stub git "rev-parse --short HEAD : echo 'some-commit-id'"
-  stub curl "-X POST --silent --show-error --max-time 30 --form format=junit --form data=\"@\\\"./tests/fixtures/junit-1.xml\\\"\" --form run_env[CI]=buildkite --form run_env[key]=an-id --form run_env[url]=https://url.com/ --form run_env[branch]=a-branch --form run_env[commit_sha]=a-commit --form run_env[number]=123 --form run_env[job_id]=321 --form run_env[message]=\"A message\" --form run_env[collector]=test-collector-buildkite-plugin --form run_env[version]=some-commit-id https://analytics-api.buildkite.com/v1/uploads -H 'Authorization: Token token=\"a-secret-analytics-token\"' : echo 'curl success'"
+  stub curl "-X POST --silent --show-error --max-time 30 --form format=junit ${COMMON_CURL_OPTIONS} \* -H \* : echo 'curl success'"
 
   run "$PWD/hooks/pre-exit"
 
   unstub curl
-  unstub git
 
   assert_success
   assert_output --partial "Uploading './tests/fixtures/junit-1.xml'..."
@@ -39,15 +39,13 @@ setup() {
 @test "Uploads multiple file" {
   export BUILDKITE_PLUGIN_TEST_COLLECTOR_FILES='**/*/junit-*.xml'
 
-  stub git "rev-parse --short HEAD : echo 'some-commit-id'"
   stub curl \
-    "-X POST --silent --show-error --max-time 30 --form format=junit --form data=\"@\\\"./tests/fixtures/junit-1.xml\\\"\" --form run_env[CI]=buildkite --form run_env[key]=an-id --form run_env[url]=https://url.com/ --form run_env[branch]=a-branch --form run_env[commit_sha]=a-commit --form run_env[number]=123 --form run_env[job_id]=321 --form run_env[message]=\"A message\" --form run_env[collector]=test-collector-buildkite-plugin --form run_env[version]=some-commit-id https://analytics-api.buildkite.com/v1/uploads -H 'Authorization: Token token=\"a-secret-analytics-token\"' : echo 'curl success 1'" \
-    "-X POST --silent --show-error --max-time 30 --form format=junit --form data=\"@\\\"./tests/fixtures/junit-2.xml\\\"\" --form run_env[CI]=buildkite --form run_env[key]=an-id --form run_env[url]=https://url.com/ --form run_env[branch]=a-branch --form run_env[commit_sha]=a-commit --form run_env[number]=123 --form run_env[job_id]=321 --form run_env[message]=\"A message\" --form run_env[collector]=test-collector-buildkite-plugin --form run_env[version]=some-commit-id https://analytics-api.buildkite.com/v1/uploads -H 'Authorization: Token token=\"a-secret-analytics-token\"' : echo 'curl success 2'"
+    "-X POST --silent --show-error --max-time 30 --form format=junit ${COMMON_CURL_OPTIONS} \* -H \* : echo 'curl success 1'" \
+    "-X POST --silent --show-error --max-time 30 --form format=junit ${COMMON_CURL_OPTIONS} \* -H \* : echo 'curl success 2'"
 
   run "$PWD/hooks/pre-exit"
 
   unstub curl
-  unstub git
 
   assert_success
   assert_output --partial "Uploading './tests/fixtures/junit-1.xml'..."
@@ -59,29 +57,26 @@ setup() {
 @test "Debug true prints the curl info w/o token" {
   export BUILDKITE_PLUGIN_TEST_COLLECTOR_DEBUG="true"
 
-  stub git "rev-parse --short HEAD : echo 'some-commit-id'"
-  stub curl "-X POST --silent --show-error --max-time 30 --form format=junit --form data=\"@\\\"./tests/fixtures/junit-1.xml\\\"\" --form run_env[CI]=buildkite --form run_env[key]=an-id --form run_env[url]=https://url.com/ --form run_env[branch]=a-branch --form run_env[commit_sha]=a-commit --form run_env[number]=123 --form run_env[job_id]=321 --form run_env[message]=\"A message\" --form run_env[collector]=test-collector-buildkite-plugin --form run_env[debug]=true --form run_env[version]=some-commit-id https://analytics-api.buildkite.com/v1/uploads -H 'Authorization: Token token=\"a-secret-analytics-token\"' : echo 'curl success'"
+  stub curl "-X POST --silent --show-error --max-time 30 --form format=junit ${COMMON_CURL_OPTIONS} --form \* \* -H \* : echo \"curl success with \${30}\""
 
   run "$PWD/hooks/pre-exit"
 
   unstub curl
-  unstub git
 
   assert_success
+  assert_output --partial "curl success"
   assert_output --partial "curl -X POST"
   refute_output --partial "a-secret-analytics-token"
 }
 
 @test "Debug env var true prints the curl info w/o token" {
   export BUILDKITE_ANALYTICS_DEBUG_ENABLED="true"
 
-  stub git "rev-parse --short HEAD : echo 'some-commit-id'"
-  stub curl "-X POST --silent --show-error --max-time 30 --form format=junit --form data=\"@\\\"./tests/fixtures/junit-1.xml\\\"\" --form run_env[CI]=buildkite --form run_env[key]=an-id --form run_env[url]=https://url.com/ --form run_env[branch]=a-branch --form run_env[commit_sha]=a-commit --form run_env[number]=123 --form run_env[job_id]=321 --form run_env[message]=\"A message\" --form run_env[collector]=test-collector-buildkite-plugin --form run_env[debug]=true --form run_env[version]=some-commit-id https://analytics-api.buildkite.com/v1/uploads -H 'Authorization: Token token=\"a-secret-analytics-token\"' : echo 'curl success'"
+  stub curl "-X POST --silent --show-error --max-time 30 --form format=junit ${COMMON_CURL_OPTIONS} --form \* \* -H \* : echo \"curl success with  with \${30}\""
 
   run "$PWD/hooks/pre-exit"
 
   unstub curl
-  unstub git
 
   assert_success
   assert_output --partial "curl -X POST"
@@ -91,13 +86,11 @@ setup() {
 @test "Debug false does not print the curl info" {
   export BUILDKITE_PLUGIN_TEST_COLLECTOR_DEBUG="false"
 
-  stub git "rev-parse --short HEAD : echo 'some-commit-id'"
-  stub curl "-X POST --silent --show-error --max-time 30 --form format=junit --form data=\"@\\\"./tests/fixtures/junit-1.xml\\\"\" --form run_env[CI]=buildkite --form run_env[key]=an-id --form run_env[url]=https://url.com/ --form run_env[branch]=a-branch --form run_env[commit_sha]=a-commit --form run_env[number]=123 --form run_env[job_id]=321 --form run_env[message]=\"A message\" --form run_env[collector]=test-collector-buildkite-plugin --form run_env[version]=some-commit-id https://analytics-api.buildkite.com/v1/uploads -H 'Authorization: Token token=\"a-secret-analytics-token\"' : echo 'curl success'"
+  stub curl "-X POST --silent --show-error --max-time 30 --form format=junit ${COMMON_CURL_OPTIONS} \* -H \* : echo 'curl success'"
 
   run "$PWD/hooks/pre-exit"
 
   unstub curl
-  unstub git
 
   assert_success
   refute_output --partial "curl -X POST"
@@ -106,8 +99,19 @@ setup() {
 @test "Timeout is configurable" {
   export BUILDKITE_PLUGIN_TEST_COLLECTOR_TIMEOUT='999'
 
+  stub curl "-X POST --silent --show-error --max-time 999 --form format=junit ${COMMON_CURL_OPTIONS} \* -H \* : echo 'curl success'"
+
+  run "$PWD/hooks/pre-exit"
+
+  unstub curl
+
+  assert_success
+  assert_output --partial "curl success"
+}
+
+@test "Git available sends plugin version" {
   stub git "rev-parse --short HEAD : echo 'some-commit-id'"
-  stub curl "-X POST --silent --show-error --max-time 999 --form format=junit --form data=\"@\\\"./tests/fixtures/junit-1.xml\\\"\" --form run_env[CI]=buildkite --form run_env[key]=an-id --form run_env[url]=https://url.com/ --form run_env[branch]=a-branch --form run_env[commit_sha]=a-commit --form run_env[number]=123 --form run_env[job_id]=321 --form run_env[message]=\"A message\" --form run_env[collector]=test-collector-buildkite-plugin --form run_env[version]=some-commit-id https://analytics-api.buildkite.com/v1/uploads -H 'Authorization: Token token=\"a-secret-analytics-token\"' : echo 'curl success'"
+  stub curl "-X POST --silent --show-error --max-time 30 --form format=junit ${COMMON_CURL_OPTIONS} --form \* \* -H \* : echo \"curl success with \${30}\""
 
   run "$PWD/hooks/pre-exit"
 
@@ -116,16 +120,34 @@ setup() {
 
   assert_success
   assert_output --partial "curl success"
+  assert_output --partial "run_env[version]=some-commit-id"
 }
 
-@test "Git unavailable sends no plugin version" {
-  stub git "rev-parse --short HEAD : echo 'git error' >&2; exit 1"
-  stub curl "-X POST --silent --show-error --max-time 30 --form format=junit --form data=\"@\\\"./tests/fixtures/junit-1.xml\\\"\" --form run_env[CI]=buildkite --form run_env[key]=an-id --form run_env[url]=https://url.com/ --form run_env[branch]=a-branch --form run_env[commit_sha]=a-commit --form run_env[number]=123 --form run_env[job_id]=321 --form run_env[message]=\"A message\" --form run_env[collector]=test-collector-buildkite-plugin https://analytics-api.buildkite.com/v1/uploads -H 'Authorization: Token token=\"a-secret-analytics-token\"' : echo 'curl success'"
+@test "Follow links option enabled adds find option" {
+  export BUILDKITE_PLUGIN_TEST_COLLECTOR_FOLLOW_LINKS='true'
+
+  stub find "-L . -path \* : echo './tests/fixtures/junit-1.xml'"
+  stub curl "-X POST --silent --show-error --max-time 30 --form format=junit ${COMMON_CURL_OPTIONS} \* -H \* : echo 'curl success'"
 
   run "$PWD/hooks/pre-exit"
 
   unstub curl
-  unstub git
+  unstub find
+
+  assert_success
+  assert_output --partial "curl success"
+}
+
+@test "Follow links option disabled does not add find option" {
+  export BUILDKITE_PLUGIN_TEST_COLLECTOR_FOLLOW_LINKS='false'
+
+  stub find ". -path \* : echo './tests/fixtures/junit-1.xml'"
+  stub curl "-X POST --silent --show-error --max-time 30 --form format=junit ${COMMON_CURL_OPTIONS} \* -H \* : echo 'curl success'"
+
+  run "$PWD/hooks/pre-exit"
+
+  unstub curl
+  unstub find
 
   assert_success
   assert_output --partial "curl success"