From 79502f8c18b76ecc1deab7c19172820834ad785e Mon Sep 17 00:00:00 2001 From: tpluscode Date: Fri, 27 Sep 2024 10:50:29 +0200 Subject: [PATCH 1/2] feat: run jobs in parallel --- .changeset/weak-ducks-hunt.md | 5 ++ run.sh | 107 +++++++++++++++++++++------------- 2 files changed, 70 insertions(+), 42 deletions(-) create mode 100644 .changeset/weak-ducks-hunt.md diff --git a/.changeset/weak-ducks-hunt.md b/.changeset/weak-ducks-hunt.md new file mode 100644 index 0000000..2f2c838 --- /dev/null +++ b/.changeset/weak-ducks-hunt.md @@ -0,0 +1,5 @@ +--- +"@zazuko/shacl-test": patch +--- + +Test cases are running in parallel. Defaults to 4 concurrent jobs. Use `--concurrency=X` to change. diff --git a/run.sh b/run.sh index 10505e9..c4fc35d 100755 --- a/run.sh +++ b/run.sh @@ -3,6 +3,8 @@ SCRIPT_PATH="$(dirname "$(realpath "$0")")" WORKING_DIR="$(pwd)" FAILED=0 +MAX_JOBS=4 +CURRENT_JOBS=0 filter='' debug=false @@ -36,6 +38,9 @@ while [ $# -gt 0 ]; do --command=*) command="${1#*=}" ;; + --concurrency=*) + MAX_JOBS="${1#*=}" + ;; *) printf "*******************************************\n" printf "* Error: Invalid argument: %s *\n" "$1" @@ -57,6 +62,7 @@ fi command="$command $shapesPath" if [ "$debug" = true ]; then echo "🐞 Command: $command" + echo "🐞 Concurrency: $MAX_JOBS" echo "🐞 Filter: $filter" echo "🐞 Approval flags: $approvalsFlags" echo "" @@ -66,54 +72,71 @@ loadFullShape() { "$SCRIPT_PATH"/load-graph.js "$1" | "$SCRIPT_PATH"/pretty-print.js --prefixes "${prefixes[@]}" } +# Semaphore function to limit parallel jobs +semaphore() { + while [ "$CURRENT_JOBS" -ge "$MAX_JOBS" ]; do + wait -n + CURRENT_JOBS=$((CURRENT_JOBS - 1)) + done + CURRENT_JOBS=$((CURRENT_JOBS + 1)) +} + # iterate over valid cases, run validation and monitor exit code for file in $validCases; do - name=$(basename "$file") - relativePath=$(node -e "console.log(require('path').relative('$WORKING_DIR', '$file'))") - - # check if filter is set and skip if not matching - if [ -n "$filter" ] && ! echo "$file" | grep -iq "$filter"; then - echo "ℹ️SKIP - $relativePath" - continue - fi - - { - sh -c "$command" > "$file.log" 2>&1 - success=$? - } < "$file" - - if [ $success -ne 0 ] ; then - "$SCRIPT_PATH"/report-failure.sh "$file" "$(loadFullShape "$shapesPath")" "$(cat "$file")" - FAILED=1 - else - echo "✅ PASS - $relativePath" - fi + semaphore + ( + name=$(basename "$file") + relativePath=$(node -e "console.log(require('path').relative('$WORKING_DIR', '$file'))") + + # check if filter is set and skip if not matching + if [ -n "$filter" ] && ! echo "$file" | grep -q "$filter"; then + echo "ℹ️SKIP - $relativePath" + exit 0 + fi + + { + sh -c "$command" > "$file.log" 2>&1 + success=$? + } < "$file" + + if [ $success -ne 0 ] ; then + "$SCRIPT_PATH"/report-failure.sh "$file" "$(loadFullShape "$shapesPath")" "$(cat "$file")" + FAILED=1 + else + echo "✅ PASS - $relativePath" + fi + ) & done # iterate over invalid cases for file in $invalidCases; do - name=$(basename "$file") - relativePath=$(node -e "console.log(require('path').relative('$WORKING_DIR', '$file'))") - - # skip if file does not exist - if [ ! -f "$file" ]; then - continue - fi - - # check if pattern is set and skip if not matching - if [ -n "$filter" ] && ! echo "$file" | grep -iq "$filter"; then - echo "ℹ️SKIP - $relativePath" - continue - fi - - report=$(sh -c "$command" < "$file" 2> "$file.log" | "$SCRIPT_PATH"/pretty-print.js --prefixes "${prefixes[@]}") - - if ! echo "$report" | npx approvals "$name" --outdir "$(dirname "$file")" "$approvalsFlags" > /dev/null 2>&1 ; then - "$SCRIPT_PATH"/report-failure.sh "$file" "$(loadFullShape "$shapesPath")" "$(cat "$file")" "check results" - FAILED=1 - else - echo "✅ PASS - $name" - fi + semaphore + ( + name=$(basename "$file") + relativePath=$(node -e "console.log(require('path').relative('$WORKING_DIR', '$file'))") + + # skip if file does not exist + if [ ! -f "$file" ]; then + exit 0 + fi + + # check if pattern is set and skip if not matching + if [ -n "$filter" ] && ! echo "$file" | grep -q "$filter"; then + echo "ℹ️SKIP - $relativePath" + exit 0 + fi + + report=$(sh -c "$command" < "$file" 2> "$file.log" | "$SCRIPT_PATH"/pretty-print.js --prefixes "${prefixes[@]}") + + if ! echo "$report" | npx approvals "$name" --outdir "$(dirname "$file")" "$approvalsFlags" > /dev/null 2>&1 ; then + "$SCRIPT_PATH"/report-failure.sh "$file" "$(loadFullShape "$shapesPath")" "$(cat "$file")" "check results" + FAILED=1 + else + echo "✅ PASS - $name" + fi + ) & done +wait + exit $FAILED From 6ec325b73f9ba3a865102a985588abc809799218 Mon Sep 17 00:00:00 2001 From: tpluscode Date: Fri, 27 Sep 2024 10:52:16 +0200 Subject: [PATCH 2/2] revert: keep case insensitive --- run.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/run.sh b/run.sh index c4fc35d..80d79db 100755 --- a/run.sh +++ b/run.sh @@ -89,7 +89,7 @@ for file in $validCases; do relativePath=$(node -e "console.log(require('path').relative('$WORKING_DIR', '$file'))") # check if filter is set and skip if not matching - if [ -n "$filter" ] && ! echo "$file" | grep -q "$filter"; then + if [ -n "$filter" ] && ! echo "$file" | grep -iq "$filter"; then echo "ℹ️SKIP - $relativePath" exit 0 fi @@ -121,7 +121,7 @@ for file in $invalidCases; do fi # check if pattern is set and skip if not matching - if [ -n "$filter" ] && ! echo "$file" | grep -q "$filter"; then + if [ -n "$filter" ] && ! echo "$file" | grep -iq "$filter"; then echo "ℹ️SKIP - $relativePath" exit 0 fi