fix benches and add verify configs

Signed-off-by: Yu Chin Fabian Lim <[email protected]>

scripts/generate_sample_configurations.py

@@ -157,7 +157,7 @@ def read_configuration(path: str) -> Dict:
 #   config.
 COMBINATIONS = [
     ("accelerated-peft-autogptq", (KEY_AUTO_GPTQ,)),
-    # ("accelerated-peft-bnb-nf4", (KEY_BNB_NF4,)),
+    ("accelerated-peft-bnb-nf4", (KEY_BNB_NF4,)),
 ]
 
 

scripts/run_benchmarks.sh

@@ -30,6 +30,7 @@ DATA_CACHE=data/cache.json
 # env inputs
 DRY_RUN=${DRY_RUN:-"false"}
 NO_DATA_PROCESSING=${NO_DATA_PROCESSING:-"false"}
+NO_OVERWRITE=${NO_OVERWRITE:-"false"}
 
 # inputs
 NUM_GPUS_MATRIX=${1-"1 2"}
@@ -42,6 +43,18 @@ echo "RESULT_DIR: $RESULT_DIR"
 echo "SCENARIOS_CONFIG: $SCENARIOS_CONFIG"
 echo "SCENARIOS_FILTER: $SCENARIOS_FILTER"
 
+if [ -n "$RESULT_DIR" ]; then
+    echo "The results directory is not empty. "
+    if [ "$NO_OVERWRITE" = "true" ]; then 
+        echo "Results dir $RESULT_DIR is not empty, but NO_OVERWRITE=true"
+        echo "If intending to overwrite please delete the folder manually"
+        echo "or do not set NO_OVERWRITE"
+        exit 1
+    fi
+    echo "Deleting $RESULT_DIR"
+    rm -rf $RESULT_DIR
+fi
+
 # tag on the directories
 SCENARIOS_CONFIG=$WORKING_DIR/$SCENARIOS_CONFIG
 DEFAULTS_CONFIG=$WORKING_DIR/$DEFAULTS_CONFIG

scripts/verify_generated_configurations.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+OUTPUT_DIR=${1:-sample-configurations}
+
+GIT_DIFF=$(git diff HEAD -- $OUTPUT_DIR)
+echo "git diff of configurations with HEAD:"
+echo "$GIT_DIFF"
+
+function echoWarning() {
+  LIGHT_YELLOW='\033[1;33m'
+  NC='\033[0m' # No Color
+  echo -e "${LIGHT_YELLOW}${1}${NC}"
+}
+
+if [ ! -z "$GIT_DIFF" ]; then
+    echoWarning "At least one of the configs in the plugins should have changed."
+    echoWarning "Please run 'tox -e gen-configs' to ensure that the sample-configurations are correctly generated!"
+    echoWarning "After that commit the generated sample-configurations to remove this error."
+    exit 1
+fi
+
+echo "sample configurations up to date with configs in plugin directories"

tox.ini

@@ -9,15 +9,27 @@ skip_install = true
 commands = 
     python scripts/generate_sample_configurations.py {posargs:sample-configurations}
 
+[testenv:verify-configs]
+description = verify that sample configurations for all plugins are properly generated
+skip_install = true
+commands = 
+    bash scripts/verify_generated_configurations.sh {posargs:sample-configurations}
+allowlist_externals = bash
+
 # put this here first, consider moving it later
 [testenv:run-benches]
 description = run benchmarks 
 skip_install = true
+setenv = 
+    FHT_BRANCH = {env:FHT_BRANCH:main}
+    DRY_RUN = {env:DRY_RUN:false}
+    NO_DATA_PROCESSING = {env:NO_DATA_PROCESSING:false}
+    NO_OVERWRITE = {env:NO_OVERWRITE:false}
 commands = 
     # need a version of fms-hf-tuning that has integrated the framework
     # NOTE: have to install this first coz havnt merged
     # - this repo has a lot of pins, so we just install it first
-    pip install "fms-hf-tuning[flash-attn] @ git+https://github.com/fabianlim/fms-hf-tuning.git@acceleration-framework"
+    pip install "fms-hf-tuning[flash-attn] @ git+https://github.com/fabianlim/fms-hf-tuning.git@"{env:FHT_BRANCH}
 
     # some models need this for tokenizers
     pip install protobuf