[MS-474] Gitlab CICD pipeline

ci-examples/.gitlab/.example-gitlab-ci.yml

@@ -0,0 +1,18 @@
+include:
+  - local: ms-gitlab-ci.yml
+
+stages:
+  - run-moonshot
+
+run-job:
+  stage: run-moonshot
+  image: python:3.11
+  script:
+    - !reference [.setup_ms_script, script]
+    - !reference [.run_ms_script, script]
+    - !reference [.display_ms_script, script]
+    - !reference [.commit_ms_script, script]
+  artifacts:
+    paths:
+      - moonshot.log
+      - moonshot-data/generated-outputs/results/

ci-examples/.gitlab/ms-gitlab-ci.yml

@@ -0,0 +1,137 @@
+# The following GitLab CI/CD variables are required for the pipeline to work:
+# Example and explanation of the new variables:
+# - COOKBOOK_RUN: "python -m moonshot cli run_cookbook '$COOKBOOK_RUN_NAME' \"['common-risk-easy','common-risk-hard','mlc-ai-safety']\" \"['azure-openai-gpt4o']\" -n 1"
+#   This variable should contain the command to run the cookbook.
+
+# - COOKBOOK_RUN_NAME: testcookbookrun-$CI_PIPELINE_ID
+#   This variable contains the name of the cookbook run, which includes the pipeline ID.
+
+# - GITLAB_MOONSHOT_CI_REPO_LOC: gitlab.com/lionel_teo/moonshot-data
+#   This variable contains the location of the forked repository.
+
+# - GITLAB_MOONSHOT_PROJECT_TOKEN: the project token for the forked moonshot-data repository to allow committing results.
+
+# - MOONSHOT_RUN_COOKBOOKS_LIST: $COOKBOOK_RUN
+#   This variable contains the command to run the cookbooks.
+
+# - MOONSHOT_UPDATE_ENDPOINTS_LIST: $AZURE_OPENAI_4o_UPDATE_ENDPOINT;$TOGETHER_LLAMA3_8B_CHAT_UPDATE_ENDPOINT;$TOGETHER_LLAMA_GUARD_7B_UPDATE_ENDPOINT
+#   This variable contains a semicolon-separated list of commands to update endpoints.
+
+# Ensure these variables are set in your GitLab CI/CD settings or in the .gitlab-ci.yml file.
+.setup_ms_script:
+  script:
+    - |
+      # Install moonshot package with all dependencies
+      echo "Installing moonshot..."
+      pip install "aiverify-moonshot[all]"
+
+      # Clone the forked repository and install its requirements
+      echo "Cloning forked repository and installing requirements..."
+      git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@$GITLAB_MOONSHOT_CI_REPO_LOC
+      pip install -r moonshot-data/requirements.txt
+
+      # Set logging information
+      echo "Setting log info..."
+      export MS_LOG_NAME=moonshot
+      export MS_LOG_LEVEL=debug
+      export MS_LOG_TO_FILE=true
+
+      # Create environment variables file
+      echo "Creating env file..."
+      cat <<EOF >> .env
+      ATTACK_MODULES="./moonshot-data/attack-modules"
+      BOOKMARKS="./moonshot-data/generated-outputs/bookmarks"
+      CONNECTORS="./moonshot-data/connectors"
+      CONNECTORS_ENDPOINTS="./moonshot-data/connectors-endpoints"
+      CONTEXT_STRATEGY="./moonshot-data/context-strategy"
+      COOKBOOKS="./moonshot-data/cookbooks"
+      DATABASES="./moonshot-data/generated-outputs/databases"
+      DATABASES_MODULES="./moonshot-data/databases-modules"
+      DATASETS="./moonshot-data/datasets"
+      IO_MODULES="./moonshot-data/io-modules"
+      METRICS="./moonshot-data/metrics"
+      PROMPT_TEMPLATES="./moonshot-data/prompt-templates"
+      RECIPES="./moonshot-data/recipes"
+      RESULTS="./moonshot-data/generated-outputs/results"
+      RESULTS_MODULES="./moonshot-data/results-modules"
+      RUNNERS="./moonshot-data/generated-outputs/runners"
+      RUNNERS_MODULES="./moonshot-data/runners-modules"
+      TOKENIZERS_PARALLELISM=false
+      HOST_ADDRESS=127.0.0.1
+      HOST_PORT=5000
+      MOONSHOT_UI_CALLBACK_URL=http://localhost:3000/api/v1/benchmarks/status
+      EOF
+
+.run_ms_script:
+  script:
+    - |
+      # Update endpoints based on the provided list
+      echo "Updating endpoints..."
+      IFS=';' read -ra ENDPOINTS <<< "$MOONSHOT_UPDATE_ENDPOINTS_LIST"
+      for cmd in "${ENDPOINTS[@]}"; do
+        echo "Executing: $cmd"
+        eval $cmd
+      done
+
+      # Trigger the cookbook run, which may take some time
+      echo "Triggering cookbook run... This may take some time."
+      IFS=';' read -ra COOKBOOKS <<< "$MOONSHOT_RUN_COOKBOOKS_LIST"
+      for cmd in "${COOKBOOKS[@]}"; do
+        echo "Executing: $cmd"
+        eval $cmd
+      done
+
+      echo "Run completed"
+
+.display_ms_script:
+  script:
+    - |
+      # Define the path to the JSON file containing the results
+      json_file="moonshot-data/generated-outputs/results/$COOKBOOK_RUN_NAME.json"
+
+      # Check if the JSON file exists
+      if [ ! -f "$json_file" ]; then
+        echo "File $json_file not found!"
+        exit 1
+      fi
+
+      # Install jq package for display
+      echo "Installing jq package"
+      apt-get update && apt-get install jq -y
+
+      # Print the table header
+      printf "+------------------+----------------------------------+------------------+-----------------+-------------------+-------+\n"
+      printf "| %-16s | %-32s | %-16s | %-15s | %-17s | %-5s |\n" "cookbook_id" "recipe_id" "model_id" "num_of_prompts" "avg_grade_value" "grade"
+      printf "+==================+==================================+==================+=================+===================+=======+\n"
+
+      # Parse the JSON file and print the results in a formatted table
+      jq -r '
+        .results.cookbooks[] |
+        .id as $cookbook_id |
+        .recipes[] |
+        .id as $recipe_id |
+        .evaluation_summary[] |
+        .model_id as $model_id |
+        .num_of_prompts as $num_of_prompts |
+        .avg_grade_value as $avg_grade_value |
+        .grade as $grade |
+        "\($cookbook_id[0:16]) \($recipe_id[0:32]) \($model_id[0:16]) \($num_of_prompts) \($avg_grade_value | tonumber | . * 100 | round / 100) \($grade)"
+      ' "$json_file" | while read -r cookbook_id recipe_id model_id num_of_prompts avg_grade_value grade; do
+        printf "| %-16s | %-32s | %-16s | %-15s | %-17s | %-5s |\n" "$cookbook_id" "$recipe_id" "$model_id" "$num_of_prompts" "$avg_grade_value" "$grade"
+        printf "+------------------+----------------------------------+------------------+-----------------+-------------------+-------+\n"
+      done
+
+.commit_ms_script:
+  script:
+    - |
+      # Change directory to moonshot-data
+      cd moonshot-data
+      # Configure git user for committing changes
+      git config --global user.email "[email protected]"
+      git config --global user.name "CI Bot"
+      # Add generated output files to the commit
+      git add -f ./generated-outputs/*
+      # Commit the changes with a message including the pipeline ID
+      git commit -m "Update generated files [ci skip] - Pipeline ID ($CI_PIPELINE_ID)"
+      # Push the changes to the repository, skipping CI for this push
+      git push -o ci.skip https://root:$GITLAB_MOONSHOT_PROJECT_TOKEN@$GITLAB_MOONSHOT_CI_REPO_LOC HEAD