diff --git a/ci-examples/.gitlab/.example-gitlab-ci.yml b/ci-examples/.gitlab/.example-gitlab-ci.yml new file mode 100644 index 0000000..c3743b9 --- /dev/null +++ b/ci-examples/.gitlab/.example-gitlab-ci.yml @@ -0,0 +1,18 @@ +include: + - local: ms-gitlab-ci.yml + +stages: + - run-moonshot + +run-job: + stage: run-moonshot + image: python:3.11 + script: + - !reference [.setup_ms_script, script] + - !reference [.run_ms_script, script] + - !reference [.display_ms_script, script] + - !reference [.commit_ms_script, script] + artifacts: + paths: + - moonshot.log + - moonshot-data/generated-outputs/results/ \ No newline at end of file diff --git a/ci-examples/.gitlab/ms-gitlab-ci.yml b/ci-examples/.gitlab/ms-gitlab-ci.yml new file mode 100644 index 0000000..80fc001 --- /dev/null +++ b/ci-examples/.gitlab/ms-gitlab-ci.yml @@ -0,0 +1,137 @@ +# The following GitLab CI/CD variables are required for the pipeline to work: +# Example and explanation of the new variables: +# - COOKBOOK_RUN: "python -m moonshot cli run_cookbook '$COOKBOOK_RUN_NAME' \"['common-risk-easy','common-risk-hard','mlc-ai-safety']\" \"['azure-openai-gpt4o']\" -n 1" +# This variable should contain the command to run the cookbook. + +# - COOKBOOK_RUN_NAME: testcookbookrun-$CI_PIPELINE_ID +# This variable contains the name of the cookbook run, which includes the pipeline ID. + +# - GITLAB_MOONSHOT_CI_REPO_LOC: gitlab.com/lionel_teo/moonshot-data +# This variable contains the location of the forked repository. + +# - GITLAB_MOONSHOT_PROJECT_TOKEN: the project token for the forked moonshot-data repository to allow committing results. + +# - MOONSHOT_RUN_COOKBOOKS_LIST: $COOKBOOK_RUN +# This variable contains the command to run the cookbooks. + +# - MOONSHOT_UPDATE_ENDPOINTS_LIST: $AZURE_OPENAI_4o_UPDATE_ENDPOINT;$TOGETHER_LLAMA3_8B_CHAT_UPDATE_ENDPOINT;$TOGETHER_LLAMA_GUARD_7B_UPDATE_ENDPOINT +# This variable contains a semicolon-separated list of commands to update endpoints. + +# Ensure these variables are set in your GitLab CI/CD settings or in the .gitlab-ci.yml file. +.setup_ms_script: + script: + - | + # Install moonshot package with all dependencies + echo "Installing moonshot..." + pip install "aiverify-moonshot[all]" + + # Clone the forked repository and install its requirements + echo "Cloning forked repository and installing requirements..." + git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@$GITLAB_MOONSHOT_CI_REPO_LOC + pip install -r moonshot-data/requirements.txt + + # Set logging information + echo "Setting log info..." + export MS_LOG_NAME=moonshot + export MS_LOG_LEVEL=debug + export MS_LOG_TO_FILE=true + + # Create environment variables file + echo "Creating env file..." + cat <> .env + ATTACK_MODULES="./moonshot-data/attack-modules" + BOOKMARKS="./moonshot-data/generated-outputs/bookmarks" + CONNECTORS="./moonshot-data/connectors" + CONNECTORS_ENDPOINTS="./moonshot-data/connectors-endpoints" + CONTEXT_STRATEGY="./moonshot-data/context-strategy" + COOKBOOKS="./moonshot-data/cookbooks" + DATABASES="./moonshot-data/generated-outputs/databases" + DATABASES_MODULES="./moonshot-data/databases-modules" + DATASETS="./moonshot-data/datasets" + IO_MODULES="./moonshot-data/io-modules" + METRICS="./moonshot-data/metrics" + PROMPT_TEMPLATES="./moonshot-data/prompt-templates" + RECIPES="./moonshot-data/recipes" + RESULTS="./moonshot-data/generated-outputs/results" + RESULTS_MODULES="./moonshot-data/results-modules" + RUNNERS="./moonshot-data/generated-outputs/runners" + RUNNERS_MODULES="./moonshot-data/runners-modules" + TOKENIZERS_PARALLELISM=false + HOST_ADDRESS=127.0.0.1 + HOST_PORT=5000 + MOONSHOT_UI_CALLBACK_URL=http://localhost:3000/api/v1/benchmarks/status + EOF + +.run_ms_script: + script: + - | + # Update endpoints based on the provided list + echo "Updating endpoints..." + IFS=';' read -ra ENDPOINTS <<< "$MOONSHOT_UPDATE_ENDPOINTS_LIST" + for cmd in "${ENDPOINTS[@]}"; do + echo "Executing: $cmd" + eval $cmd + done + + # Trigger the cookbook run, which may take some time + echo "Triggering cookbook run... This may take some time." + IFS=';' read -ra COOKBOOKS <<< "$MOONSHOT_RUN_COOKBOOKS_LIST" + for cmd in "${COOKBOOKS[@]}"; do + echo "Executing: $cmd" + eval $cmd + done + + echo "Run completed" + +.display_ms_script: + script: + - | + # Define the path to the JSON file containing the results + json_file="moonshot-data/generated-outputs/results/$COOKBOOK_RUN_NAME.json" + + # Check if the JSON file exists + if [ ! -f "$json_file" ]; then + echo "File $json_file not found!" + exit 1 + fi + + # Install jq package for display + echo "Installing jq package" + apt-get update && apt-get install jq -y + + # Print the table header + printf "+------------------+----------------------------------+------------------+-----------------+-------------------+-------+\n" + printf "| %-16s | %-32s | %-16s | %-15s | %-17s | %-5s |\n" "cookbook_id" "recipe_id" "model_id" "num_of_prompts" "avg_grade_value" "grade" + printf "+==================+==================================+==================+=================+===================+=======+\n" + + # Parse the JSON file and print the results in a formatted table + jq -r ' + .results.cookbooks[] | + .id as $cookbook_id | + .recipes[] | + .id as $recipe_id | + .evaluation_summary[] | + .model_id as $model_id | + .num_of_prompts as $num_of_prompts | + .avg_grade_value as $avg_grade_value | + .grade as $grade | + "\($cookbook_id[0:16]) \($recipe_id[0:32]) \($model_id[0:16]) \($num_of_prompts) \($avg_grade_value | tonumber | . * 100 | round / 100) \($grade)" + ' "$json_file" | while read -r cookbook_id recipe_id model_id num_of_prompts avg_grade_value grade; do + printf "| %-16s | %-32s | %-16s | %-15s | %-17s | %-5s |\n" "$cookbook_id" "$recipe_id" "$model_id" "$num_of_prompts" "$avg_grade_value" "$grade" + printf "+------------------+----------------------------------+------------------+-----------------+-------------------+-------+\n" + done + +.commit_ms_script: + script: + - | + # Change directory to moonshot-data + cd moonshot-data + # Configure git user for committing changes + git config --global user.email "ci@example.com" + git config --global user.name "CI Bot" + # Add generated output files to the commit + git add -f ./generated-outputs/* + # Commit the changes with a message including the pipeline ID + git commit -m "Update generated files [ci skip] - Pipeline ID ($CI_PIPELINE_ID)" + # Push the changes to the repository, skipping CI for this push + git push -o ci.skip https://root:$GITLAB_MOONSHOT_PROJECT_TOKEN@$GITLAB_MOONSHOT_CI_REPO_LOC HEAD \ No newline at end of file