initial workflow for updating test results

.github/workflows/update-test-results.yml

@@ -0,0 +1,37 @@
+name: update-test-results
+on:
+  push:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 0 * * *"
+permissions:
+  actions: read
+jobs:
+  update-kornia:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        file : [ test_color, test_contrib ]
+        compile : [ F ]  # to also run the tests with backend_compilation, change this to: T, F
+    steps:
+      - name: Get Job URL
+        uses: Tiryoh/gha-jobid-action@v0
+        id: jobs
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          job_name: ${{ github.job }} (${{ matrix.file }}, ${{ matrix.compile }})
+          per_page: 64
+
+      - name: Run Tests
+        id: tests
+        run: |
+          cd ivy
+          pip3 install pymongo
+          pip3 install pytest-json-report
+          echo "html_url: ${{ steps.jobs.outputs.html_url }}"
+          if [ "${{ steps.jobs.outputs.html_url }}" = "null" ]; then
+            docker run --rm -v "$(pwd)":/ivy -v "$(pwd)"/.hypothesis:/.hypothesis transpileai/ivy:latest update_tests.sh kornia ${{ matrix.file }} ${{ matrix.compile }} https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} ${{ secrets.INTEGRATION_TESTS_DB_KEY }}
+          else
+            docker run --rm -v "$(pwd)":/ivy -v "$(pwd)"/.hypothesis:/.hypothesis transpileai/ivy:latest update_tests.sh kornia ${{ matrix.file }} ${{ matrix.compile }} ${{ steps.jobs.outputs.html_url }} ${{ secrets.INTEGRATION_TESTS_DB_KEY }}
+          fi

update_db.py

@@ -0,0 +1,65 @@
+"""
+Reads in a test_report.json which has been created by pytest and adds/replaces information
+for each test that has been run in the ivy integrations testing remote db.
+"""
+
+import argparse
+import json
+from pymongo import MongoClient
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Add all the tests within test_report.json to the remote MongoDB."
+    )
+    parser.add_argument(
+        "--workflow-link",
+        type=str,
+        help="Link to the GitHub actions workflow corresponding to this test.",
+    )
+    parser.add_argument("--db-key", type=str, help="Key for the MongoDB database")
+
+    args = parser.parse_args()
+    json_report_file = "test_report.json"
+
+    # load in the test report
+    with open(json_report_file, "r") as file:
+        data = json.load(file)
+    tests_data = data.get("tests", None)
+
+    # connect to the database
+    uri = f"mongodb+srv://{args.integration_test_db_key}@ivy-integration-tests.pkt1la7.mongodb.net/?retryWrites=true&w=majority&appName=ivy-integration-tests"
+    client = MongoClient(uri)
+    db = client.ivytestdashboard
+    collection = db["test_results"]
+
+    # upload the information for each individual test ran to the database
+    for test in tests_data:
+        test_path, test_function_args = test["nodeid"].split("::")
+        split_path = test_path.split("/", 1)
+        integration = split_path[0]
+        test_file = split_path[1]
+
+        _, test_args = test_function_args.split("[")
+        test_args = test_args.replace("]", "")
+        test_args = test_args.split("-")
+
+        target, mode, backend_compile = test_args
+        integration_fn = test["call"]["stdout"].replace("\n", "")
+
+        document = {
+            "target": target,
+            "mode": mode,
+            "backend_compile": backend_compile,
+            "function": integration_fn,
+            "workflow_link": args.workflow_link,
+            "outcome": test["outcome"],
+        }
+        filter_criteria = {
+            "target": target,
+            "mode": mode,
+            "backend_compile": backend_compile,
+            "function": integration_fn,
+        }
+
+        result = collection.replace_one(filter_criteria, document, upsert=True)

update_tests.sh

@@ -0,0 +1,24 @@
+#!/bin/bash -e
+
+integration=$1
+file=$2
+backend_compile=$3
+workflow_link=$4
+db_key=$5
+
+set +e
+if $backend_compile -eq "T"; then
+    pytest $integration/$submodule.py --backend-compile -p no:warnings --tb=short --json-report --json-report-file=test_report.json
+    pytest_exit_code=$?
+else
+    pytest $integration/$submodule.py -p no:warnings --tb=short --json-report --json-report-file=test_report.json
+    pytest_exit_code=$?
+fi
+set -e
+
+if [ $pytest_exit_code -eq 0 ] || [ $pytest_exit_code -eq 1 ]; then
+    python update_db.py --workflow-link $workflow_link --db-key $db_key
+    exit 0
+else
+    exit 1
+fi