Add test script for tutorial extraction and filter

.github/workflows/run_tests.yaml

@@ -29,11 +29,17 @@ jobs:
           python-version: ${{ matrix.python-version }}
       - name: Install requirement
         run: python -m pip install -r requirements.txt
-      - name: Run script
+      - name: Test tool extraction
         # run: bash bin/extract_all_tools.sh
         run: | 
-          export GITHUB_API_KEY=${{ secrets.GH_API_TOKEN }}
           bash ./bin/extract_all_tools_test.sh "${{ matrix.subset }}"
+        env: 
+          GITHUB_API_KEY: ${{ secrets.GH_API_TOKEN }}
+      - name: Test tutorial extraction and filtering
+        run: | 
+          bash ./bin/extract_filter_tutorial_test.sh
+        env: 
+          PLAUSIBLE_API_KEY: ${{ secrets.PLAUSIBLE_API_TOKEN }}
       - name: Commit all tools 
         # add or commit any changes in results if there was a change, merge with main and push as bot
         run: |

bin/extract_filter_tutorials_test.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+
+mkdir -p 'results/'
+
+tsv_output="results/${1}_tools.tsv"
+json_output="results/${1}_tools.json"
+
+python bin/extract_gtn_tutorials.py \
+        extracttutorials \
+        --all_tutorials "results/test_tutorials.json" \
+        --tools "results/all_tools.json" \
+        --api $PLAUSIBLE_API_KEY \
+        --test
+
+if [[ ! -f "results/test_tutorials.json" ]] ; then
+    echo 'File "results/test_tutorials.json" is not there, aborting.'
+    exit
+fi
+
+python bin/extract_gtn_tutorials.py \
+        filtertutorials \
+        --all_tutorials "results/test_tutorials.json" \
+        --filtered_tutorials "results/microgalaxy/test_tutorials.tsv" \
+        --tags "data/communities/microgalaxy/tutorial_tags"
+
+if [[ ! -f "results/microgalaxy/test_tutorials.tsv" ]] ; then
+    echo 'File "results/microgalaxy/test_tutorials.tsv" is not there, aborting.'
+    exit
+fi
+
+rm "results/test_tutorials.json"
+rm "results/microgalaxy/test_tutorials.tsv"

bin/extract_gtn_tutorials.py

@@ -173,14 +173,16 @@ def get_feedback_per_tutorials() -> Dict:
     return feedback_per_tuto
 
 
-def get_tutorials(tool_fp: str, plausible_api: str) -> List[Dict]:
+def get_tutorials(tool_fp: str, plausible_api: str, run_test: bool,) -> List[Dict]:
     """
     Extract training material from the GTN API, format them, extract EDAM operations from tools, feedback stats, view stats, etc
     """
     tools = shared_functions.read_suite_per_tool_id(tool_fp) 
     feedback = get_feedback_per_tutorials()
     edam_ontology = get_ontology("https://edamontology.org/EDAM_unstable.owl").load()
     topics = get_request_json("https://training.galaxyproject.org/training-material/api/topics.json")
+    if run_test:
+        topics = ["microbiome"]
     tutos = []
     for topic in topics:
         topic_information = get_request_json(f"https://training.galaxyproject.org/training-material/api/topics/{topic}.json")
@@ -292,6 +294,13 @@ def export_tutorials_to_tsv(tutorials: List[Dict], output_fp: str) -> None:
         help="Filepath to JSON with all extracted tools, generated by extractools command",
     )
     extracttutorials.add_argument("--api", "-a", required=True, help="Plausible access token")
+    extracttutorials.add_argument(
+        "--test",
+        action="store_true",
+        default=False,
+        required=False,
+        help="Run a small test case only on one topic",
+    )
 
     # Filter tutorials
     filtertutorials = subparser.add_parser("filtertutorials", help="Filter training materials based on their tags")
@@ -316,7 +325,7 @@ def export_tutorials_to_tsv(tutorials: List[Dict], output_fp: str) -> None:
     args = parser.parse_args()
 
     if args.command == "extracttutorials":
-        tutorials = get_tutorials(args.tools, args.api)
+        tutorials = get_tutorials(args.tools, args.api, args.test)
         shared_functions.export_to_json(tutorials, args.all_tutorials)
 
     elif args.command == "filtertutorials":