Automatic CI to fetch, merge, filter tools and create static pages

.github/workflows/fetch_all_tools.yaml

@@ -0,0 +1,75 @@
+name: Fetch all tools
+
+on:
+  workflow_dispatch:
+  schedule:
+    #Every Sunday at 8:00 am
+    - cron: "0 8 * * 0" 
+
+# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
+# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
+concurrency:
+  group: "tools"
+  cancel-in-progress: false
+
+permissions:
+  contents: write
+
+jobs:
+  fetch-all-tools-stepwise:
+    runs-on: ubuntu-20.04
+    environment: fetch-tools
+    name: Fetch all tool stepwise
+    strategy:
+      matrix:
+        python-version: [3.8]
+        subset:
+         - repositories01.list
+         - repositories02.list
+         - repositories03.list
+         - repositories04.list
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install requirement
+        run: python -m pip install -r requirements.txt
+      - name: Run script
+        run: | 
+          export GITHUB_API_KEY=${{ secrets.GH_API_TOKEN }}
+          bash ./bin/extract_all_tools_stepwise.sh "${{ matrix.subset }}"
+      - name: Commit all tools 
+        # add or commit any changes in results if there was a change, merge with main and push as bot
+        run: |
+          git config user.name github-actions
+          git config user.email [email protected]
+          git add results
+          git status
+          git diff --quiet && git diff --staged --quiet || (git commit -m "fetch all tools bot  - step fetch") 
+          git pull -s recursive -X ours
+          git push
+
+  fetch-all-tools-merge:
+    runs-on: ubuntu-20.04
+    needs: fetch-all-tools-stepwise
+    name: Fetch all tools merge
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+      - name: Install requirement
+        run: python -m pip install -r requirements.txt
+      - name: Run script
+        run: | 
+          cat results/repositories*.list_tools.tsv > results/all_tools.tsv
+          bash ./bin/extract_all_tools_downstream.sh
+      - name: Commit all tools
+        # add or commit any changes in results if there was a change, merge with main and push as bot
+        run: |
+          git config user.name github-actions
+          git config user.email [email protected]
+          git add results
+          git status
+          git diff --quiet && git diff --staged --quiet || (git commit -m "fetch all tools bot  - step merge") 
+          git pull -s recursive -X ours
+          git push

.github/workflows/filter_communities.yaml

@@ -0,0 +1,38 @@
+name: Filter community tools
+
+on:
+  workflow_dispatch:
+
+  # the workflow it triggered when all_tools_tsv is changed
+  push:
+    paths:
+    - 'results/all_tools_tsv'
+    branches: ["main"]
+
+# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
+# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
+concurrency:
+  group: "filter"
+  cancel-in-progress: false
+
+permissions:
+  contents: write
+
+jobs:
+  filter-all-tools:
+    runs-on: ubuntu-20.04
+    steps:
+    - uses: actions/checkout@v3
+    - uses: actions/setup-python@v4
+    - name: Install requirement
+      run: python -m pip install -r requirements.txt
+    - name: Run script
+      run: | 
+        bash ./bin/get_community_tools.sh
+    - name: Commit results
+      # commit the new filtered data, only if stuff was changed
+      run: |
+        git config user.name github-actions
+        git config user.email [email protected]
+        git diff --quiet || (git add results && git commit -m "filter communities bot") 
+        git push

.github/workflows/run_tests.yaml

@@ -0,0 +1,66 @@
+name: Run tests
+
+on:
+  workflow_dispatch:
+
+# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
+# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
+# concurrency:
+#   group: "tools"
+#   cancel-in-progress: false
+
+permissions:
+  contents: write
+
+jobs:
+  fetch-all-tools-stepwise:
+    runs-on: ubuntu-20.04
+    environment: fetch-tools
+    name: Fetch all tool stepwise
+    strategy:
+      matrix:
+        python-version: [3.8]
+        subset:
+         - repositories02.list
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install requirement
+        run: python -m pip install -r requirements.txt
+      - name: Run script
+        # run: bash bin/extract_all_tools.sh
+        run: | 
+          export GITHUB_API_KEY=${{ secrets.GH_API_TOKEN }}
+          bash ./bin/extract_all_tools_test.sh "${{ matrix.subset }}"
+      - name: Commit all tools 
+        # add or commit any changes in results if there was a change, merge with main and push as bot
+        run: |
+          git config user.name github-actions
+          git config user.email [email protected]
+          git add results
+          git status
+          git diff --quiet && git diff --staged --quiet || (git commit -m "fetch all tools bot  - step fetch") 
+          git pull -s recursive -X ours
+          git push
+
+  # fetch-all-tools-merge:
+  #   runs-on: ubuntu-20.04
+  #   needs: fetch-all-tools-stepwise
+  #   name: Fetch all tools merge
+  #   steps:
+  #     - uses: actions/checkout@v3
+  #     - uses: actions/setup-python@v4
+  #     - name: Install requirement
+  #       run: python -m pip install -r requirements.txt
+  #     - name: Run script
+  #       run: | 
+  #         cat results/repositories*.list_tools.tsv > results/all_tools.tsv
+  #         bash ./bin/extract_all_tools_downstream.sh
+  #     - name: Commit all tools
+  #       run: |
+  #         git config user.name github-actions
+  #         git config user.email [email protected]
+  #         git diff --quiet || (git add results && git commit -m "fetch all tools bot  - step merge") 
+  #         git push

.github/workflows/static.yml

@@ -2,10 +2,13 @@
 name: Deploy static content to Pages
 
 on:
-  # Runs on pushes targeting the default branch
+  # the workflow is triggered only when results are changed
   push:
+    paths:
+    - 'results'
     branches: ["main"]
 
+
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
 

bin/extract_all_tools_downstream.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+mkdir -p 'results/'
+
+python bin/create_interactive_table.py \
+        --table "results/all_tools.tsv" \
+        --template "data/interactive_table_template.html" \
+        --output "results/index.html"

bin/extract_all_tools_stepwise.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+mkdir -p 'results/'
+
+output="results/${1}_tools.tsv"
+
+python bin/extract_galaxy_tools.py \
+        extractools \
+        --api $GITHUB_API_KEY \
+        --all_tools $output \
+        --planemorepository $1
+

bin/extract_all_tools_test.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+mkdir -p 'results/'
+
+output="results/${1}_tools.tsv"
+
+python bin/extract_galaxy_tools.py \
+        extractools \
+        --api $GITHUB_API_KEY \
+        --all_tools $output \
+        --planemorepository $1 \
+        --test
+

bin/extract_galaxy_tools.py

@@ -57,18 +57,36 @@ def get_string_content(cf: ContentFile) -> str:
     return base64.b64decode(cf.content).decode("utf-8")
 
 
-def get_tool_github_repositories(g: Github) -> List[str]:
+def get_tool_github_repositories(g: Github, RepoSelection: Optional[str], run_test: bool) -> List[str]:
     """
     Get list of tool GitHub repositories to parse
 
     :param g: GitHub instance
+    :param RepoSelection: The selection to use from the repository (needed to split the process for CI jobs)
+    :run_test: for CI testing only use one repository
     """
+
+    if run_test:
+        return ["https://github.com/TGAC/earlham-galaxytools"]
+
     repo = g.get_user("galaxyproject").get_repo("planemo-monitor")
     repo_list: List[str] = []
     for i in range(1, 5):
-        repo_f = repo.get_contents(f"repositories0{i}.list")
-        repo_l = get_string_content(repo_f).rstrip()
-        repo_list.extend(repo_l.split("\n"))
+        repo_selection = f"repositories0{i}.list"
+        if RepoSelection:  # only get these repositories
+            if RepoSelection == repo_selection:
+                repo_f = repo.get_contents(repo_selection)
+                repo_l = get_string_content(repo_f).rstrip()
+                repo_list.extend(repo_l.split("\n"))
+        else:
+            repo_f = repo.get_contents(repo_selection)
+            repo_l = get_string_content(repo_f).rstrip()
+            repo_list.extend(repo_l.split("\n"))
+
+    print("Parsing repositories from:")
+    for repo in repo_list:
+        print("\t", repo)
+
     return repo_list
 
 
@@ -475,11 +493,20 @@ def filter_tools(
     # Extract tools
     extractools = subparser.add_parser("extractools", help="Extract tools")
     extractools.add_argument("--api", "-a", required=True, help="GitHub access token")
+
+    extractools.add_argument("--all_tools", "-o", required=True, help="Filepath to TSV with all extracted tools")
     extractools.add_argument(
-        "--all_tools",
-        "-o",
-        required=True,
-        help="Filepath to TSV with all extracted tools",
+        "--planemorepository", "-pr", required=False, help="Repository list to use from the planemo-monitor repository"
+    )
+
+    extractools.add_argument(
+        "--test",
+        "-t",
+        action="store_true",
+        default=False,
+        required=False,
+        help="Run a small test case using only the repository: https://github.com/TGAC/earlham-galaxytools",
+
     )
 
     # Filter tools
@@ -517,12 +544,13 @@ def filter_tools(
         # connect to GitHub
         g = Github(args.api)
         # get list of GitHub repositories to parse
-        repo_list = get_tool_github_repositories(g)
+
+        repo_list = get_tool_github_repositories(g, args.planemorepository, args.test)
 
         # parse tools in GitHub repositories to extract metada, filter by TS categories and export to output file
         tools: List[Dict] = []
         for r in repo_list:
-            print(r)
+            print("Parsing tools from:", (r))
             if "github" not in r:
                 continue
             try: