Merge branch 'main' into pr-post-lab-link

.github/scripts/labs_post_comments.py

@@ -0,0 +1,78 @@
+"""Build URLs for each Lab page for Labs that have been changed in this PR.
+
+Test each URL to see if it returns a 200 status code, and post the results
+in a PR comment.
+"""
+
+import os
+import sys
+from github import Github
+from pathlib import Path
+
+COMMENT_TITLE_TEMPLATE = "Preview changes to {lab_name} Lab <!--=-->"
+URL_TEMPLATE = (
+    "https://labs.usegalaxy.org.au"
+    "/?content_root=https://github.com/{repo}"
+    "/blob/{branch_name}/{lab_content_path}"
+    "&cache=false"
+)
+TRY_FILES = [
+    'base.yml',
+    'usegalaxy.eu.yml',
+    'usegalaxy.org.yml',
+    'usegalaxy.org.au.yml',
+]
+
+# Environment variables from GitHub Actions
+GITHUB_TOKEN = os.getenv("GITHUB_TOKEN")
+PR_NUMBER = int(os.environ["PR_NUMBER"])
+BASE_REPO = os.getenv("BASE_REPO")
+
+
+def get_comment(pull_request, id_string):
+    """Fetches PR comments and scans for the COMMENT_TITLE_TEMPLATE."""
+    for comment in pull_request.get_issue_comments():
+        if id_string in comment.body:
+            return comment
+    return None
+
+
+def create_or_update_comment(lab_name, body_md):
+    """Creates or updates a comment for the given lab name.
+
+    Checks for an existing comment by looking for the COMMENT_TITLE_TEMPLATE
+    in existing comments.
+    """
+    divider = "\n\n" + '-' * 80 + '\n\n'
+    print("Posting comment:", divider, body_md.strip(' \n'), divider)
+    title_string = COMMENT_TITLE_TEMPLATE.format(lab_name=lab_name)
+    gh = Github(GITHUB_TOKEN)
+    print("Getting base repo:", BASE_REPO)
+    repo = gh.get_repo(BASE_REPO)
+    pull_request = repo.get_pull(PR_NUMBER)
+    comment = get_comment(pull_request, title_string)
+    if comment:
+        comment.edit(body_md)
+    else:
+        pull_request.create_issue_comment(body_md)
+
+
+def main():
+    comments_dir = Path(sys.argv[1] if len(sys.argv) else "comments")
+    if (
+        comments_dir.exists()
+        and comments_dir.is_dir()
+        and list(comments_dir.glob('*.md'))
+    ):
+        for path in comments_dir.glob('*.md'):
+            with open(path) as f:
+                comment_md = f.read()
+            lab_name = path.stem
+            print(f"Posting PR comment for {lab_name}...")
+            create_or_update_comment(lab_name, comment_md)
+    else:
+        print("No comments to post - exiting")
+
+
+if __name__ == "__main__":
+    main()

.github/scripts/post_lab_links.py → .github/scripts/labs_test.py

@@ -5,10 +5,11 @@
 """
 
 import os
-from github import Github
+import sys
 from urllib.request import urlopen
 from urllib.error import URLError, HTTPError
 
+OUTPUT_DIR = "output"
 COMMENT_TITLE_TEMPLATE = "Preview changes to {lab_name} Lab <!--=-->"
 URL_TEMPLATE = (
     "https://labs.usegalaxy.org.au"
@@ -24,57 +25,26 @@
 ]
 
 # Environment variables from GitHub Actions
-PR_NUMBER = int(os.environ["PR_NUMBER"])
 BRANCH_NAME = os.environ["BRANCH_NAME"]
-GITHUB_TOKEN = os.getenv("GITHUB_TOKEN")
-BASE_REPO = os.getenv("BASE_REPO")
 HEAD_REPO = os.getenv("HEAD_REPO")
 
 
-def get_comment(pull_request, id_string):
-    """Fetches PR comments and scans for the COMMENT_TITLE_TEMPLATE."""
-    for comment in pull_request.get_issue_comments():
-        if id_string in comment.body:
-            return comment
-    return None
-
-
-def create_or_update_comment(lab_name, body_md):
-    """Creates or updates a comment for the given lab name.
-
-    Checks for an existing comment by looking for the COMMENT_TITLE_TEMPLATE
-    in existing comments.
-    """
-    divider = "\n\n" + '-' * 80 + '\n\n'
-    print("Posting comment:", divider, body_md.strip(' \n'), divider)
-    title_string = COMMENT_TITLE_TEMPLATE.format(lab_name=lab_name)
-    gh = Github(GITHUB_TOKEN)
-    print("Getting base repo:", BASE_REPO)
-    repo = gh.get_repo(BASE_REPO)
-    pull_request = repo.get_pull(PR_NUMBER)
-    comment = get_comment(pull_request, title_string)
-    if comment:
-        comment.edit(body_md)
-    else:
-        pull_request.create_issue_comment(body_md)
-
-
-def post_lab_links(lab_name):
+def test_lab(lab_name):
     """Iterate through each YAML root file for the given lab name.
     For files that exist, build a URL for that Lab page, check the HTTP status
     code and post the URLs with pass/fail status as a comment on the PR.
     """
     success = True
     title_string = COMMENT_TITLE_TEMPLATE.format(lab_name=lab_name)
-    comment = f"### {title_string}\n\n"
+    comment_md = f"### {title_string}\n\n"
     test_paths = [
         f'communities/{lab_name}/lab/{f}'
         for f in TRY_FILES
     ]
 
     for path in test_paths:
-        if not os.path.exists(path):
-            print(f"Skipping {path}: file not found in repository")
+        if not os.path.exists('head/' + path):
+            print(f"Skipping {path}: file not found in head repo")
             continue
         url = build_url(path)
         try:
@@ -88,19 +58,25 @@ def post_lab_links(lab_name):
         except URLError as exc:
             line = f"- ❌ {filename} [URL ERROR]: {url}\n\n```\n{exc}\n```"
             success = False
-        comment += line
+        comment_md += line
 
     if not success:
-        comment += (
+        comment_md += (
             "\n\n"
             "🚨 One or more Lab pages are returning an error. "
             "Please follow the link to see details of the issue."
         )
 
-    create_or_update_comment(lab_name, comment)
+    write_comment(lab_name, comment_md)
     return success
 
 
+def write_comment(lab_name, md):
+    os.makedirs(OUTPUT_DIR, exist_ok=True)
+    with open(f'{OUTPUT_DIR}/{lab_name}.md', 'w') as f:
+        f.write(md)
+
+
 def http_status_for(url):
     try:
         response = urlopen(url)
@@ -117,7 +93,8 @@ def build_url(content_path):
 
 
 def main():
-    with open("changed_files.txt") as f:
+    path = sys.argv[1] if len(sys.argv) else "paths.txt"
+    with open(path) as f:
         files = f.read().splitlines()
 
     success = True
@@ -129,9 +106,9 @@ def main():
             lab_name = path.split("/")[1]
             if lab_name not in directories:
                 print(f"Detected change to {lab_name} Lab in file: {path}")
-                print(f"Posting link for {lab_name}...")
+                print(f"Testing {lab_name}...")
                 directories.append(lab_name)
-                result = post_lab_links(lab_name)
+                result = test_lab(lab_name)
                 success = success and result
         else:
             print(f"Ignoring changes to {path}: not in a lab directory")

.github/workflows/fetch_filter_resources.yaml

@@ -70,9 +70,34 @@ jobs:
         with:
           name: tools-${{ matrix.subset }}
           path: communities/all/resources/${{ matrix.subset }}_tools.*
+  fetch-tutorials:
+    runs-on: ubuntu-20.04
+    name: Fetch tutorials
+    steps:
+      - name: Checkout main
+        uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+      - name: Install requirement
+        run: |
+          python -m pip install -r requirements.txt
+          sudo apt-get install jq
+      - name: Fetch all tutorials
+        run: | 
+          bash sources/bin/extract_all_tutorials.sh
+        env: 
+          PLAUSIBLE_API_KEY: ${{ secrets.PLAUSIBLE_API_TOKEN }}
+      - name: Archive tutorials artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: tutorials
+          path: communities/all/resources/tutorials.json
   merge-fetch-filter:
     runs-on: ubuntu-20.04
-    needs: fetch-tools-stepwise
+    needs: 
+      - fetch-tools-stepwise
+      - fetch-tutorials
     name: Merge tools, fetch tutorials and filter the resources for communities
     steps:
       - name: Checkout main
@@ -90,6 +115,12 @@ jobs:
           pattern: tools-*
           merge-multiple: true
           path: communities/all/resources/
+      - name: Download stepwise tool lists
+        uses: actions/download-artifact@v4
+        with:
+          pattern: tutorials
+          merge-multiple: true
+          path: communities/all/resources/
       - name: Display structure of downloaded files
         run: ls -R communities/all/resources/
       - name: Merge all tools
@@ -107,11 +138,6 @@ jobs:
       - name: Filter workflows for communities
         run: | 
           bash sources/bin/get_community_workflows.sh
-      - name: Fetch all tutorials
-        run: | 
-          bash sources/bin/extract_all_tutorials.sh
-        env: 
-          PLAUSIBLE_API_KEY: ${{ secrets.PLAUSIBLE_API_TOKEN }}
       - name: Filter tutorials for communities
         run: | 
           bash sources/bin/get_community_tutorials.sh

.github/workflows/labs_post_comments.yml

@@ -0,0 +1,52 @@
+name: Galaxy Labs PR - post comment(s)
+
+on:
+  workflow_run:
+    workflows: ["Test changed Lab pages"]
+    types:
+      - completed
+
+jobs:
+  test-labs:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout base branch
+        uses: actions/checkout@v3
+        with:
+          repository: ${{ github.repository }}
+          ref: main
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.x'
+
+      - name: Install pygithub
+        run: pip install pygithub
+
+      - name: Install GitHub CLI
+        run: sudo apt-get install gh
+
+      - name: Download "Test changed Lab pages" artifact
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh repo set-default galaxyproject/galaxy_codex
+          RUN_ID=$(gh run list --workflow "Test changed Lab pages" --limit 1 | tail -n 1 | grep -oE '\b[0-9]{11}\b')
+          echo "RUN_ID: $RUN_ID"
+          gh run view $RUN_ID --log
+          gh run download $RUN_ID --name labs_test_comments --dir ./labs_test_comments
+
+      - name: Post comment if matching changes found
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}  #${{ secrets.GH_PR_TOKEN }}
+        run: |
+          if ls labs_test_comments/*.md 1> /dev/null 2>&1; then
+            echo "Changed files found"
+            echo "Sourcing environment variables from env.sh:"
+            cat labs_test_comments/env.sh
+            source labs_test_comments/env.sh
+            python3 ./.github/scripts/labs_post_comments.py labs_test_comments
+          else
+            echo "No changed files found"
+          fi