Merge pull request #5487 from galaxyproject/create-pull-request/patch…

…-1730275365

[Google Form] New FAQ!

.github/workflows/google-form-events.yml

@@ -48,8 +48,8 @@ jobs:
         uses: peter-evans/create-pull-request@v3
         with:
           commit-message: Import events from Google Form
-          branch-suffix: timestamp
           title: "[Google Form] New Event!"
           add-paths: events/
+          branch: google-form/event
           body: |
              ${{steps.generate.outputs.pr_message}}

.github/workflows/google-form-faqs.yml

@@ -48,6 +48,6 @@ jobs:
         uses: peter-evans/create-pull-request@v3
         with:
           commit-message: Import FAQs from Google Form
-          branch-suffix: timestamp
           title: "[Google Form] New FAQ!"
+          branch: google-form/faq
           add-paths: faqs/

.github/workflows/google-form-news.yml

@@ -48,8 +48,8 @@ jobs:
         uses: peter-evans/create-pull-request@v3
         with:
           commit-message: Import news posts from Google Form
-          branch-suffix: timestamp
           title: "[Google Form] New News Post!"
+          branch: google-form/news
           add-paths: news/_posts/
 
       - name: Update recordings from Google Form
@@ -65,9 +65,9 @@ jobs:
         uses: peter-evans/create-pull-request@v3
         with:
           title: "[Google Form] New Recording Submission!"
-          branch-suffix: timestamp
           commit-message: add new recording from Google Form submission
           add-paths: topics/
+          branch: google-form/recordings
           committer: "Saskia Hiltemann <[email protected]>"
           author: "Saskia Hiltemann <[email protected]>"
           body: |

.github/workflows/google-form-recordings.yml

@@ -48,11 +48,11 @@ jobs:
         uses: peter-evans/create-pull-request@v3
         with:
           title: "[Google Form] New Recording Submission!"
-          branch-suffix: timestamp
           commit-message: add new recording from Google Form submission
           add-paths: topics/
           committer: "Saskia Hiltemann <[email protected]>"
           author: "Saskia Hiltemann <[email protected]>"
+          branch: google-form/recordings
           body: |
             New recording submitted!
 

CONTRIBUTORS.yaml

@@ -2397,6 +2397,10 @@ TanguyGen:
     name: Tanguy Genthon
     joined: 2023-06
 
+thanhleviet:
+    name: Thanh Le Viet
+    joined: 2024-10
+
 themudo:
     name: Gonçalo Themudo
     joined: 2021-10

_plugins/api.rb

@@ -106,7 +106,6 @@ def generateLibrary(site)
       page2 = PageWithoutAFile.new(site, '', 'api/', 'data-library.yaml')
       data_libraries = Dir.glob('topics/**/data-library.yaml')
       data_libraries.map! { |x| YAML.load_file(x) }
-      pp data_libraries
       page2.content = JSON.pretty_generate(Gtn::Git.discover)
       page2.data['layout'] = nil
       site.pages << page2
@@ -209,7 +208,6 @@ def generate(site)
         out = site.data[topic].dup
         out['materials'] = TopicFilter.topic_filter(site, topic).map do |x|
           q = x.dup
-          puts q['url']
           q['contributors'] = Gtn::Contributors.get_contributors(q).dup.map do |c|
             mapContributor(site, c)
           end

bin/google-form-faq.rb

@@ -11,6 +11,33 @@
 url = 'https://docs.google.com/spreadsheets/d/1RFF3G9_bP8EpfACBk8lnF-Ib43ZGGAMm3ewPwW7eFT0/export?format=tsv'
 data = `curl -sL "#{url}"`
 
+# We have the problem of renamed FAQs, these will be stored somewhere other
+# than the auto-generated name.
+#
+# We need to eliminate that duplication and ensure we don't overwrite an existing one.
+#
+# So we need to discover all existing FAQs and their google form IDs.
+
+def self.discover_faqs
+  paths = []
+  paths += Dir.glob('faqs/**/*.md')
+  paths += Dir.glob('topics/**/faqs/*.md')
+  paths += Dir.glob('topics/**/faqs/*.md')
+
+  # Reject symlinks
+  paths.reject { |path| File.symlink?(path) }
+end
+
+faqs = discover_faqs.map do |path|
+  metadata = safe_load_yaml(path)
+  if metadata.is_a?(String)
+    next
+  end
+
+  [metadata['google_form_id'], path]
+end.reject{|x, y| x.nil?}.to_h
+
+# The google form data.
 data = CSV.parse(data, col_sep: "\t", headers: true)
 count = 0
 
@@ -20,6 +47,12 @@
   post_date = DateTime.strptime(row['Timestamp'], '%d/%m/%Y %H:%M:%S')
   filename = "faqs/#{row['This FAQ Concerns'].downcase}/#{row['Title'].downcase.gsub(/[^a-z0-9\s-]/i, '').gsub(/\s+/, ' ').gsub(/ /, '-')}.md"
 
+  google_form_id = post_date.to_time.to_i
+  if faqs.include?(google_form_id)
+    STDERR.puts "Already imported as #{faqs[google_form_id]}"
+    next
+  end
+
   # Skip some testing posts
   if (row['Title'] == 'TESTING')
     STDERR.puts "Skipping #{filename} as it is a test post"
@@ -29,11 +62,11 @@
   # Don't overwrite existing posts
   if File.exist?(filename)
     other_file = safe_load_yaml(filename)
-    if other_file['google_form_id'] == post_date.to_time.to_i
+    if other_file['google_form_id'] == google_form_id
       STDERR.puts "Skipping #{filename} as it already exists"
       next
     else
-      filename = filename.gsub('.md', "-#{post_date.to_time.to_i}.md")
+      filename = filename.gsub('.md', "-#{google_form_id}.md")
     end
   end
 

faqs/galaxy/importing-data-from-sierra-lims.md

@@ -0,0 +1,19 @@
+---
+title: Importing data from Sierra LIMS
+layout: faq
+area: data upload
+box_type: tip
+google_form_id: 1730224450
+contributors: [hexhowells]
+---
+
+This section will guide you through generating external links to your data stored in the Sierra LIMS system to be downloaded directly into Galaxy.
+
+1. Go to the [Sierra portal](https://www.bioinformatics.babraham.ac.uk/sierra/sierra.pl) and login to your account.
+2. Click on the **Sample ID** of the sample you want to download data from.
+3. Click on the **Edit Sample Details** button.
+4. At the bottom of the page there will be an input box for creating a link, enter a description for the link in the **Reason for link** section, and click **Create link**. This will reload the page and add a new link to the sample under **Authorised links to this sample**.
+5. Go back to the sample page or click on the hyperlink called **link** to take you back.
+6. In the **Results** section select the lane you want to access your data from.
+7. The bottom of the page, under the **Links** section, will now contain a list of `wget` commands with links for accessing all the files within that sample/lane.
+8. Since this list is for `wget` commands, you need to extract out the links from the command. You can copy the link in the first set of double quotes for each line and {% icon galaxy-wf-edit %} **Paste/Fetch Data** them directly into Galaxy to download the files.

faqs/galaxy/request-galaxy-tools-on-a-specific-server.md

@@ -0,0 +1,15 @@
+---
+title: Request Galaxy tools on a specific server
+layout: faq
+area: tools
+box_type: tip
+google_form_id: 1728577286
+contributors: [nomadscientist]
+---
+To request tools that already exist in the [Galaxy toolshed](https://toolshed.g2.bx.psu.edu), but not in your server, please raise an issue at:
+
+- *Europe - usegalaxy.eu* | https://github.com/usegalaxy-eu/usegalaxy-eu-tools
+
+- *USA - usegalaxy.org* | https://github.com/galaxyproject/usegalaxy-tools
+
+- *Australia - usegaalxy.org.au* | https://github.com/usegalaxy-au/usegalaxy-au-tools/tree/master/usegalaxy.org.au

faqs/gtn/using-answer-key-histories.md

@@ -0,0 +1,20 @@
+---
+title: Using answer key histories
+area: learners
+layout: faq
+box_type: tip
+google_form_id: 1728648609
+contributors: [nomadscientist]
+---
+
+If you are struggling with a tutorial and you can't figure out why, you can use the answer history (if it's available) which is linked at the top of the tutorial.
+
+1. Select the *Answer history* and choose the appropriate link from the resulting drop-down menu.
+
+2. Import the history (If you aren't sure how, [go to the Import History FAQ]({% link faqs/galaxy/histories_import.md %})) 3. Select *View Here* or otherwise navigate to the imported history. 4. Find the dataset that was not outputting for you. You might be able to sport errors from looking at this dataset - such as, you may see that a different tool version was used, or different parameters were run, or different inputs were used. You may find that the input dataset itself is different, and perhaps something has gone wrong earlier in the tutorial for you.
+
+ If you can't find what has gone wrong, you can then: 5. [Ask for help] ({% link faqs/gtn/instructors_getting_help.md %}).
+
+6. [Copy the necessary datasets over from the answer history into your history]({% link faqs/galaxy/histories_copy_dataset.md %}) so that you can continue your analysis from wherever you got stuck!
+
+<!-- It would be awesome if this was automatically added to any tutorial that has answer key histories -->

topics/admin/faqs/galaxy-jobs-in-containers.md

@@ -0,0 +1,123 @@
+---
+title: How can I get my container requiring jobs to run in a container?
+area: galaxy
+box_type: tip
+layout: faq
+contributors: [hexylena, thanhleviet]
+---
+
+Some tools will only run in a container, i.e. they have a container defined in the 'requirements' section of the tool's XML file.
+Galaxy will not refuse to run these tools if the container isn't available or if Galaxy isn't configured use containers. Instead it'll run in the host system and likely fail.
+
+## Job Configuration
+
+You can resolve this by configuring your job conf to have destinations that support containers (or even require them.):
+
+The destination must have `docker_enabled` (Or `singularity_enabled`), and you can consider adding `require_container` to make sure the job will fail if the container isn't available. The `docker_volumes` string will allow you to control which volumes are attached to that container;
+
+In TPV configuration (provided by @gtn:thanhleviet) this would look like:
+
+```yaml
+  docker:
+    inherits: slurm
+    scheduling:
+      require:
+        - docker
+    params:
+      docker_enabled: true
+      require_container: true
+
+  singularity:
+    inherits: slurm
+    scheduling:
+      require:
+        - singularity
+    params:
+      singularity_enabled: true
+
+  podman:
+    inherits: slurm
+    scheduling:
+      require:
+        - podman
+    params:
+      docker_enabled: true
+      require_container: true
+      docker_volumes: "$galaxy_root:ro,$tool_directory:ro,$job_directory:ro,$working_directory:z,$default_file_path:z"
+      docker_sudo: false
+      docker_cmd: /usr/bin/podman
+      docker_run_extra_arguments: "--userns=keep-id"
+```
+
+Or in XML:
+
+```xml
+<destination id="docker" runner="local">
+    <param id="docker_enabled">true</param>
+    <param id="require_container">true</param>
+</destionation>
+
+<destination id="podman" runner="local">
+    <param id="docker_enabled">true</param>
+    <param id="require_container">true</param>
+    <param id="docker_sudo">false</param>
+    <param id="docker_cmd">/usr/bin/podman</param>
+    <param id="docker_run_extra_arguments">--userns=keep-id</param>
+    <!-- This will not work until https://github.com/galaxyproject/galaxy/pull/18998 is merged for SELinux users. For now you may want to patch it manually. -->
+    <!-- <param id="docker_volumes">$galaxy_root:ro,$tool_directory:ro,$job_directory:ro,$working_directory:z,$default_file_path:z</param> -->
+</destination>
+
+<destination id="singularity" runner="local">
+    <param id="singularity_enabled">true</param>
+    <param id="require_container">true</param>
+</destionation>
+```
+
+
+Configuring a tool to use this destination would look like:
+
+```yaml
+toolshed.g2.bx.psu.edu/repos/thanhlv/metaphlan4/metaphlan4/4.0.3:
+    cores: 12
+    mem: cores * 8
+    params:
+      singularity_enabled: true
+```
+
+Or in XML:
+
+```xml
+<tools>
+    <tool id="toolshed.g2.bx.psu.edu/repos/thanhlv/metaphlan4/metaphlan4/4.0.3" destination="docker"/>
+</tools>
+```
+
+## Container Resolvers Configuration
+
+If you're using the default `container_resolvers_conf.yml` then there is nothing you need to do. Otherwise you may want to ensure that you have items in there such as `explicit` and `explicit_singularity` among others. See the [galaxy documentation](https://docs.galaxyproject.org/en/master/admin/container_resolvers.html) on the topic.
+
+## Testing
+
+Here is an example of a tool that requires a container, that you can use to test your container configuration:
+
+```xml
+<tool name="container-test" id="container" version="5.0" profile="21.09">
+        <requirements>
+                <container type="docker">ubuntu:22.04</container>
+        </requirements>
+        <command><![CDATA[
+                pwd >> '$output';
+                hostname -f >> '$output';
+                echo "" >> '$output';
+                cat /etc/os-release >> '$output';
+                echo "" >> '$output';
+                env | sort >> '$output';
+]]></command>
+        <inputs>
+        </inputs>
+        <outputs>
+                <data name="output" format="txt" label="log" />
+        </outputs>
+        <help><![CDATA[]]></help>
+</tool>
+```

topics/admin/faqs/why-isnt-my-history-updating.md

@@ -0,0 +1,49 @@
+---
+title: Why isn't my history updating?
+layout: faq
+area: debugging
+box_type: tip
+google_form_id: 1729851011
+contributors: [hexylena]
+---
+Have you ever experienced that you would submit a job but your history wouldn't update? Maybe it doesn't scroll or the datasets stay permanently grey even when you know they should be complete, *until you refresh the webpage*?
+
+One possible cause of this can be a difference in the clocks of your browser and the server. Check that your clocks match, and if not, reconfigure them! If you are following the Galaxy Admin Training, you will have setup `chrony`. Check that your `chrony` configuration is valid and requesting time from a local pool.
+
+
+```
+# chronyc -n sources
+210 Number of sources = 1
+MS Name/IP address         Stratum Poll Reach LastRx Last sample               
+===============================================================================
+^? 169.254.169.123               0   7     0     -     +0ns[   +0ns] +/-    0ns
+
+```
+
+This command should return some valid sources. THe above shows an example of a time source that isn't working, 0ns is not a realistic office and LastRx is empty. Instead it should look more like::
+
+```
+# chronyc -n sources
+210 Number of sources = 5
+MS Name/IP address         Stratum Poll Reach LastRx Last sample               
+===============================================================================
+^? 169.254.169.123               0   6     0     -     +0ns[   +0ns] +/-    0ns
+^? 178.239.19.58                 0   6     0     -     +0ns[   +0ns] +/-    0ns
+^? 194.104.0.153                 2   6     1     0   +138us[ +138us] +/-   30ms
+^? 45.138.55.61                  1   6     1     1   -103us[ -103us] +/- 3158us
+^? 178.239.19.57                 2   6     1     1   -301us[ -301us] +/- 3240us
+```
+
+Here we see a number of sources, with more plausible offsets and non-empty LastRx.
+
+If your time was misconfigured, you might now see something like:
+
+```
+# chronyc -n tracking
+Reference ID    : B950F724 (185.80.247.36)
+Stratum         : 2
+Ref time (UTC)  : Tue Oct 22 09:44:29 2024
+System time     : 929.234680176 seconds slow of NTP time
+```
+
+as chrony slowly adjusts the system clock to match NTP time.