Revert "Create a redirect loop to usegalaxy.eu (urgent)"

env/common/files/admin/tmpwatch-auto

@@ -6,6 +6,17 @@ set -euo pipefail
 
 DIR="$1"
 PERCENT=
+declare -a EXCLUDE_ARGS
+
+case "$DIR" in
+    */main/*)
+        PULSAR_PERSISTED_DATA='/srv/pulsar/main/var/persisted_data'
+        ;;
+    */test/*)
+        PULSAR_PERSISTED_DATA='/srv/pulsar/test/var/persisted_data'
+        ;;
+esac
+
 
 : ${DEBUG:=false}
 
@@ -65,12 +76,23 @@ function hours() {
 }
 
 
+function exclude_args() {
+    local exclude_id
+    local exclude_ids=($(find ${PULSAR_PERSISTED_DATA}/jetstream2-{active,preprocessing}-jobs -maxdepth 1 -type f -printf "%f\n"))
+    # this could exceed ARG_MAX but these days it's 2 million so probably not
+    for exclude_id in "${exclude_ids[@]}"; do
+        EXCLUDE_ARGS+=(-x "${DIR}/${exclude_id}")
+    done
+}
+
+
 function clean() {
     local hours=$(hours)
     local verbose=
+    exclude_args
     $DEBUG && verbose="--verbose"
     $DEBUG && set -x
-    /usr/bin/tmpwatch --mtime --dirmtime --exclude-user=root $verbose "${hours}h" "$DIR"
+    /usr/bin/tmpwatch --mtime --dirmtime --all --exclude-user=root "${EXCLUDE_ARGS[@]}" $verbose "${hours}h" "$DIR"
     { set +x; } 2>/dev/null
 }
 

env/common/group_vars/baseenv/vars.yml

@@ -5,6 +5,9 @@ influxdb_client_pass: "{{ vault_influxdb_client_pass }}"
 # run telegraf on everything
 telegraf_agent_package_state: latest
 
+# 10s is the default if unset, set `interval = "<seconds>s"` on [agent] or on plugins to override
+telegraf_agent_interval: 60
+
 telegraf_agent_output:
   - type: influxdb
     config:

env/galaxy/_inc_baseenv.yml

@@ -6,8 +6,48 @@
   gather_facts: false
   pre_tasks:
     - name: Gather facts
-      setup:
+      ansible.builtin.setup:
       tags: always
+    # FIXME: upgrade sentry first
+    #- name: Group by distribution
+    #  ansible.builtin.group_by:
+    #    key: "{{ ansible_distribution | lower }}hosts"
+    #- name: Debian block
+    #  block:
+    #    - name: Ensure security repo is enabled
+    #      ansible.builtin.apt_repository:
+    #        repo: "{{ item }}"
+    #      loop:
+    #        # FIXME: main contrib non-free if contrib and non-free are enabled
+    #        - "deb https://deb.debian.org/debian-security/ {{ ansible_distribution_release }} main"
+    #        - "deb-src https://deb.debian.org/debian-security/ {{ ansible_distribution_release }} main"
+    #  when: "'debianhosts' in group_names"
+    - name: Install unattended-upgrades
+      apt:
+        name: unattended-upgrades
+      when: ansible_os_family == "Debian"
+      tags: packages
+    - name: EL7 updates block
+      when: ansible_os_family == "RedHat" and ansible_distribution_major_version is version("8", "<") and "tacc" not in inventory_hostname
+      tags: packages
+      block:
+        - name: Install yum-cron
+          yum:
+            name: yum-cron
+        - name: Configure yum-cron
+          ini_file:
+            path: /etc/yum/yum-cron.conf
+            section: commands
+            option: "{{ item.option }}"
+            value: "{{ item.value }}"
+            mode: "0644"
+          loop:
+            - option: update_cmd
+              # doesn't work under CentOS 7
+              #value: security
+              value: default
+            - option: apply_updates
+              value: "yes"
   roles:
     - role: locale            # Configure default locale
       tags: locale

env/galaxy/files/sites/datacache/index.html

@@ -44,5 +44,7 @@ <h2>Contents</h2>
                    singularity.galaxyproject.org. This is a convenience to browse the repository, the upstream source
                    that it is mirrored from can be found at <a
                    href="https://depot.galaxyproject.org/singularity">depot.galaxyproject.org/singularity</a></li>
+            <li><a href="/idc">/idc</a> - Galaxy <a
+                   href="https://github.com/galaxyproject/idc">Intergalactic Data Commission</a></li>
     </body>
 </html>

env/galaxy/group_vars/certbotservers/vars.yml

@@ -3,6 +3,7 @@
 # vaulted as an anti-spam measure
 certbot_admin_email: "{{ vault_certbot_admin_email }}"
 
+certbot_virtualenv_package_name: python3-venv
 certbot_auto_renew_hour: "{{ 23 |random(seed=inventory_hostname)  }}"
 certbot_auto_renew_minute: "{{ 59 |random(seed=inventory_hostname)  }}"
 certbot_auth_method: --webroot

env/galaxy/group_vars/cvmfsstratum0servers/vars.yml

@@ -1,5 +1,14 @@
 ---
 
+telegraf_plugins_extra:
+  http_response:
+    config:
+      - urls = ["http://localhost/cvmfs/info/v1/meta.json"]
+      - response_status_code = 200
+      - interface = "lo"
+      - '[inputs.http_response.headers]'
+      - '  Host = "{{ inventory_hostname }}"'
+
 #cvmfs_manage_units_group: docker
 cvmfsstratum0servers_group_packages:
   # for usegalaxy-tools
@@ -17,6 +26,8 @@ cvmfs_public_site_keys:
     mAG1ceyBFowj/r3iJTa+Jcif2uAmZxg+cHkZG5KzATykF82UH1ojUzREMMDcPJi2
     dQIDAQAB
     -----END PUBLIC KEY-----
+  # FIXME: this is the per-repo cert and along with the key (not masterkey) should remain unique per repo. Only the
+  # pub (and masterkey) should be common, and only pub needs to be distributed to clients.
   crt: |
     -----BEGIN CERTIFICATE-----
     MIIDBDCCAewCCQDY/xeqeOP2rTANBgkqhkiG9w0BAQsFADBEMUIwQAYDVQQDDDlj

env/galaxy/group_vars/cvmfsstratum1servers/vars.yml

@@ -3,6 +3,15 @@
 firewalld_state: absent
 selinux_state: disabled
 
+telegraf_plugins_extra:
+  http_response:
+    config:
+      - urls = ["http://localhost/cvmfs/info/v1/meta.json"]
+      - response_status_code = 200
+      - interface = "lo"
+      - '[inputs.http_response.headers]'
+      - '  Host = "{{ inventory_hostname }}"'
+
 galaxy_cvmfs_repos_enabled: false
 
 cvmfsstratum1servers_group_authorized_key_users:
@@ -55,27 +64,35 @@ cvmfsstratum1servers_group_users:
     home: /home/cloud
     shell: /bin/bash
     system: yes
+  - name: idc
+    comment: "Intergalactic Data Commission"
+    home: /home/idc
+    shell: /bin/bash
+    system: yes
 
 cvmfsstratum1servers_group_groups:
   - name: G-803372
     gid: 803372
 
 cvmfs_manage_units_group: G-803372
 
+cvmfs_site_keys:
+  pub: |
+    -----BEGIN PUBLIC KEY-----
+    MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuJZTWTY3/dBfspFKifv8
+    TWuuT2Zzoo1cAskKpKu5gsUAyDFbZfYBEy91qbLPC3TuUm2zdPNsjCQbbq1Liufk
+    uNPZJ8Ubn5PR6kndwrdD13NVHZpXVml1+ooTSF5CL3x/KUkYiyRz94sAr9trVoSx
+    THW2buV7ADUYivX7ofCvBu5T6YngbPZNIxDB4mh7cEal/UDtxV683A/5RL4wIYvt
+    S5SVemmu6Yb8GkGwLGmMVLYXutuaHdMFyKzWm+qFlG5JRz4okUWERvtJ2QAJPOzL
+    mAG1ceyBFowj/r3iJTa+Jcif2uAmZxg+cHkZG5KzATykF82UH1ojUzREMMDcPJi2
+    dQIDAQAB
+    -----END PUBLIC KEY-----
+
 cvmfs_keys_dir: /etc/cvmfs/keys/galaxyproject.org
 cvmfs_keys:
   - path: /etc/cvmfs/keys/galaxyproject.org/test.galaxyproject.org.pub
     owner: g2test
-    key: |
-      -----BEGIN PUBLIC KEY-----
-      MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtfc5SSX9ALcrukWYcxkI
-      mkLhlkJa5tCP1oZNWFA7GfE4xQW2vcKE5qmbZqhYVfdiy+FHPnhWPJp577hekD2F
-      vMITbApdZ0265AjRC0+EKKxpMF8zZ0q71vCFxvdK0c3DtT/3LmqKrr2wimtJZjQN
-      UAZcQG2ykzeHzFZ46w74IO0o8Fv/w2XEbYI0QqbNFv+0hcp5SruFqaaLsRZdd6Bn
-      3iSylgVRQ5b+h1LfB/EuEpSmH1sDozZ4tU0fpbrBSknK76aad1o/cvWY1X87ToUV
-      helU0HE2Rw/u9EqJDvPFTbUmad3MtspkqbG5Eo7lI+ktzbcD7UTsQ/7noIXIQ5dD
-      PwIDAQAB
-      -----END PUBLIC KEY-----
+    key: "{{ cvmfs_site_keys.pub }}"
   - path: /etc/cvmfs/keys/galaxyproject.org/data.galaxyproject.org.pub
     owner: g2test
     key: |
@@ -114,16 +131,7 @@ cvmfs_keys:
       -----END PUBLIC KEY-----
   - path: /etc/cvmfs/keys/galaxyproject.org/singularity.galaxyproject.org.pub
     owner: root
-    key: |
-      -----BEGIN PUBLIC KEY-----
-      MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuJZTWTY3/dBfspFKifv8
-      TWuuT2Zzoo1cAskKpKu5gsUAyDFbZfYBEy91qbLPC3TuUm2zdPNsjCQbbq1Liufk
-      uNPZJ8Ubn5PR6kndwrdD13NVHZpXVml1+ooTSF5CL3x/KUkYiyRz94sAr9trVoSx
-      THW2buV7ADUYivX7ofCvBu5T6YngbPZNIxDB4mh7cEal/UDtxV683A/5RL4wIYvt
-      S5SVemmu6Yb8GkGwLGmMVLYXutuaHdMFyKzWm+qFlG5JRz4okUWERvtJ2QAJPOzL
-      mAG1ceyBFowj/r3iJTa+Jcif2uAmZxg+cHkZG5KzATykF82UH1ojUzREMMDcPJi2
-      dQIDAQAB
-      -----END PUBLIC KEY-----
+    key: "{{ cvmfs_site_keys.pub }}"
   - path: /etc/cvmfs/keys/galaxyproject.org/usegalaxy.galaxyproject.org.pub
     owner: root
     key: |
@@ -138,16 +146,7 @@ cvmfs_keys:
       -----END PUBLIC KEY-----
   - path: /etc/cvmfs/keys/galaxyproject.org/cvmfs-config.galaxyproject.org.pub
     owner: root
-    key: |
-      -----BEGIN PUBLIC KEY-----
-      MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuJZTWTY3/dBfspFKifv8
-      TWuuT2Zzoo1cAskKpKu5gsUAyDFbZfYBEy91qbLPC3TuUm2zdPNsjCQbbq1Liufk
-      uNPZJ8Ubn5PR6kndwrdD13NVHZpXVml1+ooTSF5CL3x/KUkYiyRz94sAr9trVoSx
-      THW2buV7ADUYivX7ofCvBu5T6YngbPZNIxDB4mh7cEal/UDtxV683A/5RL4wIYvt
-      S5SVemmu6Yb8GkGwLGmMVLYXutuaHdMFyKzWm+qFlG5JRz4okUWERvtJ2QAJPOzL
-      mAG1ceyBFowj/r3iJTa+Jcif2uAmZxg+cHkZG5KzATykF82UH1ojUzREMMDcPJi2
-      dQIDAQAB
-      -----END PUBLIC KEY-----
+    key: "{{ cvmfs_site_keys.pub }}"
   - path: /etc/cvmfs/keys/galaxyproject.org/refgenomes-databio.galaxyproject.org.pub
     owner: root
     key: |
@@ -162,16 +161,10 @@ cvmfs_keys:
       -----END PUBLIC KEY-----
   - path: /etc/cvmfs/keys/galaxyproject.org/cloud.galaxyproject.org.pub
     owner: root
-    key: |
-      -----BEGIN PUBLIC KEY-----
-      MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuJZTWTY3/dBfspFKifv8
-      TWuuT2Zzoo1cAskKpKu5gsUAyDFbZfYBEy91qbLPC3TuUm2zdPNsjCQbbq1Liufk
-      uNPZJ8Ubn5PR6kndwrdD13NVHZpXVml1+ooTSF5CL3x/KUkYiyRz94sAr9trVoSx
-      THW2buV7ADUYivX7ofCvBu5T6YngbPZNIxDB4mh7cEal/UDtxV683A/5RL4wIYvt
-      S5SVemmu6Yb8GkGwLGmMVLYXutuaHdMFyKzWm+qFlG5JRz4okUWERvtJ2QAJPOzL
-      mAG1ceyBFowj/r3iJTa+Jcif2uAmZxg+cHkZG5KzATykF82UH1ojUzREMMDcPJi2
-      dQIDAQAB
-      -----END PUBLIC KEY-----
+    key: "{{ cvmfs_site_keys.pub }}"
+  - path: /etc/cvmfs/keys/galaxyproject.org/idc.galaxyproject.org.pub
+    owner: root
+    key: "{{ cvmfs_site_keys.pub }}"
 
 cvmfs_repositories:
   - repository: test.galaxyproject.org
@@ -237,3 +230,10 @@ cvmfs_repositories:
     server_options:
       - CVMFS_AUTO_GC=false
     client_options: []
+  - repository: idc.galaxyproject.org
+    stratum0: cvmfs0-psu0.galaxyproject.org
+    owner: idc
+    key_dir: /etc/cvmfs/keys/galaxyproject.org
+    server_options:
+      - CVMFS_AUTO_GC=false
+    client_options: []

env/galaxy/group_vars/dnsprimaryservers.yml

@@ -26,7 +26,6 @@ dns_bind_views:
       - file: 'repeatme.org.zone'
       - file: 'galaxy2.org.zone'
       - file: 'galaxyworkbench.org.zone'
-      - file: 'gxy.io.zone'
       - file: 'galaxy.training.zone'
       - file: 'galaxy.tools.zone'
       - file: 'toolshed.tools.zone'
@@ -59,7 +58,6 @@ dns_bind_views:
       - file: 'repeatme.org.zone'
       - file: 'galaxy2.org.zone'
       - file: 'galaxyworkbench.org.zone'
-      - file: 'gxy.io.zone'
       - file: 'galaxy.training.zone'
       - file: 'galaxy.tools.zone'
       - file: 'toolshed.tools.zone'

env/galaxy/group_vars/dnssecondaryservers.yml

@@ -26,7 +26,6 @@ dns_bind_views:
       - name: 'galaxyworkbench.org'
       - name: 'galaxy.tools'
       - name: 'toolshed.tools'
-      - name: 'gxy.io'
       - name: 'galaxy.training'
       - name: '0.18.172.in-addr.arpa'
       - name: '2.18.172.in-addr.arpa'
@@ -54,5 +53,4 @@ dns_bind_views:
       - name: 'galaxyworkbench.org'
       - name: 'galaxy.tools'
       - name: 'toolshed.tools'
-      - name: 'gxy.io'
       - name: 'galaxy.training'

env/galaxy/group_vars/sentryservers/vars.yml

@@ -25,3 +25,11 @@ sentry_github_app_name: galaxyproject-SentryIntegration
 sentry_github_client_id: "{{ vault_sentry_github_client_id }}"
 sentry_github_client_secret: "{{ vault_sentry_github_client_secret }}"
 sentry_github_private_key: "{{ vault_sentry_github_private_key }}"
+
+sentryservers_group_crontabs:
+  - name: "Sentry cleanup"
+    id: sentry_cleanup
+    user: root
+    hour: 22
+    minute: 15
+    job: "(cd /opt/sentry && /usr/local/bin/docker-compose exec -T worker sentry cleanup --days 90 && /usr/local/bin/docker-compose exec -T postgres psql -U postgres -c VACUUM) 2>&1 | logger"

env/galaxy/group_vars/statsservers/vars.yml

@@ -1,5 +1,33 @@
 ---
 
+telegraf_cvmfs_check_servers:
+  hosts:
+    - cvmfs1-tacc0.galaxyproject.org
+    - cvmfs1-iu0.galaxyproject.org
+    - cvmfs1-psu0.galaxyproject.org
+  repos:
+    - test.galaxyproject.org
+    - main.galaxyproject.org
+    - data.galaxyproject.org
+    - singularity.galaxyproject.org
+    - idc.galaxyproject.org
+  combined:
+    - host: cvmfs0-tacc0.galaxyproject.org
+      repos:
+        - main.galaxyproject.org
+    - host: cvmfs0-psu0.galaxyproject.org
+      repos:
+        - data.galaxyproject.org
+    - host: cvmfs0-psu1.galaxyproject.org
+      repos:
+        - test.galaxyproject.org
+    - host: cvmfs-stratum0.galaxyproject.eu
+      repos:
+        - singularity.galaxyproject.org
+    - host: cvmfs0-psu0.galaxyproject.org
+      repos:
+        - idc.galaxyproject.org
+
 #stats_ssl_ca_passphrase: "{{ vault_stats_ssl_ca_passphrase }}"
 
 #stats_ssl_certs:

env/galaxy/group_vars/webservers.yml

@@ -1,3 +1,5 @@
 ---
 
+install_epel: true
+
 nginx_srv: /srv/nginx

env/galaxy/host_vars/cvmfs-stratum0.galaxyproject.eu/vars.yml

@@ -62,6 +62,10 @@ cvmfs_repositories:
       - CVMFS_GENERATE_LEGACY_BULK_CHUNKS=false
       - CVMFS_VIRTUAL_DIR=true
     client_options: []
+    prune_snapshots_count: 100
+    prune_snapshots_time:
+      minute: 45
+      hour: 0
 
 cvmfs_private_keys:
   - path: /etc/cvmfs/keys/singularity.galaxyproject.org.key