From 3c003f6d77539d240e67f234b666653854f426df Mon Sep 17 00:00:00 2001 From: cat-bro Date: Thu, 16 Jan 2025 16:01:37 +1100 Subject: [PATCH] Remove pg-post-tasks and add common bash variable and functions --- README.md | 1 - dev-handler_playbook.yml | 1 - dev_playbook.yml | 1 - galaxy-handlers_playbook.yml | 1 - galaxy_playbook.yml | 1 - group_vars/all.yml | 6 - group_vars/galaxyservers.yml | 3 + hosts | 28 ++++- roles/common/defaults/main.yml | 8 ++ roles/common/tasks/bashrc.yml | 115 ++++++++++++++++++ roles/common/tasks/machine_users.yml | 48 +++++--- roles/common/tasks/main.yml | 9 +- .../bashrc/get_jwd_path_galaxy.py.j2 | 26 ++++ roles/pg-post-tasks/tasks/main.yml | 23 ---- roles/pg-post-tasks/tasks/pgpass_bashrc.yml | 38 ------ staging_playbook.yml | 1 - 16 files changed, 216 insertions(+), 94 deletions(-) create mode 100644 roles/common/tasks/bashrc.yml create mode 100644 roles/common/templates/bashrc/get_jwd_path_galaxy.py.j2 delete mode 100644 roles/pg-post-tasks/tasks/main.yml delete mode 100644 roles/pg-post-tasks/tasks/pgpass_bashrc.yml diff --git a/README.md b/README.md index 0d6c74b0b..008aaf7fb 100644 --- a/README.md +++ b/README.md @@ -75,7 +75,6 @@ There are a few custom roles associated with these playbooks. They are: | common | Performs common tasks on our virtual machines including setting up users, installing some required packages amongst other tasks | | mariadb | Sets up MariaDB for Slurm accounting on various clusters | | mounts | Sets up NFS mounts for various clusters | -| pg-post-tasks | Adds some users and extra permissions to the Galaxy Database | | slg.db-backup | Role to setup database backups for Galaxy Australia | | slg.galaxy_stats | Sets up collection of statistics from Galaxy for Grafana | diff --git a/dev-handler_playbook.yml b/dev-handler_playbook.yml index 786136a58..1cd43a5aa 100644 --- a/dev-handler_playbook.yml +++ b/dev-handler_playbook.yml @@ -48,7 +48,6 @@ - galaxyproject.slurm - galaxyproject.cvmfs - galaxyproject.gxadmin - - pg-post-tasks - postfix-mail-relay - geerlingguy.docker - usegalaxy_eu.gie_proxy diff --git a/dev_playbook.yml b/dev_playbook.yml index 32217bc99..22214d111 100644 --- a/dev_playbook.yml +++ b/dev_playbook.yml @@ -56,7 +56,6 @@ - galaxyproject.slurm - galaxyproject.cvmfs - galaxyproject.gxadmin - - pg-post-tasks - remote-pulsar-cron - galaxy-pg-cleanup - galaxyproject.tiaas2 diff --git a/galaxy-handlers_playbook.yml b/galaxy-handlers_playbook.yml index 67c131207..99823b942 100644 --- a/galaxy-handlers_playbook.yml +++ b/galaxy-handlers_playbook.yml @@ -69,7 +69,6 @@ - galaxyproject.slurm - galaxyproject.cvmfs - galaxyproject.gxadmin - - pg-post-tasks - postfix-mail-relay - geerlingguy.docker - dj-wasabi.telegraf diff --git a/galaxy_playbook.yml b/galaxy_playbook.yml index 18ad40721..c8d3ec102 100644 --- a/galaxy_playbook.yml +++ b/galaxy_playbook.yml @@ -56,7 +56,6 @@ - dj-wasabi.telegraf - geerlingguy.nfs - galaxyproject.gxadmin - - pg-post-tasks - postfix-mail-relay - remote-pulsar-cron # - delete-tmp-jwds-cron # commented out until galaxy etca is production galaxy, also needs to be enabled diff --git a/group_vars/all.yml b/group_vars/all.yml index e700bca76..e64d58552 100644 --- a/group_vars/all.yml +++ b/group_vars/all.yml @@ -201,12 +201,6 @@ galaxy_cvmfs_server_urls: - "http://cvmfs1-tacc0.galaxyproject.org/cvmfs/@fqrn@" - "http://cvmfs1-ufr0.galaxyproject.eu/cvmfs/@fqrn@" -#sinfo and squeue configs -bashrc_env: | - export SINFO_FORMAT="%24n %.14C %.8t" - export SQUEUE_FORMAT="%8i %9P %.35j %.9T %8r %19S %.10M %.6m %.3C %.3N %.55Z" - export SACCT_FORMAT="jobid%8,partition%9,jobname%30,alloccpus,elapsed,totalcpu,END,state,MaxRSS%12,ReqMem,NodeList%24" - # miniconda miniconda_version: '4.12.0' miniconda_mamba_version: '0.23.3' diff --git a/group_vars/galaxyservers.yml b/group_vars/galaxyservers.yml index 6e62e6c48..764e5eaba 100644 --- a/group_vars/galaxyservers.yml +++ b/group_vars/galaxyservers.yml @@ -117,6 +117,8 @@ galaxy_venv_dir: "{{ galaxy_root }}/venv" # set this explicitly as it can fail w galaxy_mutable_config_dir: "{{ galaxy_root }}/var" galaxy_mutable_data_dir: "{{ galaxy_root }}" galaxy_config_dir: "{{ galaxy_root }}/config" +galaxy_job_working_directory: "{{ galaxy_tmp_dir }}/job_working_directory" +galaxy_pulsar_app_config_file: "{{ galaxy_config_dir }}/pulsar_app.yml" galaxy_conda_prefix: "{{ galaxy_tools_indices_dir }}/tool_dependencies/_conda" galaxy_conda_exec: mamba @@ -401,6 +403,7 @@ lsyncd_source: /mnt/galaxy lsyncd_target: /mnt/ghost-galaxy-app lsyncd_exclude: - cache + - log lsyncd_max_user_watches: 524288 # debugging settings diff --git a/hosts b/hosts index 421dc4b8b..7982627bc 100644 --- a/hosts +++ b/hosts @@ -146,15 +146,30 @@ pulsar_qld_blast pulsar_nci_training_head pulsar_azure_0_head +# All pulsar head nodes +[pulsar_head_nodes:children] +production_pulsars +dev_pulsar_head +staging_pulsar_head +pulsar_nci_test_head + +# All pulsar workers +[pulsar_workers:children] +pulsar_paw_workers +pulsar_mel3_workers +pulsar_mel2_workers +pulsar_QLD_workers +pulsar_nci_training_workers +pulsar_nci_test_workers + # Dev cluster (GenomicsVL_Devt) -[dev_group] -dev-pulsar ansible_ssh_host=45.113.233.82 [dev_group:children] dev_db_server dev_galaxy_server dev_handlers_server dev_slurm_head dev_workers +dev_pulsar_head [dev_db_server] dev-db ansible_ssh_host=45.113.232.252 @@ -172,14 +187,16 @@ dev-queue ansible_ssh_host=45.113.232.149 dev-w1 ansible_ssh_host=45.113.233.251 dev-w2 ansible_ssh_host=45.113.233.7 +[dev_pulsar_head] +dev-pulsar ansible_ssh_host=45.113.233.82 + # Staging cluster (GenomicsVL) -[staging_group] -staging-pulsar ansible_ssh_host=115.146.87.193 [staging_group:children] staging_db_server staging_galaxy_server staging_slurm_head staging_workers +staging_pulsar_head [staging_db_server] staging-db ansible_ssh_host=115.146.87.93 @@ -193,6 +210,9 @@ staging-queue ansible_ssh_host=115.146.84.121 [staging_workers] staging-w1 ansible_ssh_host=115.146.87.130 +[staging_pulsar_head] +staging-pulsar ansible_ssh_host=115.146.87.193 + # CVMFS [cvmfsstratum1servers] cvmfs1-mel0.gvl.org.au ansible_user=ec2-user ansible_ssh_host=115.146.85.207 diff --git a/roles/common/defaults/main.yml b/roles/common/defaults/main.yml index 5baf8e886..03d654a53 100644 --- a/roles/common/defaults/main.yml +++ b/roles/common/defaults/main.yml @@ -8,3 +8,11 @@ htpass_username: "ubuntu" internal_ssh_keys: {} extra_keys: [] + +common_sinfo_format: "%24n %.14C %.8t" +common_squeue_format: "%8i %9P %.35j %.9T %8r %19S %.10M %.6m %.3C %.3N %.55Z" +common_sacct_format: "jobid%8,partition%9,jobname%30,alloccpus,elapsed,totalcpu,END,state,MaxRSS%12,ReqMem,NodeList%24" + +common_bashrc_dir: /etc/bashrc_common +common_bashrc_vars_file: "{{ common_bashrc_dir }}/common_variables" +common_bashrc_functions_file: "{{ common_bashrc_dir }}/common_functions" diff --git a/roles/common/tasks/bashrc.yml b/roles/common/tasks/bashrc.yml new file mode 100644 index 000000000..1b548f125 --- /dev/null +++ b/roles/common/tasks/bashrc.yml @@ -0,0 +1,115 @@ +--- +- name: Ensure directory for bashrc common files + file: + state: directory + path: "{{ common_bashrc_dir }}" +- name: Template bashrc_common path + copy: + dest: "{{ common_bashrc_vars_file }}" + mode: 0755 + content: | + # slurm + export SINFO_FORMAT="{{ common_sinfo_format }}" + export SQUEUE_FORMAT="{{ common_squeue_format }}" + export SINFO_FORMAT="{{ common_sacct_format }}" + + # helpful settings + export EDITOR=vim + HISTSIZE= + HISTFILESIZE= + + {% if is_galaxy_head_node %} + # galaxy + export PGHOST='{{ db_address }}' + export PGDATABASE='galaxy' + export PGPORT='5432' + export GALAXY_CONFIG_DIR={{ galaxy_config_dir }} + export GALAXY_MUTABLE_CONFIG_DIR={{ galaxy_mutable_config_dir }} + export GALAXY_ROOT={{ galaxy_server_dir }} + export GALAXY_CONFIG_FILE={{ galaxy_config_file }} + export GALAXY_JWD_PATH="{{ galaxy_job_working_directory }}" + export GALAXY_LOG_DIR="{{ galaxy_log_dir }}" + export GALAXY_PULSAR_APP_CONF="{{ galaxy_pulsar_app_config_file }}" + {% endif %} +- name: Template common_bashrc path + copy: + dest: "{{ common_bashrc_functions_file }}" + mode: 0755 + content: | + # Common variables and functions for ubuntu and human users + + {% if is_galaxy_head_node %} + # galaxy + jwd() { # print path of job working directory for job id + python "{{ common_bashrc_dir }}/get_jwd_path_galaxy.py" $1 + } + + jwd-size() { + du -sh $(jwd $1) + } + + go-jwd() { # go to job working directory for job id + cd $(jwd $1) + } + + tail-stderr() { + stderr_filename=$(jwd $1)/outputs/tool_stderr + if [ ! -f $stderr_filename ]; then + echo "No stderr file for job $1"; + else + tail $stderr_filename; + fi + } + {% endif %} + + {% if ansible_hostname in groups['pulsar_head_nodes'] %} + # pulsar + jwd() { # print path of job working directory for job id + echo {{ pulsar_staging_dir }}/${1} + } + + jwd-size() { + du -sh $(jwd $1) + } + + go-jwd() { # go to job working directory for job id + cd $(jwd $1) + } + + tail-stderr() { + stderr_filename=$(jwd $1)/metadata/tool_stderr + if [ ! -f $stderr_filename ]; then + echo "No stderr file for job $1"; + else + tail $stderr_filename; + fi + } + {% endif %} + +- name: Template helper scripts + template: + src: "bashrc/{{ item }}.j2" + dest: "{{ common_bashrc_dir }}/{{ item }}" + with_items: + - get_jwd_path_galaxy.py + +- name: Add to .bashrc files + blockinfile: + marker: "# {mark} ANSIBLE MANAGED BLOCK (common role: source common bash env for users)" + dest: "/home/{{ item.name }}/.bashrc" + block: | + source {{ common_bashrc_vars_file }} + source {{ common_bashrc_functions_file }} + + {% if is_galaxy_head_node %} + # galaxy (per user) + {% if item == 'ubuntu' %} + export PGUSER='galaxy' + {% elif 'tiaas_admin' in item.roles|d([]) %} + export PGUSER='tiaasadmin' + {% else %} + export PGUSER='reader' + {% endif %} + {% endif %} + + with_items: "{{ machine_users + [{'name': 'ubuntu'}] }}" diff --git a/roles/common/tasks/machine_users.yml b/roles/common/tasks/machine_users.yml index fcfed25e0..eac2ff7de 100644 --- a/roles/common/tasks/machine_users.yml +++ b/roles/common/tasks/machine_users.yml @@ -4,13 +4,6 @@ become_user: root block: - # FIX for deprecated code that added user files in /etc/sudoers.d/ - can be removed in future - - name: delete /etc/sudoers.d/username files for machine users - file: - path: "/etc/sudoers.d/{{ item.name }}" - state: absent - with_items: "{{ remove_machine_users + machine_users }}" - - name: Ensure group "devs" exists group: name: devs @@ -65,18 +58,41 @@ force: yes with_items: "{{ remove_machine_users }}" - - name: Add some env_vars to .bashrcs - blockinfile: - path: "/home/{{ item.name }}/.bashrc" - block: | - {{ bashrc_env }} - marker: "# {mark} MANAGED BY ANSIBLE - DO NOT MODIFY" - with_items: "{{ machine_users + [{'name': 'ubuntu'}] }}" - when: bashrc_env is defined - - name: Add SSH keys for users authorized_key: user: "{{ item.name }}" key: "{{ lookup('file', item.key) }}" state: present with_items: "{{ machine_users }}" + + # former pg-post-tasks tasks + - name: Pgpass creation and gxadmin user tasks + when: is_galaxy_head_node|d(False) + block: + - name: Create .pgpass files for users + copy: + dest: "/home/{{ item.name }}/.pgpass" + content: "{{ db_address }}:5432:*:{{ db_role }}:{{ db_password }}" + mode: "600" + group: "{{ item.name }}" + owner: "{{ item.name }}" + vars: + db_role: "{{ 'galaxy' if item.name == 'ubuntu' else ('reader' if not 'tiaas_admin' in item.roles|d([]) else 'tiaasadmin') }}" + db_password: "{{ pg_db_password[db_role] }}" + with_items: "{{ machine_users + [{'name': 'ubuntu'}] }}" + - name: Ensure gxadmin config directory exists for all users including ubuntu + file: + path: "/home/{{ item }}/.config" + state: directory + group: "{{ item }}" + owner: "{{ item }}" + with_items: "{{ (machine_users | map(attribute='name') | list) + ['ubuntu'] }}" + - name: Ensure that all users + ubuntu have a copy of gxadmin-local.sh + copy: + src: files/galaxy/gxadmin/gxadmin-local.sh + dest: "/home/{{ item }}/.config/gxadmin-local.sh" + mode: "755" + group: "{{ item }}" + owner: "{{ item }}" + force: yes + with_items: "{{ (machine_users | map(attribute='name') | list) + ['ubuntu'] }}" diff --git a/roles/common/tasks/main.yml b/roles/common/tasks/main.yml index 65683e10f..20c8a0544 100644 --- a/roles/common/tasks/main.yml +++ b/roles/common/tasks/main.yml @@ -83,6 +83,13 @@ - name: Add machine users import_tasks: machine_users.yml + tags: + - users + +- name: Add variables to bashrc files + import_tasks: bashrc.yml + tags: + - users - name: Add extra ssh keys include_tasks: extra_keys.yml @@ -120,4 +127,4 @@ lineinfile: path: /etc/environment create: true - line: export SACCT_FORMAT="jobid%8,partition%9,jobname%30,alloccpus,elapsed,totalcpu,END,state,MaxRSS%12,ReqMem,NodeList%24" + line: export SACCT_FORMAT="{{ common_sacct_format }}" diff --git a/roles/common/templates/bashrc/get_jwd_path_galaxy.py.j2 b/roles/common/templates/bashrc/get_jwd_path_galaxy.py.j2 new file mode 100644 index 000000000..1c415ed29 --- /dev/null +++ b/roles/common/templates/bashrc/get_jwd_path_galaxy.py.j2 @@ -0,0 +1,26 @@ +import argparse +import os + +def main(): + parser = argparse.ArgumentParser(description='Get job working directory from a job id') + parser.add_argument('job_id', type=int, help='Galaxy job id') + args = parser.parse_args() + + prefix = os.environ.get("GALAXY_JWD_PATH") + if not prefix: + raise ValueError("Please set GALAXY_JWD_PATH environment variable") + + jwd_path = get_jwd_path(args.job_id, prefix) + print(jwd_path) + +def get_jwd_path(job_id, prefix): + if len(str(job_id)) > 6: # on production this is the case + nine_digit_id = '0'*(9-len(str(job_id))) + str(job_id) + return os.path.join(prefix, nine_digit_id[:3], nine_digit_id[3:6], str(job_id)) + elif len(str(job_id)) <= 6: + six_digit_id = '0'*(6-len(str(job_id))) + str(job_id) + return os.path.join(prefix, six_digit_id[:3], str(job_id)) + + +if __name__ == '__main__': + main() diff --git a/roles/pg-post-tasks/tasks/main.yml b/roles/pg-post-tasks/tasks/main.yml deleted file mode 100644 index 7be3756a0..000000000 --- a/roles/pg-post-tasks/tasks/main.yml +++ /dev/null @@ -1,23 +0,0 @@ ---- - - name: Set .pgpass file and variables for gxadmin in .bashrc for all users - include: pgpass_bashrc.yml - loop: "{{ machine_users + [{'name': 'ubuntu'}] }}" - loop_control: - loop_var: machine_user - - name: Ensure gxadmin config directory exists for all users including ubuntu - file: - path: "/home/{{ item }}/.config" - state: directory - group: "{{ item }}" - owner: "{{ item }}" - with_items: "{{ (machine_users | map(attribute='name') | list) + ['ubuntu'] }}" - - name: Ensure that all users + ubuntu have a copy of gxadmin-local.sh - copy: - src: files/galaxy/gxadmin/gxadmin-local.sh - dest: "/home/{{ item }}/.config/gxadmin-local.sh" - mode: "755" - group: "{{ item }}" - owner: "{{ item }}" - force: yes - with_items: "{{ (machine_users | map(attribute='name') | list) + ['ubuntu'] }}" - diff --git a/roles/pg-post-tasks/tasks/pgpass_bashrc.yml b/roles/pg-post-tasks/tasks/pgpass_bashrc.yml deleted file mode 100644 index c8e89899d..000000000 --- a/roles/pg-post-tasks/tasks/pgpass_bashrc.yml +++ /dev/null @@ -1,38 +0,0 @@ ---- - - name: Set pg_db_role - set_fact: - db_role: "{{ 'galaxy' if machine_user.name == 'ubuntu' else ('reader' if not 'tiaas_admin' in machine_user.roles|d([]) else 'tiaasadmin') }}" - - name: Set pg_db_password - set_fact: - db_password: "{{ pg_db_password[db_role] }}" - - name: Create .pgpass files for users - copy: - dest: "/home/{{ machine_user.name }}/.pgpass" - content: "{{ db_address }}:5432:*:{{ db_role }}:{{ db_password }}" - mode: "600" - group: "{{ machine_user.name }}" - owner: "{{ machine_user.name }}" - - name: Add env vars for gxadmin to machine_users .bashrc files - lineinfile: - path: "/home/{{ machine_user.name }}/.bashrc" - regexp: "^export {{ obj.var }}=" - line: "export {{ obj.var }}='{{ obj.val }}'" - loop: - - var: PGHOST - val: "{{ db_address }}" - - var: PGUSER - val: "{{ db_role }}" - - var: PGDATABASE - val: "galaxy" - - var: PGPORT - val: "5432" - - var: GALAXY_CONFIG_DIR - val: "{{ galaxy_config_dir }}" - - var: GALAXY_MUTABLE_CONFIG_DIR - val: "{{ galaxy_mutable_config_dir }}" - - var: GALAXY_ROOT - val: "{{ galaxy_server_dir }}" - - var: GALAXY_CONFIG_FILE - val: "{{ galaxy_config_dir }}/galaxy.yml" - loop_control: - loop_var: obj diff --git a/staging_playbook.yml b/staging_playbook.yml index 17d58ab9a..dac034ace 100644 --- a/staging_playbook.yml +++ b/staging_playbook.yml @@ -45,7 +45,6 @@ - acl-on-startup - galaxyproject.gxadmin - dj-wasabi.telegraf - - pg-post-tasks - postfix-mail-relay - role: usegalaxy_eu.walle tags: walle