diff --git a/.github/workflows/fetch_filter_resources.yaml b/.github/workflows/fetch_filter_resources.yaml index 6059947a..ee30a248 100644 --- a/.github/workflows/fetch_filter_resources.yaml +++ b/.github/workflows/fetch_filter_resources.yaml @@ -95,7 +95,9 @@ jobs: - name: Merge all tools run: | #merge files with only one header -> https://stackoverflow.com/questions/16890582/unixmerge-multiple-csv-files-with-same-header-by-keeping-the-header-of-the-firs; map(.[]) -> https://stackoverflow.com/questions/42011086/merge-arrays-of-json (get flat array, one tool per entry) awk 'FNR==1 && NR!=1{next;}{print}' communities/all/resources/repositories*.list_tools.tsv > communities/all/resources/tools.tsv - jq -s 'map(.[])' communities/all/resources/repositories*.list_tools.json > communities/all/resources/all_tools.json + jq -s 'map(.[])' communities/all/resources/repositories*.list_tools.json > communities/all/resources/tools.json + rm communities/all/resources/repositories*.list_tools.json + rm communities/all/resources/repositories*.list_tools.tsv - name: Generate wordcloud and interactive table run: | bash sources/bin/format_tools.sh @@ -107,9 +109,9 @@ jobs: - name: Filter tutorials for communities run: | bash sources/bin/get_community_tutorials.sh - - name: Update tool to keep and exclude for communities - run: | - bash sources/bin/update_tools_to_keep_exclude.sh + #- name: Update tool to keep and exclude for communities + # run: | + # bash sources/bin/update_tools_to_keep_exclude.sh - name: Filter tools for communities run: | bash sources/bin/get_community_tools.sh diff --git a/sources/bin/get_community_tools.sh b/sources/bin/get_community_tools.sh index cb9baf63..ac4ed778 100755 --- a/sources/bin/get_community_tools.sh +++ b/sources/bin/get_community_tools.sh @@ -32,33 +32,35 @@ else if [[ -d "$com_data_fp" && ! -L "$com_data_fp" ]]; then community=`basename "$com_data_fp"` - echo "$community"; + if [ "$community" != "all" ]; then - mkdir -p "communities/$community/resources" + echo "$community"; - python sources/bin/extract_galaxy_tools.py \ - filter \ - --all "communities/all/resources/tools.json" \ - --ts-filtered "communities/$community/resources/tools_filtered_by_ts_categories.tsv" \ - --filtered "communities/$community/resources/tools.tsv" \ - --categories "communities/$community/metadata/categories" \ - --status "communities/$community/metadata/tool_status.tsv" + mkdir -p "communities/$community/resources" - python sources/bin/create_interactive_table.py \ - --input "communities/$community/resources/tools.tsv" \ - --remove-col "Reviewed" \ - --remove-col "To keep" \ - --filter-col "To keep" \ - --template "sources/data/interactive_table_template.html" \ - --output "communities/$community/resources/tools.html" + python sources/bin/extract_galaxy_tools.py \ + filter \ + --all "communities/all/resources/tools.json" \ + --ts-filtered "communities/$community/resources/tools_filtered_by_ts_categories.tsv" \ + --filtered "communities/$community/resources/tools.tsv" \ + --categories "communities/$community/metadata/categories" \ + --status "communities/$community/metadata/tool_status.tsv" - python sources/bin/create_wordcloud.py \ - --input "communities/$community/resources/tools.tsv" \ - --name-col "Galaxy wrapper id" \ - --stat-col "No. of tool users (2022-2023) (usegalaxy.eu)" \ - --wordcloud_mask "sources/data/usage_stats/wordcloud_mask.png" \ - --output "communities/$community/resources/tools_wordcloud.png" + python sources/bin/create_interactive_table.py \ + --input "communities/$community/resources/tools.tsv" \ + --remove-col "Reviewed" \ + --remove-col "To keep" \ + --filter-col "To keep" \ + --template "sources/data/interactive_table_template.html" \ + --output "communities/$community/resources/tools.html" + python sources/bin/create_wordcloud.py \ + --input "communities/$community/resources/tools.tsv" \ + --name-col "Galaxy wrapper id" \ + --stat-col "No. of tool users (2022-2023) (usegalaxy.eu)" \ + --wordcloud_mask "sources/data/usage_stats/wordcloud_mask.png" \ + --output "communities/$community/resources/tools_wordcloud.png" + fi; fi; done fi diff --git a/sources/bin/get_community_tutorials.sh b/sources/bin/get_community_tutorials.sh index 2f42080c..ad0eebba 100644 --- a/sources/bin/get_community_tutorials.sh +++ b/sources/bin/get_community_tutorials.sh @@ -14,24 +14,28 @@ then --output "communities/microgalaxy/resources/tutorials.html" else - for com_data_fp in data/communities/* ; do + for com_data_fp in communities/* ; do if [[ -d "$com_data_fp" && ! -L "$com_data_fp" ]]; then community=`basename "$com_data_fp"` + if [ "$community" != "all" ]; then + echo "$community" - echo "$community"; + if [[ -f "communities/$community/metadata/tutorial_tags" && -f "communities/$community/resources/tutorials.tsv" ]]; then + echo "Filter tutorials" - if [[ -f "data/communities/$community/tutorial_tags" && -f "results/$community/tutorials.tsv" ]]; then + python sources/bin/extract_gtn_tutorials.py \ + filter \ + --all "communities/all/resources/tutorials.json" \ + --filtered "communities/$community/resources/tutorials.tsv" \ + --tags "communities/$community/metadata/tutorial_tags" - python sources/bin/extract_gtn_tutorials.py \ - filter \ - --all "communities/all/resources/tutorials.json" \ - --filtered "communities/$community/resources/tutorials.tsv" \ - --tags "communities/$community/metadata/tutorial_tags" - - python sources/bin/create_interactive_table.py \ - --input "communities/$community/resources/tutorials.tsv" \ - --template "sources/data/interactive_table_template.html" \ - --output "communities/$community/resources/tutorials.html" + python sources/bin/create_interactive_table.py \ + --input "communities/$community/resources/tutorials.tsv" \ + --template "sources/data/interactive_table_template.html" \ + --output "communities/$community/resources/tutorials.html" + fi; + + echo "" fi; fi; done