diff --git a/data_managers/data_manager_bowtie2_index_builder/data_manager/bowtie2_index_builder.py b/data_managers/data_manager_bowtie2_index_builder/data_manager/bowtie2_index_builder.py deleted file mode 100644 index 20e921bb7cc..00000000000 --- a/data_managers/data_manager_bowtie2_index_builder/data_manager/bowtie2_index_builder.py +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env python -# Dan Blankenberg -from __future__ import print_function - -import json -import optparse -import os -import subprocess -import sys - -DEFAULT_DATA_TABLE_NAMES = ["bowtie2_indexes"] - - -def get_id_name(params, dbkey, fasta_description=None): - # TODO: ensure sequence_id is unique and does not already appear in location file - sequence_id = params['param_dict']['sequence_id'] - if not sequence_id: - sequence_id = dbkey - - sequence_name = params['param_dict']['sequence_name'] - if not sequence_name: - sequence_name = fasta_description - if not sequence_name: - sequence_name = dbkey - return sequence_id, sequence_name - - -def build_bowtie2_index(data_manager_dict, fasta_filename, params, target_directory, dbkey, sequence_id, sequence_name, data_table_names=DEFAULT_DATA_TABLE_NAMES): - # TODO: allow multiple FASTA input files - fasta_base_name = os.path.split(fasta_filename)[-1] - sym_linked_fasta_filename = os.path.join(target_directory, fasta_base_name) - os.symlink(fasta_filename, sym_linked_fasta_filename) - args = ['bowtie2-build', sym_linked_fasta_filename, sequence_id] - threads = os.environ.get('GALAXY_SLOTS') - if threads: - args.extend(['--threads', threads]) - proc = subprocess.Popen(args=args, shell=False, cwd=target_directory) - return_code = proc.wait() - if return_code: - print("Error building index.", file=sys.stderr) - sys.exit(return_code) - data_table_entry = dict(value=sequence_id, dbkey=dbkey, name=sequence_name, path=sequence_id) - for data_table_name in data_table_names: - _add_data_table_entry(data_manager_dict, data_table_name, data_table_entry) - - -def _add_data_table_entry(data_manager_dict, data_table_name, data_table_entry): - data_manager_dict['data_tables'] = data_manager_dict.get('data_tables', {}) - data_manager_dict['data_tables'][data_table_name] = data_manager_dict['data_tables'].get(data_table_name, []) - data_manager_dict['data_tables'][data_table_name].append(data_table_entry) - return data_manager_dict - - -def main(): - parser = optparse.OptionParser() - parser.add_option('-f', '--fasta_filename', dest='fasta_filename', action='store', type="string", default=None, help='fasta_filename') - parser.add_option('-d', '--fasta_dbkey', dest='fasta_dbkey', action='store', type="string", default=None, help='fasta_dbkey') - parser.add_option('-t', '--fasta_description', dest='fasta_description', action='store', type="string", default=None, help='fasta_description') - parser.add_option('-n', '--data_table_name', dest='data_table_name', action='append', type="string", default=None, help='data_table_name') - (options, args) = parser.parse_args() - - filename = args[0] - - with open(filename) as fh: - params = json.load(fh) - target_directory = params['output_data'][0]['extra_files_path'] - os.mkdir(target_directory) - data_manager_dict = {} - - dbkey = options.fasta_dbkey - - if dbkey in [None, '', '?']: - raise Exception('"%s" is not a valid dbkey. You must specify a valid dbkey.' % (dbkey)) - - sequence_id, sequence_name = get_id_name(params, dbkey=dbkey, fasta_description=options.fasta_description) - - # build the index - build_bowtie2_index(data_manager_dict, options.fasta_filename, params, target_directory, dbkey, sequence_id, sequence_name, data_table_names=options.data_table_name or DEFAULT_DATA_TABLE_NAMES) - - # save info to json file - with open(filename, 'w') as fh: - json.dump(data_manager_dict, fh, sort_keys=True) - - -if __name__ == "__main__": - main() diff --git a/data_managers/data_manager_bowtie2_index_builder/data_manager/bowtie2_index_builder.xml b/data_managers/data_manager_bowtie2_index_builder/data_manager/bowtie2_index_builder.xml index 4c9eb0cbaac..9f3583f64c6 100644 --- a/data_managers/data_manager_bowtie2_index_builder/data_manager/bowtie2_index_builder.xml +++ b/data_managers/data_manager_bowtie2_index_builder/data_manager/bowtie2_index_builder.xml @@ -1,26 +1,57 @@ - + builder + + 2.5.4 + 0 + bowtie2 - - 2.4.4 - + + + - + @@ -28,7 +59,14 @@ - + + + + + + + + diff --git a/data_managers/data_manager_bowtie2_index_builder/data_manager_conf.xml b/data_managers/data_manager_bowtie2_index_builder/data_manager_conf.xml index cc875397cc4..eb0888d232c 100644 --- a/data_managers/data_manager_bowtie2_index_builder/data_manager_conf.xml +++ b/data_managers/data_manager_bowtie2_index_builder/data_manager_conf.xml @@ -1,6 +1,5 @@ - @@ -10,9 +9,9 @@ - ${dbkey}/bowtie2_index/${value} + genomes/${dbkey}/bowtie_index/v2/${value} - ${GALAXY_DATA_MANAGER_DATA_PATH}/${dbkey}/bowtie2_index/${value}/${path} + ${GALAXY_DATA_MANAGER_DATA_PATH}/genomes/${dbkey}/bowtie_index/v2/${value}/${path} abspath @@ -25,11 +24,10 @@ - ${GALAXY_DATA_MANAGER_DATA_PATH}/${dbkey}/bowtie2_index/${value}/${path} + ${GALAXY_DATA_MANAGER_DATA_PATH}/genomes/${dbkey}/bowtie_index/v2/${value}/${path} abspath - diff --git a/data_managers/data_manager_bowtie2_index_builder/test-data/all_fasta.loc b/data_managers/data_manager_bowtie2_index_builder/test-data/all_fasta.loc index c8b25f3f801..ef0c9d5364b 100644 --- a/data_managers/data_manager_bowtie2_index_builder/test-data/all_fasta.loc +++ b/data_managers/data_manager_bowtie2_index_builder/test-data/all_fasta.loc @@ -16,4 +16,4 @@ #fasta file. So there will be multiple fasta files for each build, #such as with hg19 above. # -phiX174 phiX174 phiX174 ${__HERE__}/phiX174.fasta +phiX174 phiX174 phiX 174 ${__HERE__}/phiX174.fasta diff --git a/data_managers/data_manager_bowtie2_index_builder/test-data/bowtie2_data_manager.1.json b/data_managers/data_manager_bowtie2_index_builder/test-data/bowtie2_data_manager.1.json new file mode 100644 index 00000000000..68b51bbb54f --- /dev/null +++ b/data_managers/data_manager_bowtie2_index_builder/test-data/bowtie2_data_manager.1.json @@ -0,0 +1,20 @@ +{ + "data_tables":{ + "bowtie2_indexes":[ + { + "value": "phiX174", + "dbkey": "phiX174", + "name": "phiX 174", + "path": "phiX174.fasta" + } + ], + "tophat2_indexes":[ + { + "value": "phiX174", + "dbkey": "phiX174", + "name": "phiX 174", + "path": "phiX174.fasta" + } + ] + } +} diff --git a/data_managers/data_manager_bowtie2_index_builder/test-data/bowtie2_data_manager.2.json b/data_managers/data_manager_bowtie2_index_builder/test-data/bowtie2_data_manager.2.json new file mode 100644 index 00000000000..348255abfee --- /dev/null +++ b/data_managers/data_manager_bowtie2_index_builder/test-data/bowtie2_data_manager.2.json @@ -0,0 +1,12 @@ +{ + "data_tables":{ + "bowtie2_indexes":[ + { + "value": "tigHai1", + "dbkey": "phiX174", + "name": "Galeocerdo cuvier", + "path": "phiX174.fasta" + } + ] + } +} diff --git a/data_managers/data_manager_bowtie2_index_builder/test-data/bowtie2_data_manager.json b/data_managers/data_manager_bowtie2_index_builder/test-data/bowtie2_data_manager.json deleted file mode 100644 index 68b4e24d2e3..00000000000 --- a/data_managers/data_manager_bowtie2_index_builder/test-data/bowtie2_data_manager.json +++ /dev/null @@ -1 +0,0 @@ -{"data_tables": {"bowtie2_indexes": [{"dbkey": "phiX174", "name": "phiX174", "path": "phiX174", "value": "phiX174"}], "tophat2_indexes": [{"dbkey": "phiX174", "name": "phiX174", "path": "phiX174", "value": "phiX174"}]}} \ No newline at end of file diff --git a/data_managers/data_manager_bowtie_index_builder/data_manager/bowtie_index_builder.xml b/data_managers/data_manager_bowtie_index_builder/data_manager/bowtie_index_builder.xml index 39735c2195f..d02759e1dee 100644 --- a/data_managers/data_manager_bowtie_index_builder/data_manager/bowtie_index_builder.xml +++ b/data_managers/data_manager_bowtie_index_builder/data_manager/bowtie_index_builder.xml @@ -1,17 +1,39 @@ - + builder + + 1.3.1 + 0 + - bowtie - python + bowtie + #set $fasta_file_name = str($all_fasta_source.fields.path).split('/')[-1] + mkdir -p '${out_file.extra_files_path}' && + ln -s '${all_fasta_source.fields.path}' '${out_file.extra_files_path}/${fasta_file_name}' && + bowtie-build '${out_file.extra_files_path}/${fasta_file_name}' '${out_file.extra_files_path}/${fasta_file_name}' && + cp '$dmjson' '$out_file' + ]]> + + + + @@ -22,6 +44,18 @@ python '$__tool_directory__/bowtie_index_builder.py' + + + + + + + + + + + + - ${dbkey}/bowtie_index + genomes/${dbkey}/bowtie_index/v1/${value} - ${GALAXY_DATA_MANAGER_DATA_PATH}/${dbkey}/bowtie_index/${path} - abspath - - - - - - - - - - - - - - ${dbkey}/bowtie_index/color - - ${GALAXY_DATA_MANAGER_DATA_PATH}/${dbkey}/bowtie_index/color/${path} + ${GALAXY_DATA_MANAGER_DATA_PATH}/genomes/${dbkey}/bowtie_index/v1/${value}/${path} abspath diff --git a/data_managers/data_manager_bowtie_index_builder/test-data/all_fasta.loc b/data_managers/data_manager_bowtie_index_builder/test-data/all_fasta.loc new file mode 100644 index 00000000000..ef0c9d5364b --- /dev/null +++ b/data_managers/data_manager_bowtie_index_builder/test-data/all_fasta.loc @@ -0,0 +1,19 @@ +#This file lists the locations and dbkeys of all the fasta files +#under the "genome" directory (a directory that contains a directory +#for each build). The script extract_fasta.py will generate the file +#all_fasta.loc. This file has the format (white space characters are +#TAB characters): +# +# +# +#So, all_fasta.loc could look something like this: +# +#apiMel3 apiMel3 Honeybee (Apis mellifera): apiMel3 /path/to/genome/apiMel3/apiMel3.fa +#hg19canon hg19 Human (Homo sapiens): hg19 Canonical /path/to/genome/hg19/hg19canon.fa +#hg19full hg19 Human (Homo sapiens): hg19 Full /path/to/genome/hg19/hg19full.fa +# +#Your all_fasta.loc file should contain an entry for each individual +#fasta file. So there will be multiple fasta files for each build, +#such as with hg19 above. +# +phiX174 phiX174 phiX 174 ${__HERE__}/phiX174.fasta diff --git a/data_managers/data_manager_bowtie_index_builder/test-data/bowtie_data_manager.1.json b/data_managers/data_manager_bowtie_index_builder/test-data/bowtie_data_manager.1.json new file mode 100644 index 00000000000..3e359a63855 --- /dev/null +++ b/data_managers/data_manager_bowtie_index_builder/test-data/bowtie_data_manager.1.json @@ -0,0 +1,12 @@ +{ + "data_tables":{ + "bowtie_indexes":[ + { + "value": "phiX174", + "dbkey": "phiX174", + "name": "phiX 174", + "path": "phiX174.fasta" + } + ] + } +} diff --git a/data_managers/data_manager_bowtie_index_builder/test-data/bowtie_data_manager.2.json b/data_managers/data_manager_bowtie_index_builder/test-data/bowtie_data_manager.2.json new file mode 100644 index 00000000000..350c757d05a --- /dev/null +++ b/data_managers/data_manager_bowtie_index_builder/test-data/bowtie_data_manager.2.json @@ -0,0 +1,12 @@ +{ + "data_tables":{ + "bowtie_indexes":[ + { + "value": "tigHai1", + "dbkey": "phiX174", + "name": "Galeocerdo cuvier", + "path": "phiX174.fasta" + } + ] + } +} diff --git a/data_managers/data_manager_star_index_builder/test-data/rnastar_index2_versioned.loc b/data_managers/data_manager_bowtie_index_builder/test-data/bowtie_indices.loc similarity index 100% rename from data_managers/data_manager_star_index_builder/test-data/rnastar_index2_versioned.loc rename to data_managers/data_manager_bowtie_index_builder/test-data/bowtie_indices.loc diff --git a/data_managers/data_manager_bowtie_index_builder/test-data/phiX174.fasta b/data_managers/data_manager_bowtie_index_builder/test-data/phiX174.fasta new file mode 100644 index 00000000000..53df885dc48 --- /dev/null +++ b/data_managers/data_manager_bowtie_index_builder/test-data/phiX174.fasta @@ -0,0 +1,79 @@ +>phiX174 +GAGTTTTATCGCTTCCATGACGCAGAAGTTAACACTTTCGGATATTTCTGATGAGTCGAAAAATTATCTT +GATAAAGCAGGAATTACTACTGCTTGTTTACGAATTAAATCGAAGTGGACTGCTGGCGGAAAATGAGAAA +ATTCGACCTATCCTTGCGCAGCTCGAGAAGCTCTTACTTTGCGACCTTTCGCCATCAACTAACGATTCTG +TCAAAAACTGACGCGTTGGATGAGGAGAAGTGGCTTAATATGCTTGGCACGTTCGTCAAGGACTGGTTTA +GATATGAGTCACATTTTGTTCATGGTAGAGATTCTCTTGTTGACATTTTAAAAGAGCGTGGATTACTATC +TGAGTCCGATGCTGTTCAACCACTAATAGGTAAGAAATCATGAGTCAAGTTACTGAACAATCCGTACGTT +TCCAGACCGCTTTGGCCTCTATTAAGCTCATTCAGGCTTCTGCCGTTTTGGATTTAACCGAAGATGATTT +CGATTTTCTGACGAGTAACAAAGTTTGGATTGCTACTGACCGCTCTCGTGCTCGTCGCTGCGTTGAGGCT +TGCGTTTATGGTACGCTGGACTTTGTGGGATACCCTCGCTTTCCTGCTCCTGTTGAGTTTATTGCTGCCG +TCATTGCTTATTATGTTCATCCCGTCAACATTCAAACGGCCTGTCTCATCATGGAAGGCGCTGAATTTAC +GGAAAACATTATTAATGGCGTCGAGCGTCCGGTTAAAGCCGCTGAATTGTTCGCGTTTACCTTGCGTGTA +CGCGCAGGAAACACTGACGTTCTTACTGACGCAGAAGAAAACGTGCGTCAAAAATTACGTGCAGAAGGAG +TGATGTAATGTCTAAAGGTAAAAAACGTTCTGGCGCTCGCCCTGGTCGTCCGCAGCCGTTGCGAGGTACT +AAAGGCAAGCGTAAAGGCGCTCGTCTTTGGTATGTAGGTGGTCAACAATTTTAATTGCAGGGGCTTCGGC +CCCTTACTTGAGGATAAATTATGTCTAATATTCAAACTGGCGCCGAGCGTATGCCGCATGACCTTTCCCA +TCTTGGCTTCCTTGCTGGTCAGATTGGTCGTCTTATTACCATTTCAACTACTCCGGTTATCGCTGGCGAC +TCCTTCGAGATGGACGCCGTTGGCGCTCTCCGTCTTTCTCCATTGCGTCGTGGCCTTGCTATTGACTCTA +CTGTAGACATTTTTACTTTTTATGTCCCTCATCGTCACGTTTATGGTGAACAGTGGATTAAGTTCATGAA +GGATGGTGTTAATGCCACTCCTCTCCCGACTGTTAACACTACTGGTTATATTGACCATGCCGCTTTTCTT +GGCACGATTAACCCTGATACCAATAAAATCCCTAAGCATTTGTTTCAGGGTTATTTGAATATCTATAACA +ACTATTTTAAAGCGCCGTGGATGCCTGACCGTACCGAGGCTAACCCTAATGAGCTTAATCAAGATGATGC +TCGTTATGGTTTCCGTTGCTGCCATCTCAAAAACATTTGGACTGCTCCGCTTCCTCCTGAGACTGAGCTT +TCTCGCCAAATGACGACTTCTACCACATCTATTGACATTATGGGTCTGCAAGCTGCTTATGCTAATTTGC +ATACTGACCAAGAACGTGATTACTTCATGCAGCGTTACCGTGATGTTATTTCTTCATTTGGAGGTAAAAC +CTCTTATGACGCTGACAACCGTCCTTTACTTGTCATGCGCTCTAATCTCTGGGCATCTGGCTATGATGTT +GATGGAACTGACCAAACGTCGTTAGGCCAGTTTTCTGGTCGTGTTCAACAGACCTATAAACATTCTGTGC +CGCGTTTCTTTGTTCCTGAGCATGGCACTATGTTTACTCTTGCGCTTGTTCGTTTTCCGCCTACTGCGAC +TAAAGAGATTCAGTACCTTAACGCTAAAGGTGCTTTGACTTATACCGATATTGCTGGCGACCCTGTTTTG +TATGGCAACTTGCCGCCGCGTGAAATTTCTATGAAGGATGTTTTCCGTTCTGGTGATTCGTCTAAGAAGT +TTAAGATTGCTGAGGGTCAGTGGTATCGTTATGCGCCTTCGTATGTTTCTCCTGCTTATCACCTTCTTGA +AGGCTTCCCATTCATTCAGGAACCGCCTTCTGGTGATTTGCAAGAACGCGTACTTATTCGCCACCATGAT +TATGACCAGTGTTTCCAGTCCGTTCAGTTGTTGCAGTGGAATAGTCAGGTTAAATTTAATGTGACCGTTT +ATCGCAATCTGCCGACCACTCGCGATTCAATCATGACTTCGTGATAAAAGATTGAGTGTGAGGTTATAAC +GCCGAAGCGGTAAAAATTTTAATTTTTGCCGCTGAGGGGTTGACCAAGCGAAGCGCGGTAGGTTTTCTGC +TTAGGAGTTTAATCATGTTTCAGACTTTTATTTCTCGCCATAATTCAAACTTTTTTTCTGATAAGCTGGT +TCTCACTTCTGTTACTCCAGCTTCTTCGGCACCTGTTTTACAGACACCTAAAGCTACATCGTCAACGTTA +TATTTTGATAGTTTGACGGTTAATGCTGGTAATGGTGGTTTTCTTCATTGCATTCAGATGGATACATCTG +TCAACGCCGCTAATCAGGTTGTTTCTGTTGGTGCTGATATTGCTTTTGATGCCGACCCTAAATTTTTTGC +CTGTTTGGTTCGCTTTGAGTCTTCTTCGGTTCCGACTACCCTCCCGACTGCCTATGATGTTTATCCTTTG +AATGGTCGCCATGATGGTGGTTATTATACCGTCAAGGACTGTGTGACTATTGACGTCCTTCCCCGTACGC +CGGGCAATAATGTTTATGTTGGTTTCATGGTTTGGTCTAACTTTACCGCTACTAAATGCCGCGGATTGGT +TTCGCTGAATCAGGTTATTAAAGAGATTATTTGTCTCCAGCCACTTAAGTGAGGTGATTTATGTTTGGTG +CTATTGCTGGCGGTATTGCTTCTGCTCTTGCTGGTGGCGCCATGTCTAAATTGTTTGGAGGCGGTCAAAA +AGCCGCCTCCGGTGGCATTCAAGGTGATGTGCTTGCTACCGATAACAATACTGTAGGCATGGGTGATGCT +GGTATTAAATCTGCCATTCAAGGCTCTAATGTTCCTAACCCTGATGAGGCCGCCCCTAGTTTTGTTTCTG +GTGCTATGGCTAAAGCTGGTAAAGGACTTCTTGAAGGTACGTTGCAGGCTGGCACTTCTGCCGTTTCTGA +TAAGTTGCTTGATTTGGTTGGACTTGGTGGCAAGTCTGCCGCTGATAAAGGAAAGGATACTCGTGATTAT +CTTGCTGCTGCATTTCCTGAGCTTAATGCTTGGGAGCGTGCTGGTGCTGATGCTTCCTCTGCTGGTATGG +TTGACGCCGGATTTGAGAATCAAAAAGAGCTTACTAAAATGCAACTGGACAATCAGAAAGAGATTGCCGA +GATGCAAAATGAGACTCAAAAAGAGATTGCTGGCATTCAGTCGGCGACTTCACGCCAGAATACGAAAGAC +CAGGTATATGCACAAAATGAGATGCTTGCTTATCAACAGAAGGAGTCTACTGCTCGCGTTGCGTCTATTA +TGGAAAACACCAATCTTTCCAAGCAACAGCAGGTTTCCGAGATTATGCGCCAAATGCTTACTCAAGCTCA +AACGGCTGGTCAGTATTTTACCAATGACCAAATCAAAGAAATGACTCGCAAGGTTAGTGCTGAGGTTGAC +TTAGTTCATCAGCAAACGCAGAATCAGCGGTATGGCTCTTCTCATATTGGCGCTACTGCAAAGGATATTT +CTAATGTCGTCACTGATGCTGCTTCTGGTGTGGTTGATATTTTTCATGGTATTGATAAAGCTGTTGCCGA +TACTTGGAACAATTTCTGGAAAGACGGTAAAGCTGATGGTATTGGCTCTAATTTGTCTAGGAAATAACCG +TCAGGATTGACACCCTCCCAATTGTATGTTTTCATGCCTCCAAATCTTGGAGGCTTTTTTATGGTTCGTT +CTTATTACCCTTCTGAATGTCACGCTGATTATTTTGACTTTGAGCGTATCGAGGCTCTTAAACCTGCTAT +TGAGGCTTGTGGCATTTCTACTCTTTCTCAATCCCCAATGCTTGGCTTCCATAAGCAGATGGATAACCGC +ATCAAGCTCTTGGAAGAGATTCTGTCTTTTCGTATGCAGGGCGTTGAGTTCGATAATGGTGATATGTATG +TTGACGGCCATAAGGCTGCTTCTGACGTTCGTGATGAGTTTGTATCTGTTACTGAGAAGTTAATGGATGA +ATTGGCACAATGCTACAATGTGCTCCCCCAACTTGATATTAATAACACTATAGACCACCGCCCCGAAGGG +GACGAAAAATGGTTTTTAGAGAACGAGAAGACGGTTACGCAGTTTTGCCGCAAGCTGGCTGCTGAACGCC +CTCTTAAGGATATTCGCGATGAGTATAATTACCCCAAAAAGAAAGGTATTAAGGATGAGTGTTCAAGATT +GCTGGAGGCCTCCACTATGAAATCGCGTAGAGGCTTTACTATTCAGCGTTTGATGAATGCAATGCGACAG +GCTCATGCTGATGGTTGGTTTATCGTTTTTGACACTCTCACGTTGGCTGACGACCGATTAGAGGCGTTTT +ATGATAATCCCAATGCTTTGCGTGACTATTTTCGTGATATTGGTCGTATGGTTCTTGCTGCCGAGGGTCG +CAAGGCTAATGATTCACACGCCGACTGCTATCAGTATTTTTGTGTGCCTGAGTATGGTACAGCTAATGGC +CGTCTTCATTTCCATGCGGTGCATTTTATGCGGACACTTCCTACAGGTAGCGTTGACCCTAATTTTGGTC +GTCGGGTACGCAATCGCCGCCAGTTAAATAGCTTGCAAAATACGTGGCCTTATGGTTACAGTATGCCCAT +CGCAGTTCGCTACACGCAGGACGCTTTTTCACGTTCTGGTTGGTTGTGGCCTGTTGATGCTAAAGGTGAG +CCGCTTAAAGCTACCAGTTATATGGCTGTTGGTTTCTATGTGGCTAAATACGTTAACAAAAAGTCAGATA +TGGACCTTGCTGCTAAAGGTCTAGGAGCTAAAGAATGGAACAACTCACTAAAAACCAAGCTGTCGCTACT +TCCCAAGAAGCTGTTCAGAATCAGAATGAGCCGCAACTTCGGGATGAAAATGCTCACAATGACAAATCTG +TCCACGGAGTGCTTAATCCAACTTACCAAGCTGGGTTACGACGCGACGCCGTTCAACCAGATATTGAAGC +AGAACGCAAAAAGAGAGATGAGATTGAGGCTGGGAAAAGTTACTGTAGCCGACGTTTTGGCGGCGCAACC +TGTGACGACAAATCTGCTCAAATTTATGCGCGCTTCGATAAAAATGATTGGCGTATCCAACCTGCA + diff --git a/data_managers/data_manager_bowtie_index_builder/tool_data_table_conf.xml.sample b/data_managers/data_manager_bowtie_index_builder/tool_data_table_conf.xml.sample index f2a75b1396b..62b49851dbd 100644 --- a/data_managers/data_manager_bowtie_index_builder/tool_data_table_conf.xml.sample +++ b/data_managers/data_manager_bowtie_index_builder/tool_data_table_conf.xml.sample @@ -9,9 +9,4 @@ value, dbkey, name, path - - - value, dbkey, name, path - -
diff --git a/data_managers/data_manager_bowtie_index_builder/tool_data_table_conf.xml.test b/data_managers/data_manager_bowtie_index_builder/tool_data_table_conf.xml.test new file mode 100644 index 00000000000..b48c5ec9520 --- /dev/null +++ b/data_managers/data_manager_bowtie_index_builder/tool_data_table_conf.xml.test @@ -0,0 +1,12 @@ + + + + value, dbkey, name, path + +
+ + + value, dbkey, name, path + +
+ diff --git a/data_managers/data_manager_bwa_mem2_index_builder/data_manager/bwa_mem2_index_builder.xml b/data_managers/data_manager_bwa_mem2_index_builder/data_manager/bwa_mem2_index_builder.xml index 1d7af2d5099..920a09ad447 100644 --- a/data_managers/data_manager_bwa_mem2_index_builder/data_manager/bwa_mem2_index_builder.xml +++ b/data_managers/data_manager_bwa_mem2_index_builder/data_manager/bwa_mem2_index_builder.xml @@ -1,8 +1,8 @@ - + 2.2.1 - 0 + 1 bwa-mem2 @@ -52,7 +52,7 @@ - + diff --git a/data_managers/data_manager_bwa_mem2_index_builder/data_manager_conf.xml b/data_managers/data_manager_bwa_mem2_index_builder/data_manager_conf.xml index a757b4094d1..e5b834932f6 100644 --- a/data_managers/data_manager_bwa_mem2_index_builder/data_manager_conf.xml +++ b/data_managers/data_manager_bwa_mem2_index_builder/data_manager_conf.xml @@ -9,9 +9,9 @@ - ${dbkey}/bwa_mem2_index/${value} + genomes/${dbkey}/bwa_mem_index/v2/${value} - ${GALAXY_DATA_MANAGER_DATA_PATH}/${dbkey}/bwa_mem2_index/${value}/${path} + ${GALAXY_DATA_MANAGER_DATA_PATH}/genomes/${dbkey}/bwa_mem_index/v2/${value}/${path} abspath diff --git a/data_managers/data_manager_bwa_mem2_index_builder/test-data/all_fasta.loc b/data_managers/data_manager_bwa_mem2_index_builder/test-data/all_fasta.loc index c8b25f3f801..ef0c9d5364b 100644 --- a/data_managers/data_manager_bwa_mem2_index_builder/test-data/all_fasta.loc +++ b/data_managers/data_manager_bwa_mem2_index_builder/test-data/all_fasta.loc @@ -16,4 +16,4 @@ #fasta file. So there will be multiple fasta files for each build, #such as with hg19 above. # -phiX174 phiX174 phiX174 ${__HERE__}/phiX174.fasta +phiX174 phiX174 phiX 174 ${__HERE__}/phiX174.fasta diff --git a/data_managers/data_manager_bwa_mem2_index_builder/test-data/bwa_mem2_data_manager.1.json b/data_managers/data_manager_bwa_mem2_index_builder/test-data/bwa_mem2_data_manager.1.json index 344c77f9b7e..5efd7b96128 100644 --- a/data_managers/data_manager_bwa_mem2_index_builder/test-data/bwa_mem2_data_manager.1.json +++ b/data_managers/data_manager_bwa_mem2_index_builder/test-data/bwa_mem2_data_manager.1.json @@ -4,7 +4,7 @@ { "value": "phiX174", "dbkey": "phiX174", - "name": "phiX174", + "name": "phiX 174", "path": "phiX174.fasta" } ] diff --git a/data_managers/data_manager_bwa_mem2_index_builder/test-data/bwa_mem2_data_manager.2.json b/data_managers/data_manager_bwa_mem2_index_builder/test-data/bwa_mem2_data_manager.2.json index 259853414c6..82b974d6c18 100644 --- a/data_managers/data_manager_bwa_mem2_index_builder/test-data/bwa_mem2_data_manager.2.json +++ b/data_managers/data_manager_bwa_mem2_index_builder/test-data/bwa_mem2_data_manager.2.json @@ -2,7 +2,7 @@ "data_tables":{ "bwa_mem2_indexes":[ { - "value": "fooBar1", + "value": "tigHai1", "dbkey": "phiX174", "name": "Galeocerdo cuvier", "path": "phiX174.fasta" diff --git a/data_managers/data_manager_bwa_mem_index_builder/data_manager/bwa_mem_index_builder.py b/data_managers/data_manager_bwa_mem_index_builder/data_manager/bwa_mem_index_builder.py deleted file mode 100644 index c1a4b752429..00000000000 --- a/data_managers/data_manager_bwa_mem_index_builder/data_manager/bwa_mem_index_builder.py +++ /dev/null @@ -1,102 +0,0 @@ -#!/usr/bin/env python -# Dan Blankenberg -from __future__ import print_function - -import json -import optparse -import os -import subprocess -import sys - -CHUNK_SIZE = 2**20 -TWO_GB = 2**30 * 2 -DEFAULT_DATA_TABLE_NAME = "bwa_mem_indexes" - - -def get_id_name(params, dbkey, fasta_description=None): - # TODO: ensure sequence_id is unique and does not already appear in location file - sequence_id = params['param_dict']['sequence_id'] - if not sequence_id: - sequence_id = dbkey - - sequence_name = params['param_dict']['sequence_name'] - if not sequence_name: - sequence_name = fasta_description - if not sequence_name: - sequence_name = dbkey - return sequence_id, sequence_name - - -def build_bwa_index(data_manager_dict, fasta_filename, params, target_directory, dbkey, sequence_id, sequence_name, data_table_name=DEFAULT_DATA_TABLE_NAME): - # TODO: allow multiple FASTA input files - fasta_base_name = os.path.split(fasta_filename)[-1] - sym_linked_fasta_filename = os.path.join(target_directory, fasta_base_name) - os.symlink(fasta_filename, sym_linked_fasta_filename) - if params['param_dict']['index_algorithm'] == 'automatic': - if os.stat(fasta_filename).st_size < TWO_GB: # use 2 GB as cut off for memory vs. max of 2gb database size; this is somewhat arbitrary - index_algorithm = 'is' - else: - index_algorithm = 'bwtsw' - else: - index_algorithm = params['param_dict']['index_algorithm'] - - args = ['bwa', 'index', '-a', index_algorithm] - args.append(sym_linked_fasta_filename) - proc = subprocess.Popen(args=args, shell=False, cwd=target_directory) - return_code = proc.wait() - if return_code: - print("Error building index.", file=sys.stderr) - sys.exit(return_code) - data_table_entry = dict(value=sequence_id, dbkey=dbkey, name=sequence_name, path=fasta_base_name) - _add_data_table_entry(data_manager_dict, data_table_name, data_table_entry) - - -def _add_data_table_entry(data_manager_dict, data_table_name, data_table_entry): - data_manager_dict['data_tables'] = data_manager_dict.get('data_tables', {}) - data_manager_dict['data_tables'][data_table_name] = data_manager_dict['data_tables'].get(data_table_name, []) - data_manager_dict['data_tables'][data_table_name].append(data_table_entry) - return data_manager_dict - - -def main(): - parser = optparse.OptionParser() - parser.add_option('-f', '--fasta_filename', dest='fasta_filename', action='store', type="string", default=None, help='fasta_filename') - parser.add_option('-d', '--fasta_dbkey', dest='fasta_dbkey', action='store', type="string", default=None, help='fasta_dbkey') - parser.add_option('-t', '--fasta_description', dest='fasta_description', action='store', type="string", default=None, help='fasta_description') - parser.add_option('-n', '--data_table_name', dest='data_table_name', action='store', type="string", default=None, help='data_table_name') - (options, args) = parser.parse_args() - - filename = args[0] - - with open(filename) as fh: - params = json.load(fh) - target_directory = params['output_data'][0]['extra_files_path'] - os.mkdir(target_directory) - data_manager_dict = {} - - dbkey = options.fasta_dbkey - - if dbkey in [None, '', '?']: - raise Exception('"%s" is not a valid dbkey. You must specify a valid dbkey.' % (dbkey)) - - sequence_id, sequence_name = get_id_name(params, dbkey=dbkey, fasta_description=options.fasta_description) - - # build the index - build_bwa_index( - data_manager_dict, - options.fasta_filename, - params, - target_directory, - dbkey, - sequence_id, - sequence_name, - data_table_name=options.data_table_name or DEFAULT_DATA_TABLE_NAME - ) - - # save info to json file - with open(filename, 'w') as fh: - json.dump(data_manager_dict, fh, sort_keys=True) - - -if __name__ == "__main__": - main() diff --git a/data_managers/data_manager_bwa_mem_index_builder/data_manager/bwa_mem_index_builder.xml b/data_managers/data_manager_bwa_mem_index_builder/data_manager/bwa_mem_index_builder.xml index 2fc852772a7..796cf7afc0d 100644 --- a/data_managers/data_manager_bwa_mem_index_builder/data_manager/bwa_mem_index_builder.xml +++ b/data_managers/data_manager_bwa_mem_index_builder/data_manager/bwa_mem_index_builder.xml @@ -1,17 +1,42 @@ - + builder + + 0.7.18 + 0 + - bwa - python + bwa + + + @@ -23,6 +48,7 @@ + @@ -31,7 +57,14 @@ - + + + + + + + + diff --git a/data_managers/data_manager_bwa_mem_index_builder/data_manager_conf.xml b/data_managers/data_manager_bwa_mem_index_builder/data_manager_conf.xml index 800f576dbe1..2d5c31da52b 100644 --- a/data_managers/data_manager_bwa_mem_index_builder/data_manager_conf.xml +++ b/data_managers/data_manager_bwa_mem_index_builder/data_manager_conf.xml @@ -1,6 +1,5 @@ - @@ -10,13 +9,12 @@ - ${dbkey}/bwa_mem_index/${value} + genomes/${dbkey}/bwa_mem_index/v1/${value} - ${GALAXY_DATA_MANAGER_DATA_PATH}/${dbkey}/bwa_mem_index/${value}/${path} + ${GALAXY_DATA_MANAGER_DATA_PATH}/genomes/${dbkey}/bwa_mem_index/v1/${value}/${path} abspath - diff --git a/data_managers/data_manager_bwa_mem_index_builder/test-data/all_fasta.loc b/data_managers/data_manager_bwa_mem_index_builder/test-data/all_fasta.loc index c8b25f3f801..ef0c9d5364b 100644 --- a/data_managers/data_manager_bwa_mem_index_builder/test-data/all_fasta.loc +++ b/data_managers/data_manager_bwa_mem_index_builder/test-data/all_fasta.loc @@ -16,4 +16,4 @@ #fasta file. So there will be multiple fasta files for each build, #such as with hg19 above. # -phiX174 phiX174 phiX174 ${__HERE__}/phiX174.fasta +phiX174 phiX174 phiX 174 ${__HERE__}/phiX174.fasta diff --git a/data_managers/data_manager_bwa_mem_index_builder/test-data/bwa_mem_data_manager.1.json b/data_managers/data_manager_bwa_mem_index_builder/test-data/bwa_mem_data_manager.1.json new file mode 100644 index 00000000000..3bb4143238e --- /dev/null +++ b/data_managers/data_manager_bwa_mem_index_builder/test-data/bwa_mem_data_manager.1.json @@ -0,0 +1,12 @@ +{ + "data_tables":{ + "bwa_mem_indexes":[ + { + "value": "phiX174", + "dbkey": "phiX174", + "name": "phiX 174", + "path": "phiX174.fasta" + } + ] + } +} diff --git a/data_managers/data_manager_bwa_mem_index_builder/test-data/bwa_mem_data_manager.2.json b/data_managers/data_manager_bwa_mem_index_builder/test-data/bwa_mem_data_manager.2.json new file mode 100644 index 00000000000..bdbcc0a2f05 --- /dev/null +++ b/data_managers/data_manager_bwa_mem_index_builder/test-data/bwa_mem_data_manager.2.json @@ -0,0 +1,12 @@ +{ + "data_tables":{ + "bwa_mem_indexes":[ + { + "value": "tigHai1", + "dbkey": "phiX174", + "name": "Galeocerdo cuvier", + "path": "phiX174.fasta" + } + ] + } +} diff --git a/data_managers/data_manager_bwa_mem_index_builder/test-data/bwa_mem_data_manager.json b/data_managers/data_manager_bwa_mem_index_builder/test-data/bwa_mem_data_manager.json deleted file mode 100644 index b54c20a89e7..00000000000 --- a/data_managers/data_manager_bwa_mem_index_builder/test-data/bwa_mem_data_manager.json +++ /dev/null @@ -1 +0,0 @@ -{"data_tables": {"bwa_mem_indexes": [{"dbkey": "phiX174", "name": "phiX174", "path": "phiX174.fasta", "value": "phiX174"}]}} \ No newline at end of file diff --git a/data_managers/data_manager_fetch_genome_dbkeys_all_fasta/data_manager/data_manager_fetch_genome_all_fasta_dbkeys.py b/data_managers/data_manager_fetch_genome_dbkeys_all_fasta/data_manager/data_manager_fetch_genome_all_fasta_dbkeys.py index ec5ffafaa0f..3d578f95017 100644 --- a/data_managers/data_manager_fetch_genome_dbkeys_all_fasta/data_manager/data_manager_fetch_genome_all_fasta_dbkeys.py +++ b/data_managers/data_manager_fetch_genome_dbkeys_all_fasta/data_manager/data_manager_fetch_genome_all_fasta_dbkeys.py @@ -279,6 +279,8 @@ def add_fasta_to_table(data_manager_dict, fasta_readers, target_directory, dbkey target_directory=target_directory, dbkey=dbkey, dbkey_name=dbkey_name, + sequence_id=sequence_id, + sequence_name=sequence_name, fasta_filename=fasta_filename) _add_data_table_entry(data_manager_dict, data_table_entry=dict(value=sequence_id, dbkey=dbkey, name=sequence_name, path=os.path.basename(fasta_filename)), @@ -286,11 +288,11 @@ def add_fasta_to_table(data_manager_dict, fasta_readers, target_directory, dbkey return fasta_filename -def add_dbkey_to_table(data_manager_dict, target_directory, dbkey, dbkey_name, fasta_filename): +def add_dbkey_to_table(data_manager_dict, target_directory, dbkey, dbkey_name, sequence_id, sequence_name, fasta_filename): # do len calc here - len_base_name = "%s.len" % (dbkey) + len_base_name = f"{sequence_id}.len" compute_fasta_length(fasta_filename, os.path.join(target_directory, len_base_name), keep_first_word=True) - dbkey_dict = dict(value=dbkey, name=dbkey_name, len_path=len_base_name) + dbkey_dict = dict(dbkey=dbkey, value=sequence_id, name=sequence_name, len_path=len_base_name) _add_data_table_entry(data_manager_dict, data_table_entry=dbkey_dict, data_table_name='__dbkeys__') @@ -345,7 +347,7 @@ def copy_from_directory(data_manager_dict, params, target_directory, dbkey, dbke if isinstance(input_filename, list): fasta_readers = [get_stream_reader(open(filename, 'rb'), tmp_dir) for filename in input_filename] else: - fasta_readers = get_stream_reader(open(input_filename), tmp_dir) + fasta_readers = get_stream_reader(open(input_filename, 'rb'), tmp_dir) return fasta_readers for data_table_name, data_table_entry in data_table_entries: if data_table_entry: diff --git a/data_managers/data_manager_fetch_genome_dbkeys_all_fasta/data_manager/data_manager_fetch_genome_all_fasta_dbkeys.xml b/data_managers/data_manager_fetch_genome_dbkeys_all_fasta/data_manager/data_manager_fetch_genome_all_fasta_dbkeys.xml index 1c99f6ff365..b4f984e3e16 100644 --- a/data_managers/data_manager_fetch_genome_dbkeys_all_fasta/data_manager/data_manager_fetch_genome_all_fasta_dbkeys.xml +++ b/data_managers/data_manager_fetch_genome_dbkeys_all_fasta/data_manager/data_manager_fetch_genome_all_fasta_dbkeys.xml @@ -1,7 +1,7 @@ - + fetching - python + python + - - + + + + + + + + + + + + diff --git a/data_managers/data_manager_fetch_genome_dbkeys_all_fasta/data_manager_conf.xml b/data_managers/data_manager_fetch_genome_dbkeys_all_fasta/data_manager_conf.xml index 228f82ef26e..74a383a4fcf 100644 --- a/data_managers/data_manager_fetch_genome_dbkeys_all_fasta/data_manager_conf.xml +++ b/data_managers/data_manager_fetch_genome_dbkeys_all_fasta/data_manager_conf.xml @@ -10,9 +10,9 @@ ${path} - ${dbkey}/seq/${path} + genomes/${dbkey}/seq/${path} - ${GALAXY_DATA_MANAGER_DATA_PATH}/${dbkey}/seq/${path} + ${GALAXY_DATA_MANAGER_DATA_PATH}/genomes/${dbkey}/seq/${path} abspath @@ -24,9 +24,9 @@ ${len_path} - ${value}/len/${len_path} + genomes/${dbkey}/len/${len_path} - ${GALAXY_DATA_MANAGER_DATA_PATH}/${value}/len/${len_path} + ${GALAXY_DATA_MANAGER_DATA_PATH}/genomes/${dbkey}/len/${len_path} abspath diff --git a/data_managers/data_manager_fetch_genome_dbkeys_all_fasta/test-data/fetch_data_manager.1.data_manager_json b/data_managers/data_manager_fetch_genome_dbkeys_all_fasta/test-data/fetch_data_manager.1.data_manager_json new file mode 100644 index 00000000000..9b8d1007d36 --- /dev/null +++ b/data_managers/data_manager_fetch_genome_dbkeys_all_fasta/test-data/fetch_data_manager.1.data_manager_json @@ -0,0 +1 @@ +{"data_tables": {"__dbkeys__": [{"dbkey": "phiX174", "len_path": "phiX174.len", "name": "phiX174 sequence name", "value": "phiX174"}], "all_fasta": [{"dbkey": "phiX174", "name": "phiX174 sequence name", "path": "phiX174.fa", "value": "phiX174"}]}} \ No newline at end of file diff --git a/data_managers/data_manager_fetch_genome_dbkeys_all_fasta/test-data/fetch_data_manager.2.data_manager_json b/data_managers/data_manager_fetch_genome_dbkeys_all_fasta/test-data/fetch_data_manager.2.data_manager_json new file mode 100644 index 00000000000..d1b3f3d3703 --- /dev/null +++ b/data_managers/data_manager_fetch_genome_dbkeys_all_fasta/test-data/fetch_data_manager.2.data_manager_json @@ -0,0 +1 @@ +{"data_tables": {"__dbkeys__": [{"dbkey": "phiX174", "len_path": "phiX174Variant1.len", "name": "phiX174 Variant 1 sequence name", "value": "phiX174Variant1"}], "all_fasta": [{"dbkey": "phiX174", "name": "phiX174 Variant 1 sequence name", "path": "phiX174Variant1.fa", "value": "phiX174Variant1"}]}} \ No newline at end of file diff --git a/data_managers/data_manager_fetch_genome_dbkeys_all_fasta/test-data/phiX174.data_manager_json b/data_managers/data_manager_fetch_genome_dbkeys_all_fasta/test-data/phiX174.data_manager_json deleted file mode 100644 index 905e54a5c4e..00000000000 --- a/data_managers/data_manager_fetch_genome_dbkeys_all_fasta/test-data/phiX174.data_manager_json +++ /dev/null @@ -1 +0,0 @@ -{"data_tables": {"__dbkeys__": [{"len_path": "phiX174.len", "name": "phiX174", "value": "phiX174"}], "all_fasta": [{"dbkey": "phiX174", "name": "phiX174 sequence name", "path": "phix174.fa", "value": "phix174"}]}} \ No newline at end of file diff --git a/data_managers/data_manager_sam_fasta_index_builder/data_manager/data_manager_sam_fasta_index_builder.py b/data_managers/data_manager_sam_fasta_index_builder/data_manager/data_manager_sam_fasta_index_builder.py deleted file mode 100644 index 3520e0e4f8b..00000000000 --- a/data_managers/data_manager_sam_fasta_index_builder/data_manager/data_manager_sam_fasta_index_builder.py +++ /dev/null @@ -1,102 +0,0 @@ -#!/usr/bin/env python -# Dan Blankenberg - -import json -import optparse -import os -import subprocess -import sys -import tempfile - -CHUNK_SIZE = 2**20 - -DEFAULT_DATA_TABLE_NAME = "fasta_indexes" - - -def get_id_name(params, dbkey, fasta_description=None): - # TODO: ensure sequence_id is unique and does not already appear in location file - sequence_id = params['param_dict']['sequence_id'] - if not sequence_id: - sequence_id = dbkey - - sequence_name = params['param_dict']['sequence_name'] - if not sequence_name: - sequence_name = fasta_description - if not sequence_name: - sequence_name = dbkey - return sequence_id, sequence_name - - -def build_sam_index(data_manager_dict, fasta_filename, target_directory, dbkey, sequence_id, sequence_name, data_table_name=DEFAULT_DATA_TABLE_NAME): - # TODO: allow multiple FASTA input files - assert os.path.exists(fasta_filename), 'FASTA file "%s" is missing, cannot build samtools index.' % fasta_filename - fasta_base_name = os.path.split(fasta_filename)[-1] - sym_linked_fasta_filename = os.path.join(target_directory, fasta_base_name) - os.symlink(fasta_filename, sym_linked_fasta_filename) - - args = ['samtools', 'faidx'] - args.append(sym_linked_fasta_filename) - tmp_stderr = tempfile.NamedTemporaryFile(prefix="tmp-data-manager-sam_fa_index_builder-stderr") - proc = subprocess.Popen(args=args, shell=False, cwd=target_directory, stderr=tmp_stderr.fileno()) - return_code = proc.wait() - if return_code: - tmp_stderr.flush() - tmp_stderr.seek(0) - sys.stderr.write("Error building index:\n") - while True: - chunk = tmp_stderr.read(CHUNK_SIZE) - if not chunk: - break - sys.stderr.write(chunk) - sys.exit(return_code) - tmp_stderr.close() - data_table_entry = dict(value=sequence_id, dbkey=dbkey, name=sequence_name, path=fasta_base_name) - _add_data_table_entry(data_manager_dict, data_table_name, data_table_entry) - - -def _add_data_table_entry(data_manager_dict, data_table_name, data_table_entry): - data_manager_dict['data_tables'] = data_manager_dict.get('data_tables', {}) - data_manager_dict['data_tables'][data_table_name] = data_manager_dict['data_tables'].get(data_table_name, []) - data_manager_dict['data_tables'][data_table_name].append(data_table_entry) - return data_manager_dict - - -def main(): - parser = optparse.OptionParser() - parser.add_option('-f', '--fasta_filename', dest='fasta_filename', action='store', type="string", default=None, help='fasta_filename') - parser.add_option('-d', '--fasta_dbkey', dest='fasta_dbkey', action='store', type="string", default=None, help='fasta_dbkey') - parser.add_option('-t', '--fasta_description', dest='fasta_description', action='store', type="string", default=None, help='fasta_description') - parser.add_option('-n', '--data_table_name', dest='data_table_name', action='store', type="string", default=None, help='data_table_name') - (options, args) = parser.parse_args() - - filename = args[0] - - with open(filename) as fh: - params = json.load(fh) - target_directory = params['output_data'][0]['extra_files_path'] - os.mkdir(target_directory) - data_manager_dict = {} - - if options.fasta_dbkey in [None, '', '?']: - raise Exception('"%s" is not a valid dbkey. You must specify a valid dbkey.' % (options.fasta_dbkey)) - - sequence_id, sequence_name = get_id_name(params, dbkey=options.fasta_dbkey, fasta_description=options.fasta_description) - - # build the index - build_sam_index( - data_manager_dict, - options.fasta_filename, - target_directory, - options.fasta_dbkey, - sequence_id, - sequence_name, - data_table_name=options.data_table_name or DEFAULT_DATA_TABLE_NAME - ) - - # save info to json file - with open(filename, 'w') as fh: - json.dump(data_manager_dict, fh, sort_keys=True) - - -if __name__ == "__main__": - main() diff --git a/data_managers/data_manager_sam_fasta_index_builder/data_manager/data_manager_sam_fasta_index_builder.xml b/data_managers/data_manager_sam_fasta_index_builder/data_manager/data_manager_sam_fasta_index_builder.xml index 5467df4d4c1..29a4db5f307 100644 --- a/data_managers/data_manager_sam_fasta_index_builder/data_manager/data_manager_sam_fasta_index_builder.xml +++ b/data_managers/data_manager_sam_fasta_index_builder/data_manager/data_manager_sam_fasta_index_builder.xml @@ -1,17 +1,39 @@ - + builder + + 1.21 + 0 + - samtools - python + samtools + + + @@ -25,7 +47,13 @@ - + + + + + + + diff --git a/data_managers/data_manager_sam_fasta_index_builder/data_manager_conf.xml b/data_managers/data_manager_sam_fasta_index_builder/data_manager_conf.xml index 261a52a2191..84e0e793069 100644 --- a/data_managers/data_manager_sam_fasta_index_builder/data_manager_conf.xml +++ b/data_managers/data_manager_sam_fasta_index_builder/data_manager_conf.xml @@ -1,6 +1,5 @@ - @@ -10,13 +9,12 @@ - ${dbkey}/sam_indexes/${value} + genomes/${dbkey}/sam_fasta_index/v1/${value} - ${GALAXY_DATA_MANAGER_DATA_PATH}/${dbkey}/sam_indexes/${value}/${path} + ${GALAXY_DATA_MANAGER_DATA_PATH}/genomes/${dbkey}/sam_fasta_index/v1/${value}/${path} abspath - diff --git a/data_managers/data_manager_sam_fasta_index_builder/test-data/all_fasta.loc b/data_managers/data_manager_sam_fasta_index_builder/test-data/all_fasta.loc index c8b25f3f801..ef0c9d5364b 100644 --- a/data_managers/data_manager_sam_fasta_index_builder/test-data/all_fasta.loc +++ b/data_managers/data_manager_sam_fasta_index_builder/test-data/all_fasta.loc @@ -16,4 +16,4 @@ #fasta file. So there will be multiple fasta files for each build, #such as with hg19 above. # -phiX174 phiX174 phiX174 ${__HERE__}/phiX174.fasta +phiX174 phiX174 phiX 174 ${__HERE__}/phiX174.fasta diff --git a/data_managers/data_manager_sam_fasta_index_builder/test-data/sam_fasta_data_manager.1.json b/data_managers/data_manager_sam_fasta_index_builder/test-data/sam_fasta_data_manager.1.json new file mode 100644 index 00000000000..8fdc594b3ef --- /dev/null +++ b/data_managers/data_manager_sam_fasta_index_builder/test-data/sam_fasta_data_manager.1.json @@ -0,0 +1,12 @@ +{ + "data_tables":{ + "fasta_indexes":[ + { + "value": "phiX174", + "dbkey": "phiX174", + "name": "phiX 174", + "path": "phiX174.fasta" + } + ] + } +} diff --git a/data_managers/data_manager_sam_fasta_index_builder/test-data/sam_fasta_data_manager.2.json b/data_managers/data_manager_sam_fasta_index_builder/test-data/sam_fasta_data_manager.2.json new file mode 100644 index 00000000000..5c7bef0253b --- /dev/null +++ b/data_managers/data_manager_sam_fasta_index_builder/test-data/sam_fasta_data_manager.2.json @@ -0,0 +1,12 @@ +{ + "data_tables":{ + "fasta_indexes":[ + { + "value": "tigHai1", + "dbkey": "phiX174", + "name": "Galeocerdo cuvier", + "path": "phiX174.fasta" + } + ] + } +} diff --git a/data_managers/data_manager_sam_fasta_index_builder/test-data/sam_fasta_data_manager.json b/data_managers/data_manager_sam_fasta_index_builder/test-data/sam_fasta_data_manager.json deleted file mode 100644 index d62a0f44953..00000000000 --- a/data_managers/data_manager_sam_fasta_index_builder/test-data/sam_fasta_data_manager.json +++ /dev/null @@ -1 +0,0 @@ -{"data_tables": {"fasta_indexes": [{"dbkey": "phiX174", "name": "phiX174", "path": "phiX174.fasta", "value": "phiX174"}]}} \ No newline at end of file diff --git a/data_managers/data_manager_sam_fasta_index_builder/tool_data_table_conf.xml.sample b/data_managers/data_manager_sam_fasta_index_builder/tool_data_table_conf.xml.sample index 4ff4db4a470..481d7c80120 100644 --- a/data_managers/data_manager_sam_fasta_index_builder/tool_data_table_conf.xml.sample +++ b/data_managers/data_manager_sam_fasta_index_builder/tool_data_table_conf.xml.sample @@ -1,10 +1,10 @@ - value, dbkey, name, path
+ value, dbkey, name, path diff --git a/data_managers/data_manager_star_index_builder/data_manager/rna_star_index_builder.py b/data_managers/data_manager_star_index_builder/data_manager/rna_star_index_builder.py deleted file mode 100644 index 9658cf1f3f0..00000000000 --- a/data_managers/data_manager_star_index_builder/data_manager/rna_star_index_builder.py +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env python - -import argparse -import json - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument('--config-file') - parser.add_argument('--value') - parser.add_argument('--dbkey') - parser.add_argument('--name') - parser.add_argument('--subdir') - parser.add_argument('--data-table') - parser.add_argument('--with-gene-model', action='store_true') - parser.add_argument('--index-version') - - args = parser.parse_args() - - if args.dbkey in [None, '', '?']: - raise Exception( - '"%s" is not a valid dbkey. You must specify a valid dbkey.' - % (args.dbkey) - ) - - with_gene_model = "0" - if args.with_gene_model: - with_gene_model = "1" - - data_manager_dict = { - 'data_tables': { - args.data_table: [ - { - "value": args.value, - "dbkey": args.dbkey, - "name": args.name, - "path": args.subdir, - "with_gene_model": with_gene_model, - "version": args.index_version - } - ] - } - } - with open(args.config_file, 'w') as fh: - json.dump(data_manager_dict, fh, sort_keys=True) - - -if __name__ == "__main__": - main() diff --git a/data_managers/data_manager_star_index_builder/data_manager/rna_star_index_builder.xml b/data_managers/data_manager_star_index_builder/data_manager/rna_star_index_builder.xml index 0bea85cce12..2d7bfc9712f 100644 --- a/data_managers/data_manager_star_index_builder/data_manager/rna_star_index_builder.xml +++ b/data_managers/data_manager_star_index_builder/data_manager/rna_star_index_builder.xml @@ -1,15 +1,15 @@ - + builder macros.xml - - python - + ' '${all_fasta_source.fields.path}' | tr -d '\n' | wc -c)" && +echo "Bases in reference: \$nbases" && +#end if + +#if $auto_sa_index_nbases: +saindex_nbases=\$((\$(log2 \$nbases) / 2 - 1)) && +[[ \$saindex_nbases -lt 14 ]] || saindex_nbases=14 && +#else if $advanced_options.advanced_options_selector == "advanced": +saindex_nbases=${advanced_options.genomeSAindexNbases} && +#end if + +#if $auto_chr_bin_nbits: +nseqs="\$(grep -c '>' '${all_fasta_source.fields.path}')" && +echo "Sequences in reference: \$nseqs" && +chr_bin_nbits=\$((\$(log2 \$nbases) / \$(log2 \$nseqs))) && +[[ \$chr_bin_nbits -lt 18 ]] || chr_bin_nbits=18 && +#else if $advanced_options.advanced_options_selector == "advanced": +chr_bin_nbits=${advanced_options.genomeChrBinNbits} && +#end if + STAR --runMode genomeGenerate --genomeFastaFiles '${all_fasta_source.fields.path}' @@ -31,29 +52,52 @@ STAR --sjdbGTFfile '${GTFconditional.sjdbGTFfile}' --sjdbOverhang ${GTFconditional.sjdbOverhang} #end if +#if $advanced_options.advanced_options_selector == "advanced" or $auto_sa_index_nbases: + --genomeSAindexNbases "\$saindex_nbases" +#end if +#if $advanced_options.advanced_options_selector == "advanced" or $auto_chr_bin_nbits: + --genomeChrBinNbits "\$chr_bin_nbits" +#end if #if $advanced_options.advanced_options_selector == "advanced": - --genomeSAindexNbases ${advanced_options.genomeSAindexNbases} - --genomeChrBinNbits ${advanced_options.genomeChrBinNbits} --genomeSAsparseD ${advanced_options.genomeSAsparseD} #end if --runThreadN \${GALAXY_SLOTS:-2} && -python '${__tool_directory__}/rna_star_index_builder.py' ---config-file '${out_file}' ---value '${all_fasta_source.fields.value}' ---dbkey '${all_fasta_source.fields.dbkey}' ---index-version '@IDX_VERSION@' -#if $name: - --name '$name' -#else - --name '${all_fasta_source.fields.name}' -#end if -#if str($GTFconditional.GTFselect) == "withGTF": - --with-gene-model -#end if ---data-table @IDX_DATA_TABLE@ ---subdir '${subdir}' +cp '$dmjson' '$out_file' ]]> + + > 1)) + log2=\$((log2 + 1)) + done + [[ \$log2 -gt 0 ]] && echo \$log2 || echo 1 +} +]]> + + @@ -71,6 +115,12 @@ python '${__tool_directory__}/rna_star_index_builder.py' + + @@ -90,7 +140,8 @@ python '${__tool_directory__}/rna_star_index_builder.py' of contigs, it is recommended to scale this parameter as min(18, log2[max(GenomeLength/NumberOfReferences,ReadLength)]). For example, for 3 gigaBase genome with 100,000 chromosomes/scaffolds, this is equal to 15."/> - @@ -104,7 +155,7 @@ python '${__tool_directory__}/rna_star_index_builder.py' - + diff --git a/data_managers/data_manager_star_index_builder/data_manager_conf.xml b/data_managers/data_manager_star_index_builder/data_manager_conf.xml index 3ea637445e2..62802b69f7c 100644 --- a/data_managers/data_manager_star_index_builder/data_manager_conf.xml +++ b/data_managers/data_manager_star_index_builder/data_manager_conf.xml @@ -12,9 +12,9 @@ out_file.extra_files_path is used as base by default if no source, eg for type=directory, then refers to base --> - rnastar/${version}/${dbkey}/${value}/${path} + genomes/${dbkey}/rnastar_index/v${version}/${value} - ${GALAXY_DATA_MANAGER_DATA_PATH}/rnastar/${version}/${dbkey}/${value}/${path} + ${GALAXY_DATA_MANAGER_DATA_PATH}/genomes/${dbkey}/rnastar_index/v${version}/${value} abspath diff --git a/data_managers/data_manager_star_index_builder/test-data/rnastar_index2x_versioned.loc b/data_managers/data_manager_star_index_builder/test-data/rnastar_index2x_versioned.loc new file mode 100644 index 00000000000..e69de29bb2d diff --git a/data_managers/data_manager_star_index_builder/test-data/test_star_01.data_manager_json b/data_managers/data_manager_star_index_builder/test-data/test_star_01.data_manager_json index 18d3872a851..8b5ef4d83e2 100644 --- a/data_managers/data_manager_star_index_builder/test-data/test_star_01.data_manager_json +++ b/data_managers/data_manager_star_index_builder/test-data/test_star_01.data_manager_json @@ -1 +1,14 @@ -{"data_tables": {"rnastar_index2x_versioned": \[{"dbkey": "phiX174", "name": "phiX174", "path": ".*", "value": "phiX174", "version": "2.7.4a", "with_gene_model": "0"}\]}} +{ + "data_tables":{ + "rnastar_index2x_versioned":[ + { + "value": "phiX174", + "dbkey": "phiX174", + "name": "phiX174", + "path": "SA", + "with_gene_model": "0", + "version": "2.7.4a" + } + ] + } +} diff --git a/data_managers/data_manager_star_index_builder/tool_data_table_conf.xml.test b/data_managers/data_manager_star_index_builder/tool_data_table_conf.xml.test index 6410d9b7581..20c853544cc 100644 --- a/data_managers/data_manager_star_index_builder/tool_data_table_conf.xml.test +++ b/data_managers/data_manager_star_index_builder/tool_data_table_conf.xml.test @@ -5,8 +5,8 @@
- +
value, dbkey, name, path, with_gene_model, version - +
diff --git a/data_managers/data_manager_bowtie_index_builder/data_manager/bowtie_color_space_index_builder.xml b/deprecated/data_managers/data_manager_bowtie_index_builder/data_manager/bowtie_color_space_index_builder.xml similarity index 100% rename from data_managers/data_manager_bowtie_index_builder/data_manager/bowtie_color_space_index_builder.xml rename to deprecated/data_managers/data_manager_bowtie_index_builder/data_manager/bowtie_color_space_index_builder.xml diff --git a/data_managers/data_manager_bowtie_index_builder/data_manager/bowtie_index_builder.py b/deprecated/data_managers/data_manager_bowtie_index_builder/data_manager/bowtie_index_builder.py similarity index 100% rename from data_managers/data_manager_bowtie_index_builder/data_manager/bowtie_index_builder.py rename to deprecated/data_managers/data_manager_bowtie_index_builder/data_manager/bowtie_index_builder.py diff --git a/deprecated/data_managers/data_manager_bowtie_index_builder/data_manager_conf.xml b/deprecated/data_managers/data_manager_bowtie_index_builder/data_manager_conf.xml new file mode 100644 index 00000000000..299b794e0b1 --- /dev/null +++ b/deprecated/data_managers/data_manager_bowtie_index_builder/data_manager_conf.xml @@ -0,0 +1,20 @@ + + + + + + + + + + + + ${dbkey}/bowtie_index/color + + ${GALAXY_DATA_MANAGER_DATA_PATH}/${dbkey}/bowtie_index/color/${path} + abspath + + + + + diff --git a/deprecated/data_managers/data_manager_bowtie_index_builder/tool_data_table_conf.xml.sample b/deprecated/data_managers/data_manager_bowtie_index_builder/tool_data_table_conf.xml.sample new file mode 100644 index 00000000000..16a4dd8a368 --- /dev/null +++ b/deprecated/data_managers/data_manager_bowtie_index_builder/tool_data_table_conf.xml.sample @@ -0,0 +1,12 @@ + + + + value, dbkey, name, path + +
+ + + value, dbkey, name, path + +
+ diff --git a/tools/rgrnastar/macros.xml b/tools/rgrnastar/macros.xml index d5ab47d0099..5d027ce5612 100644 --- a/tools/rgrnastar/macros.xml +++ b/tools/rgrnastar/macros.xml @@ -15,7 +15,7 @@ or by looking for the versionGenome parameter in source/parametersDefault of STAR's source code --> 2.7.4a - 2 + 3 rnastar_index2x_versioned