diff --git a/files/build-files-from-ncbi.py b/files/build-files-from-ncbi.py index 0c2c112..1b8da76 100644 --- a/files/build-files-from-ncbi.py +++ b/files/build-files-from-ncbi.py @@ -59,22 +59,6 @@ def get_genome_row(genome_info): def get_genomes_df(tax_ids): return pd.DataFrame(data=[get_genome_row(genome_info) for genome_info in requests.get(build_genomes_url(tax_ids)).json()["reports"]]) -def print_column_match_summary(from_df, in_df, from_column, in_column, important=True): - unmatched_values = from_df[from_column][~(from_df[from_column].isin(in_df[in_column]))] - message = ( - f"No values from {from_column} absent in {in_column}" if len(unmatched_values) == 0 - else f"{len(unmatched_values)} values from {from_column} absent in {in_column}: {", ".join(unmatched_values)}" - ) - if not important: - message = "(" + message + ")" - print(message) - -def print_accession_match_summaries(genomes_source_df, assemblies_df): - print_column_match_summary(genomes_source_df, assemblies_df, "pairedAccession", "genBank") - print_column_match_summary(genomes_source_df, assemblies_df, "pairedAccession", "refSeq", False) - print_column_match_summary(genomes_source_df, assemblies_df, "accession", "genBank", False) - print_column_match_summary(genomes_source_df, assemblies_df, "accession", "refSeq") - def _id_to_gene_model_url(asm_id): hubs_url = "https://hgdownload.soe.ucsc.edu/hubs/" components = [asm_id[0:3], asm_id[4:7], asm_id[7:10], asm_id[10:13], asm_id, "genes"] @@ -122,10 +106,12 @@ def build_files(): genomes_source_df = get_genomes_df(get_tax_ids(organisms_df)) assemblies_df = pd.DataFrame(requests.get(ASSEMBLIES_URL).json()["data"])[["ucscBrowser", "genBank", "refSeq"]] - gen_bank_merge_df = genomes_source_df.merge(assemblies_df, how="left", left_on="pairedAccession", right_on="genBank") + gen_bank_merge_df = genomes_source_df.merge(assemblies_df, how="left", left_on="accession", right_on="genBank") ref_seq_merge_df = genomes_source_df.merge(assemblies_df, how="left", left_on="accession", right_on="refSeq") - print_accession_match_summaries(genomes_source_df, assemblies_df) + unmatched_accessions = genomes_source_df["accession"][~(genomes_source_df["accession"].isin(assemblies_df["genBank"]) | genomes_source_df["accession"].isin(assemblies_df["refSeq"]))] + if len(unmatched_accessions) > 0: + print(f"{len(unmatched_accessions)} accessions had no match in assembly list: {", ".join(unmatched_accessions)}") genomes_df = add_gene_model_url(gen_bank_merge_df.combine_first(ref_seq_merge_df)) diff --git a/files/out/genomes.json b/files/out/genomes.json index d7bec9c..be5dfaf 100644 --- a/files/out/genomes.json +++ b/files/out/genomes.json @@ -41,7 +41,7 @@ "accession": "GCF_000002725.2", "annotationStatus": "Full annotation", "chromosomes": 36, - "coverage": "100.0x", + "coverage": null, "gcPercent": 59.5, "geneModelUrl": "https://hgdownload.soe.ucsc.edu/hubs/GCF/000/002/725/GCF_000002725.2/genes/GCF_000002725.2_ASM272v2.ncbiRefSeq.gtf.gz", "isRef": "Yes", @@ -55,24 +55,6 @@ "taxon": "Leishmania major strain Friedlin", "ucscBrowserUrl": "https://genome.ucsc.edu/h/GCF_000002725.2" }, - { - "accession": "GCF_000002765.6", - "annotationStatus": "Full annotation", - "chromosomes": 14, - "coverage": "100.0x", - "gcPercent": 19.5, - "geneModelUrl": "https://hgdownload.soe.ucsc.edu/hubs/GCF/000/002/765/GCF_000002765.6/genes/GCF_000002765.6_GCA_000002765.ncbiRefSeq.gtf.gz", - "isRef": "Yes", - "length": 23292622, - "level": "Complete Genome", - "ncbiTaxonomyId": "36329", - "scaffoldCount": 14, - "scaffoldL50": 5, - "scaffoldN50": 1687656, - "tags": [], - "taxon": "Plasmodium falciparum 3D7", - "ucscBrowserUrl": "https://genome.ucsc.edu/h/GCF_000002765.5" - }, { "accession": "GCF_000002765.6", "annotationStatus": "Full annotation", @@ -93,7 +75,7 @@ }, { "accession": "GCF_000002845.2", - "annotationStatus": "Full annotation", + "annotationStatus": null, "chromosomes": 35, "coverage": null, "gcPercent": 58, @@ -111,7 +93,7 @@ }, { "accession": "GCF_000006565.2", - "annotationStatus": "Full annotation", + "annotationStatus": null, "chromosomes": 14, "coverage": "26.5x", "gcPercent": 52.5, @@ -130,7 +112,7 @@ { "accession": "GCF_000149335.2", "annotationStatus": "Full annotation", - "chromosomes": 1, + "chromosomes": null, "coverage": null, "gcPercent": 46, "geneModelUrl": "https://hgdownload.soe.ucsc.edu/hubs/GCF/000/149/335/GCF_000149335.2/genes/GCF_000149335.2_ASM14933v2.ncbiRefSeq.gtf.gz", @@ -166,7 +148,7 @@ { "accession": "GCF_000209065.1", "annotationStatus": "Full annotation", - "chromosomes": 1, + "chromosomes": null, "coverage": null, "gcPercent": 51.5, "geneModelUrl": "https://hgdownload.soe.ucsc.edu/hubs/GCF/000/209/065/GCF_000209065.1/genes/GCF_000209065.1_ASM20906v1.ncbiRefSeq.gtf.gz", @@ -219,13 +201,13 @@ "scaffoldN50": 4411709, "tags": [], "taxon": "Mycobacterium tuberculosis H37Rv", - "ucscBrowserUrl": "https://genome.ucsc.edu/h/GCF_000857045.1" + "ucscBrowserUrl": null }, { "accession": "GCF_000857045.1", "annotationStatus": null, "chromosomes": 1, - "coverage": "100.0x", + "coverage": null, "gcPercent": 33, "geneModelUrl": null, "isRef": "No", @@ -245,7 +227,7 @@ "accession": "GCF_009858895.2", "annotationStatus": null, "chromosomes": 1, - "coverage": "20.0x", + "coverage": null, "gcPercent": 38, "geneModelUrl": null, "isRef": "No", @@ -279,24 +261,6 @@ "taxon": "Culex pipiens pallens", "ucscBrowserUrl": "https://genome.ucsc.edu/h/GCF_016801865.2" }, - { - "accession": "GCF_018416015.2", - "annotationStatus": "Full annotation", - "chromosomes": 9, - "coverage": "475.0x", - "gcPercent": 46.5, - "geneModelUrl": "https://hgdownload.soe.ucsc.edu/hubs/GCF/018/416/015/GCF_018416015.2/genes/GCF_018416015.2_ASM1841601v2.ncbiRefSeq.gtf.gz", - "isRef": "Yes", - "length": 28193268, - "level": "Complete Genome", - "ncbiTaxonomyId": "443226", - "scaffoldCount": 9, - "scaffoldL50": 2, - "scaffoldN50": 8079863, - "tags": [], - "taxon": "Coccidioides posadasii str. Silveira", - "ucscBrowserUrl": "https://genome.ucsc.edu/h/GCA_018416015.2" - }, { "accession": "GCF_018416015.2", "annotationStatus": "Full annotation", diff --git a/files/source/genomes-from-ncbi.tsv b/files/source/genomes-from-ncbi.tsv index 24926e2..a0f918b 100644 --- a/files/source/genomes-from-ncbi.tsv +++ b/files/source/genomes-from-ncbi.tsv @@ -1,23 +1,21 @@ taxon taxonomyId accession isRef level chromosomeCount length scaffoldCount scaffoldN50 scaffoldL50 coverage gcPercent annotationStatus pairedAccession ucscBrowser genBank refSeq geneModelUrl Mycobacterium tuberculosis H37Rv 83332 GCF_000195955.2 True Complete Genome 1.0 4411532 1 4411532 1 65.5 GCA_000195955.2 https://genome.ucsc.edu/h/GCF_000195955.2 GCA_000195955.2 GCF_000195955.2 -Plasmodium falciparum 3D7 36329 GCF_000002765.6 True Complete Genome 14.0 23292622 14 1687656 5 100.0x 19.5 Full annotation GCA_000002765.3 https://genome.ucsc.edu/h/GCF_000002765.5 GCA_000002765.3 GCF_000002765.5 https://hgdownload.soe.ucsc.edu/hubs/GCF/000/002/765/GCF_000002765.6/genes/GCF_000002765.6_GCA_000002765.ncbiRefSeq.gtf.gz Plasmodium falciparum 3D7 36329 GCF_000002765.6 True Complete Genome 14.0 23292622 14 1687656 5 100.0x 19.5 Full annotation GCA_000002765.3 https://genome.ucsc.edu/h/GCF_000002765.6 GCA_000002765.3 GCF_000002765.6 https://hgdownload.soe.ucsc.edu/hubs/GCF/000/002/765/GCF_000002765.6/genes/GCF_000002765.6_GCA_000002765.ncbiRefSeq.gtf.gz -Leishmania major strain Friedlin 347515 GCF_000002725.2 True Complete Genome 36.0 32855089 36 1091540 11 100.0x 59.5 Full annotation GCA_000002725.2 https://genome.ucsc.edu/h/GCF_000002725.2 GCA_000002725.2 GCF_000002725.2 https://hgdownload.soe.ucsc.edu/hubs/GCF/000/002/725/GCF_000002725.2/genes/GCF_000002725.2_ASM272v2.ncbiRefSeq.gtf.gz +Leishmania major strain Friedlin 347515 GCF_000002725.2 True Complete Genome 36.0 32855089 36 1091540 11 59.5 Full annotation GCA_000002725.2 https://genome.ucsc.edu/h/GCF_000002725.2 GCA_000002725.2 GCF_000002725.2 https://hgdownload.soe.ucsc.edu/hubs/GCF/000/002/725/GCF_000002725.2/genes/GCF_000002725.2_ASM272v2.ncbiRefSeq.gtf.gz Plasmodium yoelii 5861 GCF_900002385.2 True Complete Genome 14.0 23043114 14 2046250 5 100.0x 21.5 Full annotation GCA_900002385.2 https://genome.ucsc.edu/h/GCF_900002385.2 GCA_900002385.2 GCF_900002385.2 https://hgdownload.soe.ucsc.edu/hubs/GCF/900/002/385/GCF_900002385.2/genes/GCF_900002385.2_GCA_900002385.ncbiRefSeq.gtf.gz -Coccidioides posadasii str. Silveira 443226 GCF_018416015.2 True Complete Genome 9.0 28193268 9 8079863 2 475.0x 46.5 Full annotation GCA_018416015.2 https://genome.ucsc.edu/h/GCA_018416015.2 GCA_018416015.2 GCF_018416015.1 https://hgdownload.soe.ucsc.edu/hubs/GCF/018/416/015/GCF_018416015.2/genes/GCF_018416015.2_ASM1841601v2.ncbiRefSeq.gtf.gz Coccidioides posadasii str. Silveira 443226 GCF_018416015.2 True Complete Genome 9.0 28193268 9 8079863 2 475.0x 46.5 Full annotation GCA_018416015.2 https://genome.ucsc.edu/h/GCF_018416015.2 GCA_018416015.2 GCF_018416015.2 https://hgdownload.soe.ucsc.edu/hubs/GCF/018/416/015/GCF_018416015.2/genes/GCF_018416015.2_ASM1841601v2.ncbiRefSeq.gtf.gz Plasmodium vinckei vinckei 54757 GCF_900681995.1 True Chromosome 14.0 18338688 14 1692345 5 155.0x 23.0 Full annotation GCA_900681995.1 https://genome.ucsc.edu/h/GCF_900681995.1 GCA_900681995.1 GCF_900681995.1 https://hgdownload.soe.ucsc.edu/hubs/GCF/900/681/995/GCF_900681995.1/genes/GCF_900681995.1_PVVCY_v1.ncbiRefSeq.gtf.gz Leishmania donovani 5661 GCF_000227135.1 True Chromosome 36.0 32444968 36 1024085 11 59.5 Full annotation GCA_000227135.2 https://genome.ucsc.edu/h/GCF_000227135.1 GCA_000227135.2 GCF_000227135.1 https://hgdownload.soe.ucsc.edu/hubs/GCF/000/227/135/GCF_000227135.1/genes/GCF_000227135.1_ASM22713v2.ncbiRefSeq.gtf.gz -Toxoplasma gondii ME49 508771 GCF_000006565.2 True Chromosome 14.0 65633124 2276 4973582 6 26.5x 52.5 Full annotation GCA_000006565.2 https://genome.ucsc.edu/h/GCF_000006565.2 GCA_000006565.2 GCF_000006565.2 https://hgdownload.soe.ucsc.edu/hubs/GCF/000/006/565/GCF_000006565.2/genes/GCF_000006565.2_TGA4.ncbiRefSeq.gtf.gz +Toxoplasma gondii ME49 508771 GCF_000006565.2 True Chromosome 14.0 65633124 2276 4973582 6 26.5x 52.5 GCA_000006565.2 https://genome.ucsc.edu/h/GCF_000006565.2 GCA_000006565.2 GCF_000006565.2 https://hgdownload.soe.ucsc.edu/hubs/GCF/000/006/565/GCF_000006565.2/genes/GCF_000006565.2_TGA4.ncbiRefSeq.gtf.gz Trypanosoma brucei brucei TREU927 185431 GCF_000002445.2 True Chromosome 11.0 26075494 12 2481190 4 46.5 Full annotation GCA_000002445.1 https://genome.ucsc.edu/h/GCF_000002445.2 GCA_000002445.1 GCF_000002445.2 https://hgdownload.soe.ucsc.edu/hubs/GCF/000/002/445/GCF_000002445.2/genes/GCF_000002445.2_ASM244v1.ncbiRefSeq.gtf.gz Anopheles gambiae 7165 GCF_943734735.2 True Chromosome 3.0 264451381 190 99149756 2 54.0x 44.5 Full annotation GCA_943734735.2 https://genome.ucsc.edu/h/GCF_943734735.2 GCA_943734735.2 GCF_943734735.2 https://hgdownload.soe.ucsc.edu/hubs/GCF/943/734/735/GCF_943734735.2/genes/GCF_943734735.2_idAnoGambNW_F1_1.ncbiRefSeq.gtf.gz Plasmodium vivax 5855 GCF_000002415.2 True Chromosome 14.0 27007701 2747 1678596 6 42.5 Full annotation GCA_000002415.2 https://genome.ucsc.edu/h/GCF_000002415.2 GCA_000002415.2 GCF_000002415.2 https://hgdownload.soe.ucsc.edu/hubs/GCF/000/002/415/GCF_000002415.2/genes/GCF_000002415.2_ASM241v2.ncbiRefSeq.gtf.gz Culex pipiens pallens 42434 GCF_016801865.2 True Chromosome 3.0 566339288 289 186194774 2 250.0x 37.0 Full annotation GCA_016801865.2 https://genome.ucsc.edu/h/GCF_016801865.2 GCA_016801865.2 GCF_016801865.2 https://hgdownload.soe.ucsc.edu/hubs/GCF/016/801/865/GCF_016801865.2/genes/GCF_016801865.2_TS_CPP_V2.ncbiRefSeq.gtf.gz -Leishmania braziliensis MHOM/BR/75/M2904 420245 GCF_000002845.2 True Chromosome 35.0 32068771 138 992961 11 58.0 Full annotation GCA_000002845.2 https://genome.ucsc.edu/h/GCF_000002845.2 GCA_000002845.2 GCF_000002845.2 https://hgdownload.soe.ucsc.edu/hubs/GCF/000/002/845/GCF_000002845.2/genes/GCF_000002845.2_ASM284v2.ncbiRefSeq.gtf.gz -Trypanosoma cruzi 5693 GCF_000209065.1 True Scaffold 1.0 89937456 29495 88624 212 51.5 Full annotation GCA_000209065.1 https://genome.ucsc.edu/h/GCF_000209065.1 GCA_000209065.1 GCF_000209065.1 https://hgdownload.soe.ucsc.edu/hubs/GCF/000/209/065/GCF_000209065.1/genes/GCF_000209065.1_ASM20906v1.ncbiRefSeq.gtf.gz -Coccidioides immitis RS 246410 GCF_000149335.2 True Scaffold 1.0 28947925 6 4323945 3 46.0 Full annotation GCA_000149335.2 https://genome.ucsc.edu/h/GCF_000149335.2 GCA_000149335.2 GCF_000149335.2 https://hgdownload.soe.ucsc.edu/hubs/GCF/000/149/335/GCF_000149335.2/genes/GCF_000149335.2_ASM14933v2.ncbiRefSeq.gtf.gz -Mycobacterium tuberculosis H37Rv 83332 GCF_000277735.2 False Complete Genome 1.0 4411709 1 4411709 1 65.5 GCA_000277735.2 https://genome.ucsc.edu/h/GCF_000857045.1 GCA_000857045.1 GCF_000857045.1 -Severe acute respiratory syndrome coronavirus 2 2697049 GCF_009858895.2 False Complete Genome 1.0 29903 1 29903 1 20.0x 38.0 GCA_009858895.3 https://genome.ucsc.edu/h/GCF_009858895.2 GCA_009858895.3 GCF_009858895.2 -Monkeypox virus 10244 GCF_000857045.1 False Complete Genome 1.0 196858 1 196858 1 100.0x 33.0 GCA_000857045.1 https://genome.ucsc.edu/h/GCF_000857045.1 GCA_000857045.1 GCF_000857045.1 +Leishmania braziliensis MHOM/BR/75/M2904 420245 GCF_000002845.2 True Chromosome 35.0 32068771 138 992961 11 58.0 GCA_000002845.2 https://genome.ucsc.edu/h/GCF_000002845.2 GCA_000002845.2 GCF_000002845.2 https://hgdownload.soe.ucsc.edu/hubs/GCF/000/002/845/GCF_000002845.2/genes/GCF_000002845.2_ASM284v2.ncbiRefSeq.gtf.gz +Trypanosoma cruzi 5693 GCF_000209065.1 True Scaffold 89937456 29495 88624 212 51.5 Full annotation GCA_000209065.1 https://genome.ucsc.edu/h/GCF_000209065.1 GCA_000209065.1 GCF_000209065.1 https://hgdownload.soe.ucsc.edu/hubs/GCF/000/209/065/GCF_000209065.1/genes/GCF_000209065.1_ASM20906v1.ncbiRefSeq.gtf.gz +Coccidioides immitis RS 246410 GCF_000149335.2 True Scaffold 28947925 6 4323945 3 46.0 Full annotation GCA_000149335.2 https://genome.ucsc.edu/h/GCF_000149335.2 GCA_000149335.2 GCF_000149335.2 https://hgdownload.soe.ucsc.edu/hubs/GCF/000/149/335/GCF_000149335.2/genes/GCF_000149335.2_ASM14933v2.ncbiRefSeq.gtf.gz +Mycobacterium tuberculosis H37Rv 83332 GCF_000277735.2 False Complete Genome 1.0 4411709 1 4411709 1 65.5 GCA_000277735.2 +Severe acute respiratory syndrome coronavirus 2 2697049 GCF_009858895.2 False Complete Genome 1.0 29903 1 29903 1 38.0 GCA_009858895.3 https://genome.ucsc.edu/h/GCF_009858895.2 GCA_009858895.3 GCF_009858895.2 +Monkeypox virus 10244 GCF_000857045.1 False Complete Genome 1.0 196858 1 196858 1 33.0 GCA_000857045.1 https://genome.ucsc.edu/h/GCF_000857045.1 GCA_000857045.1 GCF_000857045.1 Mycobacterium tuberculosis 1773 GCF_030566675.1 False Complete Genome 1.0 4516435 1 4516435 1 20.0x 65.5 GCA_030566675.1 Mycobacterium tuberculosis 1773 GCF_963525475.1 False Complete Genome 1.0 4469156 1 4469156 1 100.0x 65.5 GCA_963525475.1