From a67652af5428cd35e1a77cad04f4530e1911f37b Mon Sep 17 00:00:00 2001 From: jrobinso <933148+jrobinso@users.noreply.github.com> Date: Tue, 3 Dec 2024 22:17:45 -0800 Subject: [PATCH] refine genome "is downloadable" tests --- .../feature/genome/GenomeDownloadUtils.java | 33 ++++++++++++++----- .../igv/feature/genome/load/GenomeConfig.java | 20 +++++++++++ .../feature/genome/load/JsonGenomeLoader.java | 22 ------------- .../HostedGenomeSelectionDialog.java | 4 +-- .../genome/GenomeDownloadUtilsTest.java | 24 ++++++++++++++ test/data/genomes/json/hg38_twobit.json | 21 ++++++++++++ 6 files changed, 91 insertions(+), 33 deletions(-) create mode 100644 src/test/java/org/broad/igv/feature/genome/GenomeDownloadUtilsTest.java create mode 100644 test/data/genomes/json/hg38_twobit.json diff --git a/src/main/java/org/broad/igv/feature/genome/GenomeDownloadUtils.java b/src/main/java/org/broad/igv/feature/genome/GenomeDownloadUtils.java index a18d8139f2..8c20632aef 100644 --- a/src/main/java/org/broad/igv/feature/genome/GenomeDownloadUtils.java +++ b/src/main/java/org/broad/igv/feature/genome/GenomeDownloadUtils.java @@ -20,8 +20,7 @@ import java.lang.reflect.Field; import java.net.MalformedURLException; import java.net.URL; -import java.util.List; -import java.util.Map; +import java.util.*; /** * Class of static functions for managing genome downloads @@ -30,20 +29,27 @@ public class GenomeDownloadUtils { private static Logger log = LogManager.getLogger(GenomeDownloadUtils.class); - public static boolean isAnnotationsDownloadable(GenomeListItem item) { - return item.getPath().endsWith(".json"); + public static boolean isAnnotationsDownloadable(String path) { + return path.endsWith(".json"); } - public static boolean isSequenceDownloadable(GenomeListItem item) { - if (item.getPath().endsWith(".json")) { + public static boolean isSequenceDownloadable(String path) { + if (path != null && path.endsWith(".json")) { try { - String jsonString = HttpUtils.getInstance().getContentsAsJSON(new URL(item.getPath())); - return jsonString.contains("twoBitURL"); + String jsonString = FileUtils.getContents(path); + GenomeConfig genomeConfig = GenomeConfig.fromJson(jsonString); + String sequenceURL = genomeConfig.getTwoBitURL(); + if (sequenceURL == null) { + sequenceURL = genomeConfig.getFastaURL(); + } + return isRemoteURL(sequenceURL) && !disallowedBuckets.stream().anyMatch(sequenceURL::contains); + } catch (IOException e) { - log.error("Error fetching genome json " + item.getPath()); + log.error("Error fetching genome json " + path); } } return false; + } public static File downloadGenome(GenomeConfig c, boolean downloadSequence, boolean downloadAnnotations) throws IOException { @@ -161,4 +167,13 @@ private static File download(URL url, File directory) throws MalformedURLExcepti return localFile; } + private static boolean isRemoteURL(String url) { + return url.startsWith("http://") || url.startsWith("https://"); + } + + static Set disallowedBuckets = new HashSet<>(Arrays.asList( + "igv.org.genomes", + "igv-genepattern-org", + "igv.broadinstitute.org")); + } diff --git a/src/main/java/org/broad/igv/feature/genome/load/GenomeConfig.java b/src/main/java/org/broad/igv/feature/genome/load/GenomeConfig.java index d698d1ffaa..5fd57257ec 100644 --- a/src/main/java/org/broad/igv/feature/genome/load/GenomeConfig.java +++ b/src/main/java/org/broad/igv/feature/genome/load/GenomeConfig.java @@ -63,6 +63,9 @@ public class GenomeConfig implements Cloneable { private List> chromAliases; public static GenomeConfig fromJson(String json) { + if (json.contains("chromosomeOrder")) { + json = fixChromosomeOrder(json); + } return (new Gson()).fromJson(json, GenomeConfig.class); } @@ -307,4 +310,21 @@ protected GenomeConfig clone() { } } + /** + * Fix deprecated form of chromosome order (comma delimited list of strings) + * + * @param jsonString + * @return + */ + private static String fixChromosomeOrder(String jsonString) { + Map obj = (new Gson()).fromJson(jsonString, Map.class); + Object chromosomeOrder = obj.get("chromosomeOrder"); + if (chromosomeOrder != null) { + if (chromosomeOrder instanceof String) { + obj.put("chromosomeOrder", Arrays.stream(((String) chromosomeOrder).split(",")).map(c -> c.trim()).toArray()); + } + } + return (new Gson()).toJson(obj); + } + } diff --git a/src/main/java/org/broad/igv/feature/genome/load/JsonGenomeLoader.java b/src/main/java/org/broad/igv/feature/genome/load/JsonGenomeLoader.java index c0750a7f99..b2361fa489 100644 --- a/src/main/java/org/broad/igv/feature/genome/load/JsonGenomeLoader.java +++ b/src/main/java/org/broad/igv/feature/genome/load/JsonGenomeLoader.java @@ -66,10 +66,6 @@ public GenomeConfig loadGenomeConfig() throws IOException { String jsonString = ParsingUtils.readContentsFromStream(is); - if (jsonString.contains("chromosomeOrder")) { - jsonString = fixChromosomeOrder(jsonString); - } - GenomeConfig genomeConfig = GenomeConfig.fromJson(jsonString); fixPaths(genomeConfig); @@ -79,24 +75,6 @@ public GenomeConfig loadGenomeConfig() throws IOException { } } - /** - * Fix deprecated form of chromosome order (comma delimited list of strings) - * - * @param jsonString - * @return - */ - private String fixChromosomeOrder(String jsonString) { - Map obj = (new Gson()).fromJson(jsonString, Map.class); - Object chromosomeOrder = obj.get("chromosomeOrder"); - if (chromosomeOrder != null) { - if (chromosomeOrder instanceof String) { - obj.put("chromosomeOrder", Arrays.stream(((String) chromosomeOrder).split(",")).map(c -> c.trim()).toArray()); - } - } - return (new Gson()).toJson(obj); - - } - /** * JSON paths/urls can be relative to the path to the genome json file. This method converts them to absolulte * paths/urls. diff --git a/src/main/java/org/broad/igv/ui/commandbar/HostedGenomeSelectionDialog.java b/src/main/java/org/broad/igv/ui/commandbar/HostedGenomeSelectionDialog.java index 7a9b9ccd8b..1f89e9b3d9 100644 --- a/src/main/java/org/broad/igv/ui/commandbar/HostedGenomeSelectionDialog.java +++ b/src/main/java/org/broad/igv/ui/commandbar/HostedGenomeSelectionDialog.java @@ -207,8 +207,8 @@ private void okButtonActionPerformed(java.awt.event.ActionEvent evt) { private void configureDownloadButtons(GenomeListItem item) { if (item != null) { - final boolean sequenceDownloadable = GenomeDownloadUtils.isSequenceDownloadable(item); - final boolean annotationsDownloadable = GenomeDownloadUtils.isAnnotationsDownloadable(item); + final boolean sequenceDownloadable = GenomeDownloadUtils.isSequenceDownloadable(item.getPath()); + final boolean annotationsDownloadable = GenomeDownloadUtils.isAnnotationsDownloadable(item.getPath()); downloadSequenceCB.setEnabled(sequenceDownloadable); downloadAnnotationsCB.setEnabled(annotationsDownloadable); } diff --git a/src/test/java/org/broad/igv/feature/genome/GenomeDownloadUtilsTest.java b/src/test/java/org/broad/igv/feature/genome/GenomeDownloadUtilsTest.java new file mode 100644 index 0000000000..82f721afda --- /dev/null +++ b/src/test/java/org/broad/igv/feature/genome/GenomeDownloadUtilsTest.java @@ -0,0 +1,24 @@ +package org.broad.igv.feature.genome; + +import org.broad.igv.util.TestUtils; +import org.junit.Test; + +import static org.junit.Assert.*; + +public class GenomeDownloadUtilsTest { + + @Test + public void isAnnotationsDownloadable() { + assertTrue(GenomeDownloadUtils.isAnnotationsDownloadable(TestUtils.DATA_DIR + "genomes/json/hg38_twobit.json")); + + } + + @Test + public void isSequenceDownloadable() { + + assertTrue(GenomeDownloadUtils.isSequenceDownloadable(TestUtils.DATA_DIR + "genomes/json/hg38_twobit.json")); + assertFalse(GenomeDownloadUtils.isSequenceDownloadable(TestUtils.DATA_DIR + "genomes/json/hg18.unittest.json")); + assertFalse(GenomeDownloadUtils.isSequenceDownloadable(TestUtils.DATA_DIR + "genomes/json/sacCer3.json")); + + } +} \ No newline at end of file diff --git a/test/data/genomes/json/hg38_twobit.json b/test/data/genomes/json/hg38_twobit.json new file mode 100644 index 0000000000..df9ed956f5 --- /dev/null +++ b/test/data/genomes/json/hg38_twobit.json @@ -0,0 +1,21 @@ +{ + "id": "hg38", + "name": "Human (GRCh38/hg38)", + "fastaURL": "https://igv.org/genomes/data/hg38/hg38.fa", + "indexURL": "https://igv.org/genomes/data/hg38/hg38.fa.fai", + "cytobandURL": "https://hgdownload.soe.ucsc.edu/goldenPath/hg38/database/cytoBandIdeo.txt.gz", + "aliasURL": "https://igv.org/genomes/data/hg38/hg38_alias.tab", + "twoBitURL": "https://hgdownload.soe.ucsc.edu/goldenPath/hg38/bigZips/hg38.2bit", + "chromSizesURL": "https://hgdownload.soe.ucsc.edu/goldenPath/hg38/bigZips/hg38.chrom.sizes", + "chromosomeOrder": "chr1,chr2,chr3,chr4,chr5,chr6,chr7,chr8,chr9,chr10,chr11,chr12,chr13,chr14,chr15,chr16,chr17,chr18,chr19,chr20,chr21,chr22,chrX,chrY", + "tracks": [ + { + "name": "Refseq Select", + "format": "refgene", + "url": "https://hgdownload.soe.ucsc.edu/goldenPath/hg38/database/ncbiRefSeqSelect.txt.gz", + "indexed": false, + "order": 1000001, + "infoURL": "https://www.ncbi.nlm.nih.gov/gene/?term=$$" + } + ] +}