Bb extended index (#1705)

"search" refactor (i.e. search by gene name)
* Add support for bigbed extra indeces and trix
* Move feature search map from "genome" to track to conform to BB model, and simplify management when genome is changed

dev/annotation/searchableGFF.html

@@ -12,7 +12,7 @@
 <h1>Searchable GFF</h1>
 <p>
     Illustrates searching for a feature in a GFF track from a column 9 field.  Search for
-<pre>MYC+proto-oncogene,+bHLH+transcription+factor,+transcript+variant+1</pre>
+<pre>ENST00000524013</pre>
 </p>
 <p>
     Track configuration:
@@ -23,7 +23,7 @@ <h1>Searchable GFF</h1>
       format: "gff",
       indexed: false,
       searchable: true,
-      searchableFields: ["product"]
+      searchableFields: ["transcript_id"]
     }
 </pre>
 </p>
@@ -40,12 +40,12 @@ <h1>Searchable GFF</h1>
             tracks:
                 [
                     {
-                        name: "NCBI gff",
-                        url: "../../test/data/gff/NCBI_hg38_MYC.gff",
-                        format: "gff",
-                        indexed: false,
-                        searchable: true,
-                        searchableFields: ["product"]
+                        "name": "MYC-205",
+                        "url": "../../test/data/gff/Ensembl_MYC-205.gff3",
+                        "format": "gff3",
+                        "indexed": false,
+                        "searchable": true,
+                        searchableFields: ["transcript_id"]
                     }
                 ]
         };

dev/ucsc/hub.html

@@ -15,6 +15,7 @@
     </option>
     <option value="https://hgdownload.soe.ucsc.edu/hubs/GCF/013/103/735/GCF_013103735.1/hub.txt">24 large chromosomes
     </option>
+    <option value="https://hgdownload.soe.ucsc.edu/hubs/GCA/009/914/755/GCA_009914755.4/hub.txt">T2T</option>
 </select>
 
 <label>
@@ -34,7 +35,7 @@
 
 
     const hubOptions = {
-        url: "https://hgdownload.soe.ucsc.edu/hubs/GCA/022/747/635/GCA_022747635.1/hub.txt",
+        url: "https://hgdownload.soe.ucsc.edu/hubs/GCA/009/914/755/GCA_009914755.4/hub.txt",
         includeTracks: "all"
     }
 

js/bigwig/bpTree.js

@@ -0,0 +1,93 @@
+import {igvxhr} from "../../node_modules/igv-utils/src/index.js"
+import {buildOptions} from "../util/igvUtils.js"
+import BinaryParser from "../binary.js"
+
+/**
+ * A UCSC BigBed B+ tree, used to support searching the "extra indexes".
+ *
+ * Nodes are loaded on demand during search, avoiding the need to read the entire tree into
+ * memory.  Tree nodes can be scattered across the file, making loading the entire tree unfeasible in reasonable time.
+ */
+export default class BPTree {
+
+    static async loadBpTree(path, startOffset) {
+        const bpTree = new BPTree(path, startOffset)
+        return bpTree.init()
+    }
+
+    constructor(path, startOffset) {
+        this.path = path
+        this.startOffset = startOffset
+    }
+
+    async init() {
+        const binaryParser = await this.#getParserFor(this.startOffset, 32)
+        const magic = binaryParser.getInt()
+        const blockSize = binaryParser.getInt()
+        const keySize = binaryParser.getInt()
+        const valSize = binaryParser.getInt()
+        const itemCount = binaryParser.getLong()
+        const reserved = binaryParser.getLong()
+        const nodeOffset = this.startOffset + 32
+        this.header = {magic, blockSize, keySize, valSize, itemCount, reserved, nodeOffset}
+        return this
+    }
+
+    async search(term) {
+
+        const {keySize, valSize} = this.header
+
+        if (valSize !== 16) {
+            throw Error(`Unexpected valSize: ${valSize}`)
+        }
+
+        const readTreeNode = async (offset) => {
+
+            let binaryParser = await this.#getParserFor(offset, 4)
+            const type = binaryParser.getByte()
+            const reserved = binaryParser.getByte()
+            const count = binaryParser.getUShort()
+
+            if (type === 1) {
+                // Leaf node
+                const size = count * (keySize + valSize)
+                binaryParser = await this.#getParserFor(offset + 4, size)
+                for (let i = 0; i < count; i++) {
+                    const key = binaryParser.getFixedLengthString(keySize)
+                    const offset = binaryParser.getLong()
+                    const length = binaryParser.getInt()
+                    const reserved = binaryParser.getInt()
+                    const value = {offset, length}
+                    if (term === key) return value
+                }
+            } else {
+                // Non leaf node
+                const size = count * (keySize + 8)
+                binaryParser = await this.#getParserFor(offset + 4, size)
+
+                // Read and discard the first key.
+                const firstKey = binaryParser.getFixedLengthString(keySize)
+                let childOffset = binaryParser.getLong()
+
+                for (let i = 1; i < count; i++) {
+                    const key = binaryParser.getFixedLengthString(keySize)
+                    if (term.localeCompare(key) < 0) {
+                        break
+                    }
+                    childOffset = binaryParser.getLong()
+                }
+
+                return readTreeNode(childOffset)
+            }
+        }
+
+        // Kick things off
+        return readTreeNode(this.header.nodeOffset)
+    }
+
+    async #getParserFor(start, size) {
+        const data = await igvxhr.loadArrayBuffer(this.path, {range: {start, size}})
+        return new BinaryParser(new DataView(data))
+    }
+
+}

js/bigwig/bufferedReader.js

@@ -1,36 +1,12 @@
-/*
- * The MIT License (MIT)
- *
- * Copyright (c) 2014 Broad Institute
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
 
 import {igvxhr} from "../../node_modules/igv-utils/src/index.js"
 import {buildOptions} from "../util/igvUtils.js"
 
 class BufferedReader {
 
-    constructor(config, contentLength, bufferSize) {
+    constructor(config, bufferSize = 512000) {
         this.path = config.url
-        this.bufferSize = bufferSize ? bufferSize : 512000
+        this.bufferSize = bufferSize
         this.range = {start: -1, size: -1}
         this.config = config
     }