DEV: making the develop branch the default

.github/workflows/codeql.yml

@@ -0,0 +1,74 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ "master" ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ "master" ]
+  schedule:
+    - cron: '39 20 * * 6'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'python' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
+        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v2
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+
+        # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
+
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, Go, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v2
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+
+    #   If the Autobuild fails above, remove it and uncomment the following three lines. 
+    #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
+
+    # - run: |
+    #   echo "Run, Build Application using script"
+    #   ./location_of_script_within_repo/buildscript.sh
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v2
+      with:
+        category: "/language:${{matrix.language}}"

.github/workflows/testing_develop.yml

@@ -0,0 +1,57 @@
+name: CI
+
+on:
+  push:
+    branches: [ "master"]
+  pull_request:
+    branches: [ "master"]
+
+jobs:
+  tests:
+    name: "Python ${{ matrix.python-version }}"
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        python-version: ["3.10", "3.11"]
+
+    steps:
+      - uses: "actions/checkout@v2"
+        with:
+          fetch-depth: 0
+
+      # Setup env
+      - uses: "actions/setup-python@v2"
+        with:
+            python-version: "${{ matrix.python-version }}"
+
+      - name: "Installs for ${{ matrix.python-version }}"
+        run: |
+          python --version
+          pip install --upgrade pip wheel setuptools nox
+
+      - name: "Run nox for ${{ matrix.python-version }}"
+        run: "nox -s test-${{ matrix.python-version }}"
+
+      - name: Coveralls Parallel
+        uses: coverallsapp/github-action@master
+        with:
+          parallel: true
+          github-token: ${{ secrets.github_token }}
+          flag-name: run-${{ matrix.test_number }}
+          path-to-lcov: "tests/lcov-${{ matrix.python-version }}.info"
+
+  finish:
+    needs: tests
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [macos-latest, ubuntu-latest, windows-latest]
+        python-version: ["3.10", "3.11"]
+    steps:
+    - name: Coveralls Finished
+      uses: coverallsapp/github-action@master
+      with:
+        github-token: ${{ secrets.github_token }}
+        parallel-finished: true

.sourcery.yaml

@@ -0,0 +1,2 @@
+refactor:
+  python_version: '3.8'

README.md

@@ -1,6 +1,106 @@
+[![CI](https://github.com/cogent3/ensembl_cli/actions/workflows/testing_develop.yml/badge.svg)](https://github.com/cogent3/ensembl_cli/actions/workflows/testing_develop.yml)
+[![CodeQL](https://github.com/cogent3/ensembl_cli/actions/workflows/codeql.yml/badge.svg)](https://github.com/cogent3/ensembl_cli/actions/workflows/codeql.yml)
+[![Coverage Status](https://coveralls.io/repos/github/cogent3/ensembl_cli/badge.svg?branch=master)](https://coveralls.io/github/cogent3/ensembl_cli?branch=master)
+
 # ensembl_cli
 
+## Installation
+
+Suggest creating a conda environment or a python virtual environment, using python3.11. Then install directly into that environment from the GitHub repo as
+
+```
+$ python -m pip install "ensembl_cli  @ git+https://github.com/cogent3/ensembl_cli.git@develop"
+```
+
+Then run for the first time using
+
+```
+$ ensembl_cli tui
+```
+
+The first start takes a while as, behind the scenes, cogent3 is transpiling various functions into C and compiling them. Eventually, you get a very neat terminal interface you can click around in. To exit, make sure the "root" is selected on the left panel then `^+r`.
+
+## Usage
+
+The setup is (for now) controlled using a config file, defined in `ini` format. To get a starting template use the `exportrc` subcommand.
+
+<!-- [[[cog
+import cog
+from ensembl_cli import cli
+from click.testing import CliRunner
+runner = CliRunner()
+result = runner.invoke(cli.main, ["exportrc", "--help"])
+help = result.output.replace("Usage: main", "Usage: ensembl_cli")
+cog.out(
+    "```\n{}\n```".format(help)
+)
+]]] -->
+```
+Usage: ensembl_cli exportrc [OPTIONS]
+
+  exports sample config and species table to the nominated path
+
+  setting an environment variable ENSEMBLDBRC with this path will force its
+  contents to override the default ensembl_cli settings
+
+Options:
+  -o, --outpath PATH  path to directory to export all rc contents
+  --help              Show this message and exit.
+
+```
+<!-- [[[end]]] -->
+
+<details>
+    <summary> Click to see a sample config file I've been using for development </summary>
+
+Using this config, it takes approximately 16' to download (over a ~200MB/s WiFi connection) and ~45' to install on my M2 Macbook Pro (note the install is incomplete). (Note this step uses up to  10 CPU cores.)
+
+```
+[remote path]
+host=ftp.ensembl.org
+path=pub
+[local path]
+staging_path=~/Desktop/Outbox/ensembl_download
+install_path=~/Desktop/Outbox/ensembl_install
+[release]
+release=110
+[Mouse Lemur]
+db=core
+[Macaque]
+db=core
+[Gibbon]
+db=core
+[Orangutan]
+db=core
+[Bonobo]
+db=core
+[Human]
+db=core
+[Chimp]
+db=core
+[Gorilla]
+db=core
+[compara]
+align_names=10_primates.epo
+```
+</details>
+
+### Download
+
+Downloads the species indicated in the config file:
+
+- genomes sequences as fasta format 
+- annotations as gff3
+- gene homologies for individual genomes in tsv format
+
+Alignments indicated in the config file will be downloaded in `.maf` format.
+
+Downloads are written to a local directory, specified in the config file. Downloads are done in parallel (using threads).
+
+### Install
+
+"Installation" involves transforming downloaded files into local sqlite3 databases which are saved to the location specified in the config file.
 
-## Downloading
+From the maf alignment files, the "ancestral" sequences are discarded and for every aligned sequence only the gap data is stored (i.e. gap position and length) along with the genomic coordinates. These alignments will be reconstructable by combining this information with the whole genome sequence. (This approach reduces storage requirements ~5-fold).
 
-Make the setting in 
+Installation is done in parallel on multiple CPUs (since the data need to be decompressed on the fly).

pyproject.toml

@@ -18,10 +18,10 @@ dependencies = ["click",
         "scitrack",
         "typing_extensions",
         "cogent3 @ git+https://github.com/cogent3/cogent3.git@develop",
-        "pydantic",
         "rich",
         "numba",
         "numpy",
+        "trogon",
         "unsync",
         "wakepy",
         ]
@@ -64,6 +64,7 @@ doc  = ["click==8.1.3",
         "sphinxcontrib-bibtex"]
 dev = ["black==23.3.0",
        "click==8.1.3",
+       "cogapp",
        "flit",
        "ipykernel",
        "ipython",