From 998cd562c161160f1ac969adfffe56da4fda6815 Mon Sep 17 00:00:00 2001
From: Pablo Moreno <pcm32@users.noreply.github.com>
Date: Mon, 16 Jan 2023 15:05:43 +0000
Subject: [PATCH 01/10] Update cellxgene interactive tool to 1.1.1

Call is the same on 1.1.1, at least as it is used here. Will try to add some options here as well (to specify gene symbols field and others).
---
 tools/interactive/interactivetool_cellxgene.xml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tools/interactive/interactivetool_cellxgene.xml b/tools/interactive/interactivetool_cellxgene.xml
index e9069afcb0af..06f39d7110d1 100644
--- a/tools/interactive/interactivetool_cellxgene.xml
+++ b/tools/interactive/interactivetool_cellxgene.xml
@@ -1,6 +1,6 @@
-<tool id="interactive_tool_cellxgene" tool_type="interactive" name="Interactive CellXgene Environment" version="0.16.2">
+<tool id="interactive_tool_cellxgene" tool_type="interactive" name="Interactive CellXgene Environment" version="1.1.1">
     <requirements>
-        <container type="docker">quay.io/biocontainers/cellxgene:0.16.2--py_0</container>
+        <container type="docker">quay.io/biocontainers/cellxgene:1.1.1--pyhdfd78af_0</container>
     </requirements>
     <entry_points>
         <entry_point name="Cellxgene Single Cell Visualisation on $infile.display_name" requires_domain="True">

From b860465f9c248171a87f742c04332dd6a3754534 Mon Sep 17 00:00:00 2001
From: Pablo Moreno <pcm32@users.noreply.github.com>
Date: Mon, 16 Jan 2023 17:28:43 +0000
Subject: [PATCH 02/10] Handling of files not indexed by gene symbols and
 output recovery

---
 .../interactive/interactivetool_cellxgene.xml | 51 +++++++++++++++++--
 1 file changed, 46 insertions(+), 5 deletions(-)

diff --git a/tools/interactive/interactivetool_cellxgene.xml b/tools/interactive/interactivetool_cellxgene.xml
index 06f39d7110d1..584423b59251 100644
--- a/tools/interactive/interactivetool_cellxgene.xml
+++ b/tools/interactive/interactivetool_cellxgene.xml
@@ -1,6 +1,9 @@
+<!-- <tool id="interactive_tool_cellxgene" tool_type="interactive" name="Interactive CellXgene Environment" version="0.16.2">
+-->
 <tool id="interactive_tool_cellxgene" tool_type="interactive" name="Interactive CellXgene Environment" version="1.1.1">
     <requirements>
-        <container type="docker">quay.io/biocontainers/cellxgene:1.1.1--pyhdfd78af_0</container>
+	<!-- <container type="docker">quay.io/biocontainers/cellxgene:0.16.2py_0</container> -->
+	<container type="docker">quay.io/biocontainers/cellxgene:1.1.1--pyhdfd78af_0</container>
     </requirements>
     <entry_points>
         <entry_point name="Cellxgene Single Cell Visualisation on $infile.display_name" requires_domain="True">
@@ -10,16 +13,54 @@
     <command><![CDATA[
         #import re
         #set $fancy_name = '/tmp/galaxy_cellxgene_' + re.sub('[^\w\-_]', '_', $infile.element_identifier) + '.h5ad'
+	
+	#if $var_name
+	python '${anndata_unique_var_gene_symbols}' '${infile}' '${fancy_name}' '${var_name}'
+        #else  
         cp '${infile}' '${fancy_name}'
-        &&
-        cellxgene launch --host 0.0.0.0 --port 80 '${fancy_name}'
+        #end if
+	&&
+	mkdir annotation_files
+	&&
+	cellxgene launch --host 0.0.0.0 --port 80 
+        --user-generated-data-dir annotation_files
+	'${fancy_name}'
     ]]>
     </command>
+    <configfiles>
+	    <configfile name="anndata_unique_var_gene_symbols"><![CDATA[
+import anndata as ad
+import sys
+
+def rn(df, field, suffix = '-duplicate-'):
+    appendents = (suffix + df.groupby(field).cumcount().astype(str).replace('0','')).replace(suffix, '')
+    df[f"{field}_u"] = df[field].astype(str) + appendents.astype(str)
+    return df
+
+adata = ad.read_h5ad(sys.argv[1])
+output = sys.argv[2]
+gene_symbol_field = sys.argv[3]
+if gene_symbol_field not in adata.var.keys():
+   sys.exit(f"Field {gene_symbol_field} set as var_name does not exist in the var object. AnnData object will be used as it was given")
+
+adata.var = rn(adata.var, gene_symbol_field, suffix = "_d")
+
+adata.var["extra_gene_id"] = adata.var.index
+adata.var = adata.var.set_index(f"{gene_symbol_field}_u")
+
+
+adata.write_h5ad(output, compression="gzip")
+]]></configfile>
+    </configfiles>
     <inputs>
-        <param name="infile" type="data" format="h5ad" label="Concatenate Dataset"/>
+	<param name="infile" type="data" format="h5ad" label="Concatenate Dataset"/>
+        <param name="var_name" type="text" optional="true" label="Var field for gene symbols" help="Optionaly specify the var field from the AnnData file provided where gene symbols are available. Usually the AnnData file var slot will be indexed with gene symbols and this is not required. Using this option will delay the startup of the cellxgene tool, as the AnnData needs to be modified and re-written to disk."/>
     </inputs>
     <outputs>
-        <data name="out_file1" format="txt" />
+	<data name="out_file1" format="txt" />
+	<collection name="user_generated" type="list" label="Cellxgene user annotations and gene sets">
+            <discover_datasets pattern="__name_and_ext__" directory="annotation_files" />
+        </collection>
     </outputs>
     <tests>
     </tests>

From acd862c6329438d0281016b4e9907fce6f390d36 Mon Sep 17 00:00:00 2001
From: Pablo Moreno <pcm32@users.noreply.github.com>
Date: Mon, 16 Jan 2023 17:29:22 +0000
Subject: [PATCH 03/10] Clean up

---
 tools/interactive/interactivetool_cellxgene.xml | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/tools/interactive/interactivetool_cellxgene.xml b/tools/interactive/interactivetool_cellxgene.xml
index 584423b59251..e1983198ba33 100644
--- a/tools/interactive/interactivetool_cellxgene.xml
+++ b/tools/interactive/interactivetool_cellxgene.xml
@@ -1,8 +1,5 @@
-<!-- <tool id="interactive_tool_cellxgene" tool_type="interactive" name="Interactive CellXgene Environment" version="0.16.2">
--->
 <tool id="interactive_tool_cellxgene" tool_type="interactive" name="Interactive CellXgene Environment" version="1.1.1">
     <requirements>
-	<!-- <container type="docker">quay.io/biocontainers/cellxgene:0.16.2py_0</container> -->
 	<container type="docker">quay.io/biocontainers/cellxgene:1.1.1--pyhdfd78af_0</container>
     </requirements>
     <entry_points>

From acc34f6d2110d09858f5ff843f4c8d623a0617d9 Mon Sep 17 00:00:00 2001
From: Pablo Moreno <pcm32@users.noreply.github.com>
Date: Tue, 7 Mar 2023 08:59:43 +0000
Subject: [PATCH 04/10] Adds help

---
 .../interactive/interactivetool_cellxgene.xml | 25 ++++++++++++++++---
 1 file changed, 22 insertions(+), 3 deletions(-)

diff --git a/tools/interactive/interactivetool_cellxgene.xml b/tools/interactive/interactivetool_cellxgene.xml
index e1983198ba33..590ea03374d9 100644
--- a/tools/interactive/interactivetool_cellxgene.xml
+++ b/tools/interactive/interactivetool_cellxgene.xml
@@ -61,7 +61,26 @@ adata.write_h5ad(output, compression="gzip")
     </outputs>
     <tests>
     </tests>
-    <help>
-    Interactive tool for visualising AnnData.
-    </help>
+<help><![CDATA[
+cellxgene - "Cell by gene" interactive viewer for single cell data in AnnData
+=============================================================================
+Purpose
+-------
+Enables interactive visualisation of Single Cell RNA-Seq datasets stored as 
+AnnData files (the format used by Scanpy).
+
+Selecting the Var name
+----------------------
+It can happen that the main index for the var element of AnnData is not the 
+gene symbol field, in which case search by genes will probably be by identifier.
+You can choose a different field (which contains the gene symbols) so that
+the AnnData is modified (in a new copy) so that that field is made unique and
+it is indexed by it. Then cellxgene will allow searches by genes in that field.
+
+Outputs
+-------
+
+If you create gene sets or differential experession sets, this will be available as a collection of files under
+"Cellxgene user annotations and gene sets" at the end of the execution (when you stop the interactive environment).
+]]></help>
 </tool>

From e9e8f9be27b02815fc3ddbbe6211aa58593c1488 Mon Sep 17 00:00:00 2001
From: Pablo Moreno <pcm32@users.noreply.github.com>
Date: Tue, 7 Mar 2023 12:03:34 +0000
Subject: [PATCH 05/10] Change name and tutorial link

---
 ...ool_cellxgene.xml => interactivetool_cellxgene_1.1.1.xml} | 5 +++++
 1 file changed, 5 insertions(+)
 rename tools/interactive/{interactivetool_cellxgene.xml => interactivetool_cellxgene_1.1.1.xml} (95%)

diff --git a/tools/interactive/interactivetool_cellxgene.xml b/tools/interactive/interactivetool_cellxgene_1.1.1.xml
similarity index 95%
rename from tools/interactive/interactivetool_cellxgene.xml
rename to tools/interactive/interactivetool_cellxgene_1.1.1.xml
index 590ea03374d9..2fec1b7865ac 100644
--- a/tools/interactive/interactivetool_cellxgene.xml
+++ b/tools/interactive/interactivetool_cellxgene_1.1.1.xml
@@ -82,5 +82,10 @@ Outputs
 
 If you create gene sets or differential experession sets, this will be available as a collection of files under
 "Cellxgene user annotations and gene sets" at the end of the execution (when you stop the interactive environment).
+
+Tutorials
+---------
+
+You can find cellxgene tuorials `here <https://cellxgene.cziscience.com/docs/04__Analyze%20Public%20Data/4_1__Hosted%20Tutorials>`_ .
 ]]></help>
 </tool>

From 999322b35438b1dd5d6d6a70e3d9b6f43b2cd1d4 Mon Sep 17 00:00:00 2001
From: Pablo Moreno <pcm32@users.noreply.github.com>
Date: Tue, 7 Mar 2023 12:05:10 +0000
Subject: [PATCH 06/10] Adds previous version as separate file

---
 .../interactivetool_cellxgene_0.16.2.xml      | 29 +++++++++++++++++++
 1 file changed, 29 insertions(+)
 create mode 100644 tools/interactive/interactivetool_cellxgene_0.16.2.xml

diff --git a/tools/interactive/interactivetool_cellxgene_0.16.2.xml b/tools/interactive/interactivetool_cellxgene_0.16.2.xml
new file mode 100644
index 000000000000..e9069afcb0af
--- /dev/null
+++ b/tools/interactive/interactivetool_cellxgene_0.16.2.xml
@@ -0,0 +1,29 @@
+<tool id="interactive_tool_cellxgene" tool_type="interactive" name="Interactive CellXgene Environment" version="0.16.2">
+    <requirements>
+        <container type="docker">quay.io/biocontainers/cellxgene:0.16.2--py_0</container>
+    </requirements>
+    <entry_points>
+        <entry_point name="Cellxgene Single Cell Visualisation on $infile.display_name" requires_domain="True">
+            <port>80</port>
+        </entry_point>
+    </entry_points>
+    <command><![CDATA[
+        #import re
+        #set $fancy_name = '/tmp/galaxy_cellxgene_' + re.sub('[^\w\-_]', '_', $infile.element_identifier) + '.h5ad'
+        cp '${infile}' '${fancy_name}'
+        &&
+        cellxgene launch --host 0.0.0.0 --port 80 '${fancy_name}'
+    ]]>
+    </command>
+    <inputs>
+        <param name="infile" type="data" format="h5ad" label="Concatenate Dataset"/>
+    </inputs>
+    <outputs>
+        <data name="out_file1" format="txt" />
+    </outputs>
+    <tests>
+    </tests>
+    <help>
+    Interactive tool for visualising AnnData.
+    </help>
+</tool>

From 6fe3db1db7abd4a2a1b921755f9fa7b1434405b1 Mon Sep 17 00:00:00 2001
From: Pablo Moreno <pcm32@users.noreply.github.com>
Date: Tue, 7 Mar 2023 12:06:55 +0000
Subject: [PATCH 07/10] Update tool_conf.xml.sample

---
 lib/galaxy/config/sample/tool_conf.xml.sample | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/galaxy/config/sample/tool_conf.xml.sample b/lib/galaxy/config/sample/tool_conf.xml.sample
index b713c208d1a6..401d10a838a6 100644
--- a/lib/galaxy/config/sample/tool_conf.xml.sample
+++ b/lib/galaxy/config/sample/tool_conf.xml.sample
@@ -133,7 +133,8 @@
     <tool file="interactive/codingSnps.xml" />
     <tool file="interactive/interactivetool_askomics.xml" />
     <tool file="interactive/interactivetool_bam_iobio.xml" />
-    <tool file="interactive/interactivetool_cellxgene.xml" />
+    <tool file="interactive/interactivetool_cellxgene_0.16.2.xml" />
+    <tool file="interactive/interactivetool_cellxgene_1.1.1.xml" />
     <tool file="interactive/interactivetool_ethercalc.xml" />
     <tool file="interactive/interactivetool_hicbrowser.xml" />
     <tool file="interactive/interactivetool_jupyter_notebook_1.0.0.xml" />

From 703751b65c7b3a8c2656dea896a5ea3b4f69b8a7 Mon Sep 17 00:00:00 2001
From: Pablo Moreno <pcm32@users.noreply.github.com>
Date: Wed, 8 Mar 2023 13:17:11 +0000
Subject: [PATCH 08/10] Avoid errors automatically detected by galaxy due to
 stderr output

---
 tools/interactive/interactivetool_cellxgene_1.1.1.xml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/tools/interactive/interactivetool_cellxgene_1.1.1.xml b/tools/interactive/interactivetool_cellxgene_1.1.1.xml
index 2fec1b7865ac..08edd9273b4d 100644
--- a/tools/interactive/interactivetool_cellxgene_1.1.1.xml
+++ b/tools/interactive/interactivetool_cellxgene_1.1.1.xml
@@ -7,6 +7,13 @@
             <port>80</port>
         </entry_point>
     </entry_points>
+    <stdio>
+		<regex match="WARNING"
+			   source="both"
+			   level="warning"
+			   description="cellxgene warnings"
+			   />
+    </stdio>
     <command><![CDATA[
         #import re
         #set $fancy_name = '/tmp/galaxy_cellxgene_' + re.sub('[^\w\-_]', '_', $infile.element_identifier) + '.h5ad'

From 0b204731296c794704cb135b85ba2be86274688c Mon Sep 17 00:00:00 2001
From: Pablo Moreno <pablo.moreno@astrazeneca.com>
Date: Mon, 27 Mar 2023 15:06:15 +0100
Subject: [PATCH 09/10] Layer and optionally make var field unique.

---
 .../interactivetool_cellxgene_1.1.1.xml       | 60 +++++++++++++------
 1 file changed, 42 insertions(+), 18 deletions(-)

diff --git a/tools/interactive/interactivetool_cellxgene_1.1.1.xml b/tools/interactive/interactivetool_cellxgene_1.1.1.xml
index 08edd9273b4d..14bbcfc97bd3 100644
--- a/tools/interactive/interactivetool_cellxgene_1.1.1.xml
+++ b/tools/interactive/interactivetool_cellxgene_1.1.1.xml
@@ -15,19 +15,22 @@
 			   />
     </stdio>
     <command><![CDATA[
-        #import re
-        #set $fancy_name = '/tmp/galaxy_cellxgene_' + re.sub('[^\w\-_]', '_', $infile.element_identifier) + '.h5ad'
+    #import re
+    #set $fancy_name = '/tmp/galaxy_cellxgene_' + re.sub('[^\w\-_]', '_', $infile.element_identifier) + '.h5ad'
 	
-	#if $var_name
-	python '${anndata_unique_var_gene_symbols}' '${infile}' '${fancy_name}' '${var_name}'
-        #else  
-        cp '${infile}' '${fancy_name}'
-        #end if
+    #if ($var_name and $make_unique) or $layer:
+    python '${anndata_unique_var_gene_symbols}' '${infile}' '${fancy_name}' '${var_name}' '${make_unique}' '${layer}'
+    #else  
+    cp '${infile}' '${fancy_name}'
+    #end if
 	&&
 	mkdir annotation_files
 	&&
-	cellxgene launch --host 0.0.0.0 --port 80 
+    cellxgene launch --host 0.0.0.0 --port 80 
         --user-generated-data-dir annotation_files
+    #if $var_name and not $make_unique:
+        --var-names $var_name
+    #end if
 	'${fancy_name}'
     ]]>
     </command>
@@ -44,21 +47,31 @@ def rn(df, field, suffix = '-duplicate-'):
 adata = ad.read_h5ad(sys.argv[1])
 output = sys.argv[2]
 gene_symbol_field = sys.argv[3]
-if gene_symbol_field not in adata.var.keys():
-   sys.exit(f"Field {gene_symbol_field} set as var_name does not exist in the var object. AnnData object will be used as it was given")
+make_unique = (sys.argv[4].lower() == "true")
+layer = sys.argv[5]
 
-adata.var = rn(adata.var, gene_symbol_field, suffix = "_d")
+if gene_symbol_field and make_unique:
+    if gene_symbol_field not in adata.var.keys():
+        sys.exit(f"Field {gene_symbol_field} set as var_name does not exist in the var object. AnnData object will be used as it was given")
+    
+    adata.var = rn(adata.var, gene_symbol_field, suffix = "_d")
+    adata.var["extra_gene_id"] = adata.var.index
+    adata.var = adata.var.set_index(f"{gene_symbol_field}_u")
 
-adata.var["extra_gene_id"] = adata.var.index
-adata.var = adata.var.set_index(f"{gene_symbol_field}_u")
+if layer:
+    if layer not in adata.layers.keys():
+        sys.exit(f"Layer {layer} is not present in AnnData, only available layers are: {', '.join(adata.layers.keys())}")
+    else:
+        adata.X = adata.layers[layer]
 
-
-adata.write_h5ad(output, compression="gzip")
+adata.write_h5ad(output)
 ]]></configfile>
     </configfiles>
     <inputs>
 	<param name="infile" type="data" format="h5ad" label="Concatenate Dataset"/>
-        <param name="var_name" type="text" optional="true" label="Var field for gene symbols" help="Optionaly specify the var field from the AnnData file provided where gene symbols are available. Usually the AnnData file var slot will be indexed with gene symbols and this is not required. Using this option will delay the startup of the cellxgene tool, as the AnnData needs to be modified and re-written to disk."/>
+    <param name="var_name" type="text" optional="true" label="Var field for gene symbols" help="Optionaly specify the var field from the AnnData file provided where gene symbols are available. Usually the AnnData file var slot will be indexed with gene symbols and this is not required. Using this option will delay the startup of the cellxgene tool, as the AnnData needs to be modified and re-written to disk."/>
+    <param name="make_unique" type="boolean" checked="false" label="Make specified var field unique" help="It will copy the specified var field above and make its values unique. This means reading the AnnData object into Python and re-writing it again, increasing waiting time for the container to be up."/>
+    <param name="layer" type="text" optional="true" label="Specifies the AnnData layer to use as matrix" help="The layer name needs to be present in the AnnData file or this tool will fail"/>
     </inputs>
     <outputs>
 	<data name="out_file1" format="txt" />
@@ -80,9 +93,20 @@ Selecting the Var name
 ----------------------
 It can happen that the main index for the var element of AnnData is not the 
 gene symbol field, in which case search by genes will probably be by identifier.
-You can choose a different field (which contains the gene symbols) so that
+You can choose a different field and celxgene will use this. If in addition you choose "Make unique", 
 the AnnData is modified (in a new copy) so that that field is made unique and
-it is indexed by it. Then cellxgene will allow searches by genes in that field.
+it is indexed by it. Making it unique entails though loading the object into memory, 
+modifying and writing it back, which can delay the execution of cellxgene.
+
+Then cellxgene will allow searches by genes in that field.
+
+Selecting the layer
+-------------------
+
+It can happen that the AnnData object contains multiple layers, for example
+one with the raw counts and another with the normalised counts. You can select
+which layer to use as the matrix for cellxgene. By default cellxgene will use the X slot, but
+that slot might not contain the matrix that you want to visualise.
 
 Outputs
 -------

From 1741fa45ee1840603e06ede10367614e0f6f9a13 Mon Sep 17 00:00:00 2001
From: Pablo Moreno <pablo.moreno@astrazeneca.com>
Date: Tue, 28 Mar 2023 10:19:54 +0100
Subject: [PATCH 10/10] Some tabs to spaces

---
 tools/interactive/interactivetool_cellxgene_1.1.1.xml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/tools/interactive/interactivetool_cellxgene_1.1.1.xml b/tools/interactive/interactivetool_cellxgene_1.1.1.xml
index 14bbcfc97bd3..c8c6c4cb72bc 100644
--- a/tools/interactive/interactivetool_cellxgene_1.1.1.xml
+++ b/tools/interactive/interactivetool_cellxgene_1.1.1.xml
@@ -17,21 +17,21 @@
     <command><![CDATA[
     #import re
     #set $fancy_name = '/tmp/galaxy_cellxgene_' + re.sub('[^\w\-_]', '_', $infile.element_identifier) + '.h5ad'
-	
+
     #if ($var_name and $make_unique) or $layer:
     python '${anndata_unique_var_gene_symbols}' '${infile}' '${fancy_name}' '${var_name}' '${make_unique}' '${layer}'
     #else  
     cp '${infile}' '${fancy_name}'
     #end if
-	&&
-	mkdir annotation_files
-	&&
+    &&
+    mkdir annotation_files
+    &&
     cellxgene launch --host 0.0.0.0 --port 80 
         --user-generated-data-dir annotation_files
     #if $var_name and not $make_unique:
         --var-names $var_name
     #end if
-	'${fancy_name}'
+    '${fancy_name}'
     ]]>
     </command>
     <configfiles>