08_Fig5_overview.python.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "aa30c71e-7186-4970-a687-b388897dc57d",
   "metadata": {},
   "source": [
    "## Figure 5. Expanded similarity network with additional connections to different BGC knowledgebases\n",
    "This notebook integrates BiG-SCAPE network with additional information"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7d383b5c-dcb3-471b-8119-4d39056be727",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import networkx as nx\n",
    "import plotly.graph_objects as go\n",
    "import numpy as np\n",
    "import scipy.spatial as spatial\n",
    "\n",
    "from pathlib import Path\n",
    "import json, shutil, yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4bf26bae-bd3d-4d85-883a-8e48e1e400fa",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def create_node_trace(G, node_trace_category, color, showtextlabel=False, nodesize=10, nodeopacity=0.8, \n",
    "                      nodesymbol=\"circle\", linewidth=1, linecolor=\"black\", textposition=\"top center\", showlegend=False):\n",
    "    if showtextlabel:\n",
    "        markermode = \"markers+text\"\n",
    "    else:\n",
    "        markermode = \"markers\"\n",
    "    nodes = np.array([node for node in G.nodes() if G.nodes[node][\"node_trace\"] == node_trace_category])\n",
    "    pos = np.array([G.nodes[node]['pos'] for node in nodes.flatten()]).reshape(-1, 2)\n",
    "    xs, ys = pos[:, 0], pos[:, 1]\n",
    "    texts = np.array([G.nodes[node]['text'] for node in nodes])\n",
    "    node_trace = go.Scatter(\n",
    "        x=xs.tolist(),\n",
    "        y=ys.tolist(),\n",
    "        text=texts.tolist(),\n",
    "        textposition=textposition,\n",
    "        mode=markermode,\n",
    "        hoverinfo='text',\n",
    "        name=node_trace_category,\n",
    "        showlegend=showlegend,\n",
    "        marker=dict(\n",
    "            symbol=nodesymbol,\n",
    "            opacity=nodeopacity,\n",
    "            showscale=False,\n",
    "            color=color,\n",
    "            size=nodesize,\n",
    "            line=dict(width=linewidth, color=linecolor)))\n",
    "    return node_trace"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b53bc502-7803-4983-8689-18821ebc279a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def create_edge_trace(Graph, name, showlegend=False, color='#888', width=0.5, opacity=0.8):\n",
    "    edge_trace = go.Scatter(\n",
    "        x=[],\n",
    "        y=[],\n",
    "        name=name,\n",
    "        opacity=opacity,\n",
    "        line=dict(width=width,color=color),\n",
    "        hoverinfo='none',\n",
    "        mode='lines',\n",
    "        showlegend=showlegend)\n",
    "\n",
    "    edges = np.array([edge for edge in Graph.edges() if G.edges[edge][\"relation_type\"] == name])\n",
    "    pos = np.array([Graph.nodes[e]['pos'] for e in edges.flatten()]).reshape(-1, 2)\n",
    "    xs = np.insert(pos[:, 0], np.arange(2, len(pos[:, 0]), 2), None)\n",
    "    ys = np.insert(pos[:, 1], np.arange(2, len(pos[:, 1]), 2), None)\n",
    "    edge_trace['x'] = xs\n",
    "    edge_trace['y'] = ys\n",
    "\n",
    "    return edge_trace"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "62d796f8-d3eb-4891-bbd3-9189332d178f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def group_markers(points, name, color=\"red\", width=0.5, showlegend=False, fill=\"toself\", opacity=0.8, textposition=\"bottom center\", offset='default'):\n",
    "    # Find the convex hull of the points\n",
    "    hull = spatial.ConvexHull(points)\n",
    "\n",
    "    # Get the x and y coordinates of the vertices of the hull\n",
    "    x = [points[i][0] for i in hull.vertices] + [points[hull.vertices[0]][0]]\n",
    "    y = [points[i][1] for i in hull.vertices] + [points[hull.vertices[0]][1]]\n",
    "\n",
    "    # Calculate the center of the hull\n",
    "    center_x = np.mean(x)\n",
    "    center_y = np.mean(y)\n",
    "    \n",
    "    if offset == \"default\":\n",
    "        if len(points) < 50:\n",
    "            offset = len(points) * (-0.5/47) + 1.8\n",
    "        else:\n",
    "            offset = 1\n",
    "\n",
    "    # Add an offset of 1 pixel to each vertex\n",
    "    offset_x = [(xi - center_x) * offset + center_x for xi in x]\n",
    "    offset_y = [(yi - center_y) * offset + center_y for yi in y]\n",
    "    trace = go.Scatter(name=name, x=offset_x, y=offset_y, mode='lines', line=dict(width=width, color=color), \n",
    "                       showlegend=showlegend, fill=fill, opacity=opacity, textposition=textposition)\n",
    "    return trace"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "03b8f6c6-3d55-427d-8fa9-3918250a6265",
   "metadata": {},
   "source": [
    "## File Configuration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eb7ea48e-961a-4b07-872b-0a686c2f03dd",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "with open(\"config.yaml\", \"r\") as f:\n",
    "    notebook_configuration = yaml.safe_load(f)\n",
    "notebook_configuration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "575bf3cb-9d40-4895-aa06-763d7a89d266",
   "metadata": {},
   "outputs": [],
   "source": [
    "bgcflow_dir = Path(notebook_configuration[\"bgcflow_dir\"])\n",
    "project_name = \"mq_saccharopolyspora\"\n",
    "report_dir = bgcflow_dir / f\"data/processed/{project_name}\"\n",
    "FIGURE = \"Figure_5\"\n",
    "FIGURE_ARTS = \"Figure_4\"\n",
    "FIGURE_MASH = \"Figure_3\"\n",
    "FIGURE_BIGFAM = \"Figure_4\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "df16d0c5-65c4-439a-a9a1-0dbb89e50729",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Setting Up Cutoff and Inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "16f4f557-54e8-40d3-86d0-51fa3ac31034",
   "metadata": {},
   "outputs": [],
   "source": [
    "cutoff = \"0.30\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3faa412d-23ab-4260-8eee-068bfdedc908",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "path_bigscape = report_dir / \"bigscape/for_cytoscape_antismash_6.1.1/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d6b15211-e1a3-40c6-88a6-2b5dce43940e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "df_bigscape = pd.read_csv([i for i in path_bigscape.glob(f\"*df_network*{cutoff}*\")][0])\n",
    "df_bigscape_cluster = pd.read_csv([i for i in path_bigscape.glob(f\"*df_cluster*{cutoff}*\")][0])\n",
    "df_antismash = pd.read_csv(report_dir / \"tables/df_regions_antismash_6.1.1.csv\")\n",
    "df_bigfam = pd.read_csv(report_dir / \"bigslice/query_as_6.1.1/query_network.csv\")\n",
    "df_arts = pd.read_csv(f\"assets/tables/{FIGURE_ARTS}_df_arts_hits.csv\")\n",
    "df_mash = pd.read_csv(f\"assets/tables/{FIGURE_MASH}b_mash_hcluster.csv\", index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "085afcac-e1b6-4a8f-b3f5-e47a73ae6ab1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# clean up MIBIG ids with extra .1\n",
    "for i in df_bigscape.index:\n",
    "    if 'BGC' in df_bigscape.loc[i, 'Clustername 1']:\n",
    "        df_bigscape.loc[i, 'Clustername 1'] = df_bigscape.loc[i, 'Clustername 1'].split(\".\")[0]\n",
    "        print(df_bigscape.loc[i, 'Clustername 1'])\n",
    "    if 'BGC' in df_bigscape.loc[i, 'Clustername 2']:\n",
    "        df_bigscape.loc[i, 'Clustername 2'] = df_bigscape.loc[i, 'Clustername 2'].split(\".\")[0]\n",
    "        print(df_bigscape.loc[i, 'Clustername 2'])\n",
    "\n",
    "for i in df_bigscape_cluster.index:\n",
    "    if 'BGC' in df_bigscape_cluster.loc[i, 'bgc_id']:\n",
    "        df_bigscape_cluster.loc[i, 'bgc_id'] = df_bigscape_cluster.loc[i, 'bgc_id'].split(\".\")[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "123804bb-7a33-4ec7-8680-649cad37c257",
   "metadata": {},
   "outputs": [],
   "source": [
    "bigfam_models_stats = pd.read_csv(f\"assets/tables/{FIGURE_BIGFAM}_bigfam_models.csv\", index_col=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "663805f6-6c9c-41c1-9bc5-71481206a904",
   "metadata": {},
   "source": [
    "## Building Edges for Networkx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ed1ea922-5905-4c70-a16e-ab17c98acc8b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "edge_bigscape = df_bigscape.loc[:, ['Clustername 1','Clustername 2']]\n",
    "edge_bigscape = edge_bigscape.rename(columns={'Clustername 1' : 'source',\n",
    "                                              'Clustername 2' : 'target'})\n",
    "edge_bigscape['relation_type'] = 'bigscape_similarity'\n",
    "edge_bigscape['weight'] = 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "40698abb-493e-42a1-9ffa-de9b1d447db9",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "edge_antismash = df_antismash.loc[:, ['bgc_id', 'most_similar_known_cluster_id', 'similarity']].dropna()\n",
    "edge_antismash = edge_antismash.rename(columns={'bgc_id' : 'source', \n",
    "                                                'most_similar_known_cluster_id' : 'target',\n",
    "                                                'similarity' : 'value'\n",
    "                                               })\n",
    "edge_antismash['relation_type'] = 'antismash_knownclusterblast' \n",
    "edge_antismash['weight'] = 0.01"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ebcff61c-c771-442e-8c0c-3de54d8bc1f7",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "edge_arts = df_arts.loc[:, ['profile', 'bgc_id']]\n",
    "edge_arts = edge_arts.rename(columns={'profile' : 'source', \n",
    "                                      'bgc_id' : 'target'})\n",
    "edge_arts['relation_type'] = 'arts_hits'\n",
    "edge_arts['weight'] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "df5f269d-c2d9-43c2-a02c-33605a338566",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "edge_bigscape_class = df_bigscape_cluster.loc[:, ['bgc_id', 'bigscape_class']]\n",
    "edge_bigscape_class = edge_bigscape_class.rename(columns={'bgc_id' : 'source', \n",
    "                                      'bigscape_class' : 'target'})\n",
    "edge_bigscape_class['relation_type'] = 'bigscape_class'\n",
    "edge_bigscape_class['weight'] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dc8b6564-3393-4a27-bbdb-7ab77e070a4e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "edge_bigfam = df_bigfam[df_bigfam['rank'] == 0].loc[:, ['gcf_id', 'bgc_id']]\n",
    "edge_bigfam = edge_bigfam.rename(columns={'gcf_id' : 'source', \n",
    "                                          'bgc_id' : 'target'})\n",
    "edge_bigfam['relation_type'] = 'bigfam_hits'\n",
    "edge_bigfam['weight'] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a6db58d4-2e60-446e-ba33-87b09791acce",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "for i in df_bigscape_cluster.index:\n",
    "    genome_id = df_bigscape_cluster.loc[i, \"genome_id\"] \n",
    "    df_bigscape_cluster.loc[i, \"phylogroup\"] = df_mash.loc[genome_id, \"phylogroup\"]\n",
    "edge_mash = df_bigscape_cluster.loc[:, [\"bgc_id\", \"phylogroup\"]]\n",
    "edge_mash = edge_mash.rename(columns={'bgc_id' : 'source', \n",
    "                                          'phylogroup' : 'target'})\n",
    "edge_mash['relation_type'] = 'phylogroup'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ebe25335-47f2-4486-b40a-d2b18c609719",
   "metadata": {},
   "source": [
    "## Build mapping categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b19bde59-2f80-4f86-ae1c-b98a044558a5",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "nodemap_antismash = df_antismash.loc[:, ['bgc_id', 'product', 'most_similar_known_cluster_description', 'similarity']].set_index('bgc_id')\n",
    "nodemap_antismash['similarity'] = [f\"{i:.2f}\" for i in nodemap_antismash['similarity']]\n",
    "nodemap_antismash = nodemap_antismash.astype(str).T.to_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eb3b9f21-11c7-4c14-8273-b28886a853be",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "nodemap_mibig = pd.read_csv(bgcflow_dir / \"resources/mibig/df_mibig_bgcs.csv\", index_col=0)\n",
    "nodemap_mibig = nodemap_mibig.loc[:, [\"biosyn_class\", \"compounds\"]].T.to_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "62e63dec-2210-4957-94da-20b88b60bc09",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "nodemap_mibig_bigscape = list(set([i for i in edge_bigscape.source if i.startswith(\"BGC\")]).union(set([i for i in edge_bigscape.target if i.startswith(\"BGC\")])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1f4e0844-b67f-43b4-bb3a-2c01a7eed289",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "nodemap_arts = df_arts.set_index(\"profile\").fillna(\"\").loc[:, [\"hits_type\", \"function\", \"description\"]].T.to_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "880798fc-d693-4268-a87c-aaaf94524e64",
   "metadata": {},
   "outputs": [],
   "source": [
    "nodemap_bigfam = bigfam_models_stats.loc[:, [\"top_chemical_class\",\"top_chemical_class_proportion\",\"top_chemical_subclass\",\"top_chemical_subclass_proportion\", \"H-index\",\"richness\",\"top_taxa\",\"top_taxa_proportion\"]]\n",
    "nodemap_bigfam = nodemap_bigfam.astype(str).T.to_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7cac5d9a-4c88-45b8-add7-4a9bd7e08014",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "arts_map = list(df_arts.profile.unique())\n",
    "bigfam_map = list(df_bigfam.astype(\"str\").gcf_id.unique())\n",
    "antismash_map = list(df_antismash.bgc_id.unique())\n",
    "phylogroup_map = list(df_mash.phylogroup.unique())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "94ee8dd7-f57c-436b-bffd-d6c1e0c39d95",
   "metadata": {},
   "source": [
    "## Build Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "42d46897-b4de-40f2-91b8-1e6f3b329526",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "df = pd.concat([edge_antismash, edge_bigscape, edge_bigfam, edge_arts])\n",
    "df.to_csv(f\"assets/tables/{FIGURE}_raw_network.csv\")\n",
    "df = df.astype(str)\n",
    "G = nx.from_pandas_edgelist(df, source='source', target='target', edge_attr=['weight', 'relation_type', 'value'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f35071c7-0f8f-49a2-8e0e-2a5417186ee2",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "### Create groupings\n",
    "# drop all ARTS\n",
    "G_arts = G.copy()\n",
    "for n in df_arts.profile.unique():\n",
    "    try:\n",
    "        G_arts.remove_node(str(n))\n",
    "    except nx.NetworkXError as e:\n",
    "        print(e)\n",
    "    \n",
    "\n",
    "for num, query in enumerate(nx.connected_components(G_arts)):\n",
    "    for n in query:\n",
    "        G.nodes[n]['group'] = num\n",
    "    if len(query) > 1:\n",
    "        for n in query:\n",
    "            G.nodes[n]['cluster'] = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "31365a25-4706-49d6-b69e-10cb9aed80f2",
   "metadata": {},
   "outputs": [],
   "source": [
    "nx.write_edgelist(G, f\"assets/data/{FIGURE}_integrated_network_{cutoff}.graphml\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "79309120-6e5a-4507-92e0-f87debbff124",
   "metadata": {},
   "source": [
    "## Filtering\n",
    "### Remove Unreliable BiG-FAM models\n",
    "This part requires analysis from [FigS12_query_bigfam.ipynb](FigS12_query_bigfam.ipynb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "999f481b-b32d-49a2-8852-ec50989688e8",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "deleted_bigfam = []\n",
    "bigfam_taxa_cutoff = 0.3\n",
    "bigfam_filter = bigfam_models_stats[bigfam_models_stats.top_taxa_proportion <= bigfam_taxa_cutoff]\n",
    "for n in bigfam_filter.index:\n",
    "    try:\n",
    "        G.remove_node(str(n))\n",
    "        deleted_bigfam.append(n)  \n",
    "    except nx.NetworkXError as e:\n",
    "        print(e)\n",
    "\n",
    "deleted_bigfam = ', '.join([str(i) for i in deleted_bigfam])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "19b8daa5-963e-4a81-b20d-adcd13c4f41e",
   "metadata": {},
   "source": [
    "### Network Annotation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7bf9e4ed-d9f7-4a79-a3d1-68ac874478be",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# define layout options\n",
    "options = {\n",
    "    'prog': 'neato',\n",
    "    'args': ' '.join(['-Gstart=10', '-Goverlap_scaling=-100'])\n",
    "#    'args': ' '.join([\n",
    "#        '-Goverlap=false',\n",
    "#        '-Gsplines=true',\n",
    "#        '-Gnodesep=0.5',\n",
    "#        '-Granksep=1'\n",
    "#    ])\n",
    "}\n",
    "\n",
    "# position nodes\n",
    "pos = nx.nx_agraph.graphviz_layout(G, **options)#, args='-Goverlap=false -Elen=weight')\n",
    "for n, p in pos.items():\n",
    "    G.nodes[n]['pos'] = p\n",
    "    G.nodes[n]['text'] = n\n",
    "    if n in arts_map:\n",
    "        node_trace = 'ARTS model'\n",
    "        G.nodes[n]['text'] = n + \"<br>\" + \"<br>\".join(nodemap_arts[n].values())\n",
    "    elif n in bigfam_map:\n",
    "        node_trace = 'BiG-FAM model'\n",
    "        try:\n",
    "            G.nodes[n]['text'] = n + \"<br>\" + \"<br>\".join(nodemap_bigfam[int(n)].values())\n",
    "        except KeyError:\n",
    "            print(f\"Node {n} not found in bigfam_mapping\")\n",
    "    elif n in antismash_map:\n",
    "        node_trace = \"BGC\"\n",
    "        try:\n",
    "            G.nodes[n]['text'] = n + \"<br>\" + \"<br>\".join(nodemap_antismash[n].values())\n",
    "            #if float(nodemap_antismash[n]['similarity']) < 0.4:\n",
    "            #    node_trace = \"BGC < 40% similarity\"\n",
    "            #elif float(nodemap_antismash[n]['similarity']) < 0.8:\n",
    "            #    node_trace = \"BGC < 80% similarity\"\n",
    "            #elif float(nodemap_antismash[n]['similarity']) >= 0.8:\n",
    "            #    node_trace = \"BGC >= 80% similarity\"\n",
    "        except TypeError as e:\n",
    "            print(n, e)\n",
    "    elif n.startswith(\"BGC\"):\n",
    "        node_trace = \"MIBIG (KnownClusterBlast)\"\n",
    "        if n in nodemap_mibig_bigscape:\n",
    "            node_trace = \"MIBIG (BiG-SCAPE)\"\n",
    "        try:\n",
    "            G.nodes[n]['text'] = n + \"<br>\" + \"<br>\".join(nodemap_mibig[n].values())\n",
    "        except KeyError:\n",
    "            pass\n",
    "    if n in list(edge_mash.source):\n",
    "        phylogroup = edge_mash.set_index(\"source\").loc[n, \"target\"]\n",
    "        G.nodes[n]['text'] = \"<br>\".join([phylogroup, G.nodes[n]['text']])\n",
    "    G.nodes[n]['node_trace'] = node_trace"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "db5b935c-34b2-463d-880c-a35837512e91",
   "metadata": {},
   "source": [
    "### Export positions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f528a133-67ec-4f78-ab3f-989dd6c24ffa",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "df_export_network = pd.DataFrame.from_dict(dict(G.nodes(data=True)), orient='index')\n",
    "for i in df_export_network.index:\n",
    "    df_export_network.loc[i, \"x_pos\"] = df_export_network.loc[i, \"pos\"][0]\n",
    "    df_export_network.loc[i, \"y_pos\"] = df_export_network.loc[i, \"pos\"][1]\n",
    "    df_export_network.loc[i, \"cutoff\"] = cutoff\n",
    "df_export_network.to_csv(f\"assets/tables/{FIGURE}_network_{cutoff}.csv\")"
   ]
  },
  {
   "cell_type": "raw",
   "id": "6d250eda-802e-4580-a73c-9a3eea6edd5b",
   "metadata": {
    "tags": []
   },
   "source": [
    "text_map = {}\n",
    "for g in G.nodes:\n",
    "    if 'ranthipeptide' in G.nodes[g]['text']:\n",
    "        try:\n",
    "            text = df_bigscape_cluster.set_index(\"bgc_id\").loc[g, [\"fam_known_compounds_0.30\"]].to_list()\n",
    "            gcf = text[0]\n",
    "            text_as = df_antismash.set_index(\"bgc_id\").loc[g, [\"most_similar_known_cluster_description\", \"similarity\"]].to_list()\n",
    "            if any([type(i) == str for i in text_as]) == True:\n",
    "                text = text + text_as\n",
    "                text = f'{[\" \".join(text[:2])][0]} ({text[-1]:.0%})'\n",
    "                #print(text)\n",
    "            else:\n",
    "                text = \" \".join(text)\n",
    "            text_map[gcf] = {\"text\" : text, \"node\" : g}\n",
    "        except KeyError:\n",
    "            text = \"\"\n",
    "    else:\n",
    "        text = \"\"\n",
    "\n",
    "for g in G.nodes:\n",
    "    if 'ranthipeptide' in G.nodes[g]['text']:\n",
    "        text = [v['text'] for k, v in text_map.items() if v['node'] == g]\n",
    "        if len(text) == 1:\n",
    "            G.nodes[g]['text'] = text[0]\n",
    "        else:\n",
    "             G.nodes[g]['text'] = \"\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4d0c26bf-301c-41ec-bfb0-73f3cbd1f29d",
   "metadata": {},
   "source": [
    "# Calculate Statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b1bb233b-fe06-4970-8744-e42008e408ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "def write_stats(subgraph, condition, antismash_map):   \n",
    "    number_of_bgc_nodes = [n for n in subgraph.nodes if subgraph.nodes[n]['node_trace'] == \"BGC\"]\n",
    "    number_of_mibig_bigscape_nodes = [n for n in subgraph.nodes if subgraph.nodes[n]['node_trace'] == \"MIBIG (BiG-SCAPE)\"]\n",
    "    number_of_mibig_knownclusterblast_nodes = [n for n in subgraph.nodes if subgraph.nodes[n]['node_trace'] == \"MIBIG (KnownClusterBlast)\"]\n",
    "    number_of_mibig_nodes = number_of_mibig_bigscape_nodes + number_of_mibig_knownclusterblast_nodes\n",
    "    connected_comp = [n for n in nx.connected_components(subgraph)]\n",
    "    connected_comp_MIBIG = [n for n in connected_comp if any([x.startswith(\"BGC\") for x in n])]\n",
    "    connected_comp_MIBIG_BGC_only = [[x for x in n if subgraph.nodes[x]['node_trace'] == \"BGC\" ] for n in connected_comp_MIBIG]\n",
    "    singleton_bgc = [n for n in connected_comp if len([x for x in n if G.nodes[x]['node_trace'] == \"BGC\"]) == 1]\n",
    "    true_singleton_bgc = [n for n in connected_comp if len(n) == 1]\n",
    "\n",
    "    number_of_mibig_bigscape_nodes = [n for n in subgraph.nodes if subgraph.nodes[n]['node_trace'] == \"MIBIG (BiG-SCAPE)\"]\n",
    "    number_of_mibig_knownclusterblast_nodes = [n for n in subgraph.nodes if subgraph.nodes[n]['node_trace'] == \"MIBIG (KnownClusterBlast)\"]\n",
    "    connected_comp = [n for n in nx.connected_components(subgraph)]\n",
    "    connected_comp_MIBIG_bigscape_nodes = [n for n in connected_comp if any([x in number_of_mibig_bigscape_nodes for x in n])]\n",
    "    connected_comp_MIBIG_knownclusterblast_nodes = [n for n in connected_comp if any([x in number_of_mibig_knownclusterblast_nodes for x in n])]\n",
    "    connected_comp_MIBIG_knownclusterblast_nodes = [n for n in connected_comp_MIBIG_knownclusterblast_nodes if n not in connected_comp_MIBIG_bigscape_nodes]\n",
    "    connected_comp_no_MIBIG = [n for n in connected_comp if not any([x.startswith(\"BGC\") for x in n])]\n",
    "    connected_comp_MIBIG_knownclusterblast_nodes_above_80 = []\n",
    "    connected_comp_MIBIG_knownclusterblast_nodes_below_80 = []\n",
    "    \n",
    "    \n",
    "    for n in connected_comp_MIBIG_knownclusterblast_nodes:\n",
    "        result = []\n",
    "        for x in n:\n",
    "            if x in antismash_map:\n",
    "                if float(nodemap_antismash[x]['similarity']) >= 0.8:\n",
    "                    result.append(x)\n",
    "        if len(result) > 0:\n",
    "            connected_comp_MIBIG_knownclusterblast_nodes_above_80.append(n)\n",
    "        else:\n",
    "            connected_comp_MIBIG_knownclusterblast_nodes_below_80.append(n)\n",
    "                \n",
    "    GCF_summary = {\"total_gcf\" : len(connected_comp),\n",
    "                   \"total_bgc\" : sum([len([x for x in n if subgraph.nodes[x]['node_trace'].startswith(\"BGC\")]) for n in connected_comp]),\n",
    "                   \"MIBIG_bigscape_gcf\" : len(connected_comp_MIBIG_bigscape_nodes), \n",
    "                   \"MIBIG_bigscape_bgc\" : sum([len([x for x in n if subgraph.nodes[x]['node_trace'].startswith(\"BGC\")]) for n in connected_comp_MIBIG_bigscape_nodes]), \n",
    "                   \"MIBIG_knownclusterblast_gcf\" : len(connected_comp_MIBIG_knownclusterblast_nodes),\n",
    "                   \"MIBIG_knownclusterblast_bgc\" : sum([len([x for x in n if subgraph.nodes[x]['node_trace'].startswith(\"BGC\")]) for n in connected_comp_MIBIG_knownclusterblast_nodes]),\n",
    "                   \"MIBIG_knownclusterblast_above_80_gcf\" : len(connected_comp_MIBIG_knownclusterblast_nodes_above_80),\n",
    "                   \"MIBIG_knownclusterblast_above_80_bgc\" : sum([len([x for x in n if subgraph.nodes[x]['node_trace'].startswith(\"BGC\")]) for n in connected_comp_MIBIG_knownclusterblast_nodes_above_80]),\n",
    "                   \"MIBIG_knownclusterblast_below_80_gcf\" : len(connected_comp_MIBIG_knownclusterblast_nodes_below_80),\n",
    "                   \"MIBIG_knownclusterblast_below_80_bgc\" : sum([len([x for x in n if subgraph.nodes[x]['node_trace'].startswith(\"BGC\")]) for n in connected_comp_MIBIG_knownclusterblast_nodes_below_80]),\n",
    "                   \"unknown_gcf\" : len(connected_comp_no_MIBIG),\n",
    "                   \"unknown_bgc\" : sum([len([x for x in n if subgraph.nodes[x]['node_trace'].startswith(\"BGC\")]) for n in connected_comp_no_MIBIG])\n",
    "                 }\n",
    "    \n",
    "    text = f\"\"\"\n",
    "In the {condition}, there are {nx.number_of_nodes(subgraph)} nodes and {nx.number_of_edges(subgraph)} edges.\n",
    "From these, {len(number_of_bgc_nodes)} are BGCs, {len(number_of_mibig_bigscape_nodes)} are MIBIG entries identified with BiG-SCAPE, and {len(number_of_mibig_knownclusterblast_nodes)} are MIBIG entries identified with KnownClusterBlast.\n",
    "In these network, there are {len(connected_comp)} connected components (GCFs), with {len(singleton_bgc)} singleton BGCs ({len(true_singleton_bgc)} true singletons). \n",
    "Of these, there are {sum([len(n) for n in connected_comp_MIBIG_BGC_only])} BGCs that belongs to {len(connected_comp_MIBIG)} GCFs with similariy to MIBIG nodes.\n",
    "\n",
    "{\" ; \".join([f'{k} : {v}' for k,v in GCF_summary.items()])}\n",
    "    \n",
    "    \"\"\"\n",
    "    return text"
   ]
  },
  {
   "cell_type": "raw",
   "id": "a5174232-6717-4128-acc8-026619a83804",
   "metadata": {},
   "source": [
    "[[x for x in n if subgraph.nodes[x]['node_trace'].startswith(\"BGC\")] for n in connected_comp_no_MIBIG]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6792f78f-0aff-4248-9124-5457f3a474e3",
   "metadata": {},
   "outputs": [],
   "source": [
    "edges_to_keep = [e for e in G.edges if G.edges[e]['relation_type'] == 'bigscape_similarity']\n",
    "subgraph = G.edge_subgraph(edges_to_keep)\n",
    "condition = f\"BiG-SCAPE network with ({cutoff} cutoff)\"\n",
    "text = write_stats(subgraph, condition, antismash_map)\n",
    "\n",
    "outfile = f\"assets/figures/{FIGURE}/{FIGURE}_text_bigscape_only_{cutoff}.txt\"\n",
    "Path(outfile).parent.mkdir(parents=True, exist_ok=True)\n",
    "with open(outfile, \"w\") as f:\n",
    "    f.writelines(text)\n",
    "print(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "318db7d9-fcb3-4081-bc1e-bad12ad9778b",
   "metadata": {},
   "outputs": [],
   "source": [
    "edges_to_keep = [e for e in G.edges if G.edges[e]['relation_type'] != 'arts_hits']\n",
    "subgraph = G.edge_subgraph(edges_to_keep)\n",
    "condition = f\"integrated BiG-SCAPE network with ({cutoff} cutoff)\"\n",
    "text = write_stats(subgraph, condition, antismash_map)\n",
    "\n",
    "outfile = f\"assets/figures/{FIGURE}/{FIGURE}_text_bigscape_integrated_{cutoff}.txt\"\n",
    "Path(outfile).parent.mkdir(parents=True, exist_ok=True)\n",
    "with open(outfile, \"w\") as f:\n",
    "    f.writelines(text)\n",
    "print(text)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bbefce77-0561-4a7b-8b66-1bae3fbaaf6f",
   "metadata": {},
   "source": [
    "## Define visualizations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "86400ccf-9ccd-4076-aaa9-f13aed84d000",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "edge_annotation_map = {'bigscape_similarity' : {'color':'black',\n",
    "                                                'width':0.5}, \n",
    "                       'antismash_knownclusterblast': {'color':'blue', \n",
    "                                                       'width':0.5}, \n",
    "                       'arts_hits': {'color':'red', \n",
    "                                     'width':0.5}, \n",
    "                       'bigfam_hits': {'color':'orange', \n",
    "                                       'width':0.5},\n",
    "                       'phylogroup' : {'color' : 'black',\n",
    "                                     'width' : 0.5}}\n",
    "\n",
    "node_annotation_map = {'ARTS model' : {'color':'red',\n",
    "                                       'node_symbol' : 'star'},\n",
    "                       'BiG-FAM model' : {'color':'orange',\n",
    "                                          'node_symbol' : 'triangle-up'},\n",
    "                       'MIBIG (KnownClusterBlast)' : {'color':'green',\n",
    "                                                      'node_symbol' : 'square'},\n",
    "                       'MIBIG (BiG-SCAPE)' : {'color':'blue',\n",
    "                                              'node_symbol' : 'square'},\n",
    "                       \"BGC >= 80% similarity\" : {'color':'blue',\n",
    "                                                  'node_symbol' : 'circle'},\n",
    "                       \"BGC < 80% similarity\" : {'color':'#0077b6',\n",
    "                                                 'node_symbol' : 'circle'},\n",
    "                       \"BGC < 40% similarity\" : {'color':'#90e0ef',\n",
    "                                                 'node_symbol' : 'circle'},\n",
    "                       'BGC' : {'color':'#90e0ef',\n",
    "                                          'node_symbol' : 'circle'},\n",
    "                       'phylogroup' : {'color' : 'black',\n",
    "                                       'node_symbol' : 'star'}\n",
    "                      }"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "76709272-16de-473f-b98c-f641bdc8824c",
   "metadata": {},
   "source": [
    "## Draw Target Box"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7d6ae274-2fc6-423d-b785-af8853ecea46",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "x, y = G.nodes['BGC0000148']['pos']\n",
    "x1 = x - 200\n",
    "x2 = x + 200\n",
    "y1 = y - 200\n",
    "y2 = y + 200"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e94a33db-2889-40ca-b5ee-1762ecace13a",
   "metadata": {},
   "source": [
    "# Figure 5D"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c055310e-b3fd-49c3-b0f8-d00f73f39b48",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "traces = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7d626aec-9fa5-499b-9de1-2ea5db796bba",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "showlegend=False\n",
    "color=\"grey\"\n",
    "\n",
    "# drop all ARTS\n",
    "G_arts = G.copy()\n",
    "for n in df_arts.profile.unique():\n",
    "    try:\n",
    "        G_arts.remove_node(str(n))\n",
    "    except nx.NetworkXError as e:\n",
    "        print(e)\n",
    "    \n",
    "\n",
    "for num, query in enumerate(nx.connected_components(G_arts)):\n",
    "    if len(query) > 2:\n",
    "        points = []\n",
    "        for n in query:\n",
    "            q = G.nodes[n]['pos']\n",
    "            points.append(q)\n",
    "        traces.append(group_markers(points, str(num), color=\"blue\", #color=\"#2a9d8f\", \n",
    "                                    width=1, showlegend=showlegend, fill=None, \n",
    "                                    #offset=1.5, \n",
    "                                    opacity=0.8))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7ef0f54b-a17e-44b3-8c3d-54584b6f4842",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "for e in ['bigscape_similarity', 'arts_hits', 'bigfam_hits', \"antismash_knownclusterblast\"]:\n",
    "    edge_trace = create_edge_trace(G, e, color=edge_annotation_map[e]['color'], showlegend=True)\n",
    "    traces.append(edge_trace)\n",
    "\n",
    "for trace in ['ARTS model', 'BiG-FAM model', 'MIBIG (KnownClusterBlast)', 'MIBIG (BiG-SCAPE)', 'BGC']:\n",
    "    nodeopacity = 0.5\n",
    "    showtextlabel = False\n",
    "    linecolor = \"black\"\n",
    "    linewidth = 0.5\n",
    "    textposition=\"top left\"\n",
    "    node_size = 8\n",
    "    if trace.startswith(\"BGC\") or trace.startswith(\"MIBIG (BiG-SCAPE)\"):\n",
    "        nodeopacity = 0.8\n",
    "    node_trace = create_node_trace(G, trace, node_annotation_map[trace]['color'], showtextlabel=showtextlabel, \n",
    "                                   nodesymbol=node_annotation_map[trace]['node_symbol'], nodeopacity=nodeopacity, \n",
    "                                   showlegend=True, linecolor=linecolor, linewidth=linewidth, nodesize=node_size,\n",
    "                                   textposition=textposition)\n",
    "    traces.append(node_trace)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2be00a83-a635-448d-84c9-556f25d858c2",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "fig5D = go.Figure(data=traces,\n",
    "                layout=go.Layout(\n",
    "                    paper_bgcolor='rgba(0,0,0,0)',\n",
    "                    plot_bgcolor='white',\n",
    "                    showlegend=False,\n",
    "                    hovermode='closest',\n",
    "                    margin=dict(b=20,l=5,r=5,t=40),\n",
    "                    xaxis=dict(showgrid=False, zeroline=False, showticklabels=False, linecolor='black', mirror=True, linewidth=1),\n",
    "                    yaxis=dict(showgrid=False, zeroline=False, showticklabels=False, linecolor='black', mirror=True, linewidth=1),\n",
    "                    width=600, height=600)\n",
    "                )\n",
    "\n",
    "outfile = Path(f\"assets/figures/{FIGURE}/{FIGURE}D_{cutoff}.html\")\n",
    "outfile.parent.mkdir(exist_ok=True, parents=True)\n",
    "fig5d_html = fig5D.update_layout(width=600, height=600)\n",
    "fig5d_html.write_html(outfile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6cd80643-fd6d-42af-8325-2daba0bd0e2f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "square = [x1, y1, x2, y2]\n",
    "fig5D.add_shape(type=\"rect\",\n",
    "               x0=square[0], y0=square[1], x1=square[2], y1=square[3],\n",
    "               line=dict(color=\"Red\", width=2),\n",
    ")\n",
    "\n",
    "fig5D.update_layout(width=600, height=600,\n",
    "    font=dict(\n",
    "        #family=\"Courier New, monospace\",\n",
    "        size=8,  # Set the font size here\n",
    "        #color=\"RebeccaPurple\"\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "40b30854-8c37-4541-b890-e8b3646b644f",
   "metadata": {},
   "outputs": [],
   "source": [
    "outfile = Path(f\"assets/figures/{FIGURE}/{FIGURE}D_{cutoff}.svg\")\n",
    "outfile.parent.mkdir(exist_ok=True, parents=True)\n",
    "fig5D.write_image(outfile)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d68a2d44-f0ff-4727-8da9-76a94894f976",
   "metadata": {},
   "source": [
    "# Figure 5B"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0f059635-39de-4969-9e14-073a57d7740b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "traces = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "42865fba-b724-4428-918f-8d94f72440ea",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "for e in ['bigscape_similarity', 'arts_hits', 'bigfam_hits', \"antismash_knownclusterblast\"]:\n",
    "    edge_trace = create_edge_trace(G, e, color=edge_annotation_map[e]['color'], showlegend=True, width=3)\n",
    "    traces.append(edge_trace)\n",
    "\n",
    "for trace in ['ARTS model', 'BiG-FAM model', 'MIBIG (KnownClusterBlast)', 'MIBIG (BiG-SCAPE)', 'BGC']:\n",
    "    nodeopacity = 0.5\n",
    "    showtextlabel = False\n",
    "    linecolor = \"black\"\n",
    "    linewidth = 0.5\n",
    "    textposition=\"top left\"\n",
    "    node_size = 25\n",
    "    if trace.startswith(\"BGC\") or trace.startswith(\"MIBIG (BiG-SCAPE)\"):\n",
    "        nodeopacity = 0.8\n",
    "    node_trace = create_node_trace(G, trace, node_annotation_map[trace]['color'], showtextlabel=showtextlabel, \n",
    "                                   nodesymbol=node_annotation_map[trace]['node_symbol'], nodeopacity=nodeopacity, \n",
    "                                   showlegend=True, linecolor=linecolor, linewidth=linewidth, nodesize=node_size,\n",
    "                                   textposition=textposition)\n",
    "    traces.append(node_trace)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cb60db0d-37ef-4b69-af49-46eaa904fa00",
   "metadata": {},
   "outputs": [],
   "source": [
    "showlegend=False\n",
    "color=\"grey\"\n",
    "for num, query in enumerate(nx.connected_components(G)):\n",
    "    if len(query) > 2:\n",
    "        points = []\n",
    "        for n in query:\n",
    "            q = G.nodes[n]['pos']\n",
    "            points.append(q)\n",
    "        traces.append(group_markers(points, num, color=\"blue\", width=1, showlegend=showlegend, fill=None, opacity=0.8))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "36d0cc5c-5624-49df-acb8-687dd374c81c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "fig5B = go.Figure(data=traces,\n",
    "                layout=go.Layout(\n",
    "                    paper_bgcolor='rgba(0,0,0,0)',\n",
    "                    plot_bgcolor='white',\n",
    "                    showlegend=False,\n",
    "                    hovermode='closest',\n",
    "                    margin=dict(b=20,l=5,r=5,t=40),\n",
    "                    xaxis=dict(showgrid=False, zeroline=False, showticklabels=False, linecolor='black', mirror=True, linewidth=1),\n",
    "                    yaxis=dict(showgrid=False, zeroline=False, showticklabels=False, linecolor='black', mirror=True, linewidth=1),\n",
    "                    width=600, height=600)\n",
    "                )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "86520253-624f-46ec-8f38-d40db075ae71",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "square = [x1, y1, x2, y2]\n",
    "\n",
    "fig5B.update_layout(xaxis=dict(showgrid=False, zeroline=False, showticklabels=False, linecolor='black', mirror=True, linewidth=1, range=[square[0], square[2]]),\n",
    "                    yaxis=dict(showgrid=False, zeroline=False, showticklabels=False, linecolor='black', mirror=True, linewidth=1, range=[square[1], square[3]]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6a47202e-04ed-4e8f-a233-9b687a695e03",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "outfile = Path(f\"assets/figures/{FIGURE}/{FIGURE}B_{cutoff}.svg\")\n",
    "outfile.parent.mkdir(exist_ok=True, parents=True)\n",
    "fig5B.write_image(outfile)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "87cacd56-c5b1-41d0-b41f-b4fcd2027d19",
   "metadata": {},
   "source": [
    "# Create a subproject for spinosyn cluster comparison"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ace829d6-d511-4c60-a935-0fb95acea426",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "query = 'BGC0000148'\n",
    "neighbor = [i for i in G.neighbors(query)]\n",
    "new_project_name = \"spinosyn\"\n",
    "outfile_dir = bgcflow_dir / f\"config/{new_project_name}\"\n",
    "outfile_dir.mkdir(parents=True, exist_ok=True)\n",
    "new_project_sample = df_antismash.set_index('bgc_id', drop=False).loc[neighbor]\n",
    "\n",
    "#.to_csv(outfile_dir / \"samples.csv\")\n",
    "\n",
    "query_gbk = bgcflow_dir / f\"data/external/bgc_selection/MIBIG/{query}.region001.gbk\"\n",
    "#! wget https://dl.secondarymetabolites.org/mibig/mibig_gbk_3.1.tar.gz -nc\n",
    "#! tar -xvzf mibig_gbk_3.1.tar.gz mibig_gbk_3.1/BGC0000148.gbk\n",
    "query_gbk.parent.mkdir(parents=True, exist_ok=True)\n",
    "! wget -O {query_gbk} https://mibig.secondarymetabolites.org/repository/BGC0000148/generated/BGC0000148.gbk\n",
    "#shutil.copy(\"mibig_gbk_3.1/BGC0000148.gbk\", Path(\"../../../../\") / query_gbk)\n",
    "assert query_gbk.is_file()\n",
    "\n",
    "index_col = len(new_project_sample)\n",
    "new_project_sample.loc[index_col, \"genome_id\"] = \"GCF_000062885.1\"\n",
    "new_project_sample.loc[index_col, \"bgc_id\"] = query\n",
    "new_project_sample.loc[index_col, \"source\"] = \"bgcflow\"\n",
    "new_project_sample.loc[index_col, \"gbk_path\"] = query_gbk.resolve()\n",
    "new_project_sample.to_csv(outfile_dir / \"samples.csv\", index=False)\n",
    "\n",
    "\n",
    "project_config = {'name': 'Spinosyn',\n",
    "                  'schema': 'BGC',\n",
    "                  'pep_version': '2.1.0',\n",
    "                  'description': \"A selection of putative Spynosin (BGC0000148) BGCs from mq_saccharopolyspora\",\n",
    "                  'sample_table': 'samples.csv',\n",
    "                  'gtdb-tax': 'gtdbtk.bac120.summary.tsv',\n",
    "                  'rules' : {'bigslice': False,\n",
    "                             'bigscape': False,\n",
    "                             'query-bigslice': False,\n",
    "                             'clinker': True,\n",
    "                             'mmseqs2': True,\n",
    "                             'interproscan': False}\n",
    "                 }\n",
    "\n",
    "source = bgcflow_dir / \"config/mq_saccharopolyspora/gtdbtk.bac120.summary.tsv\"\n",
    "destination = outfile_dir / source.name\n",
    "shutil.copy(source, destination)\n",
    "\n",
    "with open(outfile_dir / \"project_config.yaml\", \"w\") as f:\n",
    "    json.dump(project_config, f, indent=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5f10bc9c-1f26-4d69-bbe5-b73817eddaad",
   "metadata": {},
   "source": [
    "# Figure 5C"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e7c525d0-7e73-4179-a08d-b9bd90b2db60",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "traces = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b48abbb0-a6a9-4afd-bb3d-cea3e57ae563",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "for e in ['bigscape_similarity']:\n",
    "    edge_trace = create_edge_trace(G, e, color=edge_annotation_map[e]['color'], showlegend=True)\n",
    "    traces.append(edge_trace)\n",
    "\n",
    "for trace in ['MIBIG (BiG-SCAPE)', 'BGC']:\n",
    "    nodeopacity = 0.5\n",
    "    showtextlabel = False\n",
    "    linecolor = \"black\"\n",
    "    linewidth = 0.5\n",
    "    textposition=\"top left\"\n",
    "    node_size = 8\n",
    "    if trace.startswith(\"BGC\") or trace.startswith(\"MIBIG (BiG-SCAPE)\"):\n",
    "        nodeopacity = 0.8\n",
    "    node_trace = create_node_trace(G, trace, node_annotation_map[trace]['color'], showtextlabel=showtextlabel, \n",
    "                                   nodesymbol=node_annotation_map[trace]['node_symbol'], nodeopacity=nodeopacity, \n",
    "                                   showlegend=True, linecolor=linecolor, linewidth=linewidth, nodesize=node_size,\n",
    "                                   textposition=textposition)\n",
    "    traces.append(node_trace)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "025d8406-edb5-477e-8524-aaea9bb8a345",
   "metadata": {},
   "outputs": [],
   "source": [
    "for group in df_bigscape_cluster[f'fam_known_compounds_{cutoff}'].unique():\n",
    "    query = list(df_bigscape_cluster[df_bigscape_cluster[f'fam_known_compounds_{cutoff}'] == group].bgc_id)\n",
    "    if len(query) > 2:\n",
    "        points = []\n",
    "        for n in query:\n",
    "            q = G.nodes[n]['pos']\n",
    "            points.append(q)\n",
    "        if str(group).startswith(\"u\"):\n",
    "            showlegend=False\n",
    "            color=\"blue\"\n",
    "        else:\n",
    "            showlegend=False\n",
    "            color=\"blue\"\n",
    "        traces.append(group_markers(points, str(group), color=color, width=1, showlegend=showlegend, fill=None))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "47a3d7d1-5a3f-464a-ba55-0ec2112d1432",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "fig5C = go.Figure(data=traces,\n",
    "                layout=go.Layout(\n",
    "                    paper_bgcolor='rgba(0,0,0,0)',\n",
    "                    plot_bgcolor='white',\n",
    "                    showlegend=False,\n",
    "                    hovermode='closest',\n",
    "                    margin=dict(b=20,l=5,r=5,t=40),\n",
    "                    xaxis=dict(showgrid=False, zeroline=False, showticklabels=False, linecolor='black', mirror=True, linewidth=1),\n",
    "                    yaxis=dict(showgrid=False, zeroline=False, showticklabels=False, linecolor='black', mirror=True, linewidth=1),\n",
    "                    width=600, height=600)\n",
    "                )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "03630d22-856f-4043-a21c-66968d5339dc",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "square = [x1, y1, x2, y2]\n",
    "fig5C.add_shape(type=\"rect\",\n",
    "               x0=square[0], y0=square[1], x1=square[2], y1=square[3],\n",
    "               line=dict(color=\"Red\", width=2),\n",
    ")\n",
    "\n",
    "fig5C.update_layout(\n",
    "    font=dict(\n",
    "        #family=\"Courier New, monospace\",\n",
    "        size=8,  # Set the font size here\n",
    "        #color=\"RebeccaPurple\"\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "49f3ad66-57b4-4816-9381-21f121781e97",
   "metadata": {},
   "outputs": [],
   "source": [
    "outfile = Path(f\"assets/figures/{FIGURE}/{FIGURE}C_{cutoff}.svg\")\n",
    "outfile.parent.mkdir(exist_ok=True, parents=True)\n",
    "fig5C.write_image(outfile)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b6340eba-0c62-4610-af36-12f57a9585d3",
   "metadata": {},
   "source": [
    "# Figure 5A"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7a93c32f-8590-432a-9991-0eb75e27fdd6",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "traces = []\n",
    "\n",
    "for group in df_bigscape_cluster[f'fam_known_compounds_{cutoff}'].unique():\n",
    "    query = list(df_bigscape_cluster[df_bigscape_cluster[f'fam_known_compounds_{cutoff}'] == group].bgc_id)\n",
    "    if len(query) > 2:\n",
    "        points = []\n",
    "        for n in query:\n",
    "            q = G.nodes[n]['pos']\n",
    "            points.append(q)\n",
    "        if str(group).startswith(\"u\"):\n",
    "            showlegend=False\n",
    "            color=\"blue\"\n",
    "        else:\n",
    "            showlegend=False\n",
    "            color=\"blue\"\n",
    "        traces.append(group_markers(points, str(group), color=color, width=1, showlegend=showlegend, fill=None))\n",
    "\n",
    "for e in ['bigscape_similarity']:\n",
    "    edge_trace = create_edge_trace(G, e, color=edge_annotation_map[e]['color'], width=3, showlegend=True)\n",
    "    traces.append(edge_trace)\n",
    "\n",
    "for trace in ['MIBIG (BiG-SCAPE)', 'BGC']:\n",
    "    nodeopacity = 0.5\n",
    "    showtextlabel = False\n",
    "    linecolor = \"black\"\n",
    "    linewidth = 0.5\n",
    "    textposition=\"top left\"\n",
    "    node_size = 25\n",
    "    if trace.startswith(\"BGC\") or trace.startswith(\"MIBIG (BiG-SCAPE)\"):\n",
    "        nodeopacity = 0.8\n",
    "    node_trace = create_node_trace(G, trace, node_annotation_map[trace]['color'], showtextlabel=showtextlabel, \n",
    "                                   nodesymbol=node_annotation_map[trace]['node_symbol'], nodeopacity=nodeopacity, \n",
    "                                   showlegend=True, linecolor=linecolor, linewidth=linewidth, nodesize=node_size,\n",
    "                                   textposition=textposition)\n",
    "    traces.append(node_trace)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eefb055e-a428-4311-8d4c-25a26f04ba2b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "fig5A = go.Figure(data=traces,\n",
    "                layout=go.Layout(\n",
    "                    paper_bgcolor='rgba(0,0,0,0)',\n",
    "                    plot_bgcolor='white',\n",
    "                    showlegend=False,\n",
    "                    hovermode='closest',\n",
    "                    margin=dict(b=20,l=5,r=5,t=40),\n",
    "                    xaxis=dict(showgrid=False, zeroline=False, showticklabels=False, linecolor='black', mirror=True, linewidth=1),\n",
    "                    yaxis=dict(showgrid=False, zeroline=False, showticklabels=False, linecolor='black', mirror=True, linewidth=1),\n",
    "                    width=600, height=600)\n",
    "                )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e074eb0a-f13c-4d1d-8bcc-633e5cae2514",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "square = [x1, y1, x2, y2]\n",
    "\n",
    "fig5A.update_layout(xaxis=dict(showgrid=False, zeroline=False, showticklabels=False, linecolor='black', mirror=True, linewidth=1, range=[square[0], square[2]]),\n",
    "                    yaxis=dict(showgrid=False, zeroline=False, showticklabels=False, linecolor='black', mirror=True, linewidth=1, range=[square[1], square[3]]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "42371413-1eab-4fdb-b816-1f6669b41bb0",
   "metadata": {},
   "outputs": [],
   "source": [
    "outfile = Path(f\"assets/figures/{FIGURE}/{FIGURE}A_{cutoff}.svg\")\n",
    "outfile.parent.mkdir(exist_ok=True, parents=True)\n",
    "fig5A.write_image(outfile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3c8e90d8-0fd7-45c7-9768-92f67176aa75",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "16cf70f4-61ae-42c7-9ee1-8ae96912926d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "fig_legend = go.Figure(data=traces,\n",
    "                layout=go.Layout(\n",
    "                    paper_bgcolor='rgba(0,0,0,0)',\n",
    "                    plot_bgcolor='white',\n",
    "                    showlegend=True,\n",
    "                    hovermode='closest',\n",
    "                    margin=dict(b=20,l=5,r=5,t=40),\n",
    "                    xaxis=dict(showgrid=False, zeroline=False, showticklabels=False, linecolor='black', mirror=True, linewidth=1),\n",
    "                    yaxis=dict(showgrid=False, zeroline=False, showticklabels=False, linecolor='black', mirror=True, linewidth=1),\n",
    "                    width=1200, height=1200)\n",
    "                )\n",
    "outfile = Path(f\"assets/figures/{FIGURE}/{FIGURE}_legend_{cutoff}.svg\")\n",
    "outfile.parent.mkdir(exist_ok=True, parents=True)\n",
    "fig_legend.write_image(outfile)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aaf99feb-fad0-497d-b557-c2b259808695",
   "metadata": {},
   "source": [
    "# Figure 5 Final"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a064f91a-5db4-454b-9600-bdba6fbfbded",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from svgutils.compose import *\n",
    "from svgutils.compose import Figure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d97938c3-2359-40ac-b937-3276d433e547",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "final_figure = Figure(\"1200\", \"1200\",\n",
    "                      Panel(\n",
    "                          SVG(f\"assets/figures/{FIGURE}/{FIGURE}A_{cutoff}.svg\").scale(1).move(0, 0),\n",
    "                          Text(\"A\", 20, 80, size=24, weight='bold'),\n",
    "                      ),\n",
    "                      Panel(\n",
    "                          SVG(f\"assets/figures/{FIGURE}/{FIGURE}B_{cutoff}.svg\").scale(1).move(600, 0),\n",
    "                          Text(\"B\", 620, 80, size=24, weight='bold'),\n",
    "                      ),\n",
    "                      Panel(\n",
    "                          SVG(f\"assets/figures/{FIGURE}/{FIGURE}C_{cutoff}.svg\").scale(1).move(0, 550),\n",
    "                          Text(\"C\", 20, 630, size=24, weight='bold'),\n",
    "                      ),\n",
    "                      Panel(\n",
    "                          SVG(f\"assets/figures/{FIGURE}/{FIGURE}D_{cutoff}.svg\").scale(1).move(600, 550),\n",
    "                          Text(\"D\", 620, 630, size=24, weight='bold'),\n",
    "                      )\n",
    "                     )\n",
    "outfile = Path(f\"assets/figures/{FIGURE}/{FIGURE}_{cutoff}.svg\")\n",
    "outfile.parent.mkdir(parents=True, exist_ok=True)\n",
    "final_figure.save(outfile)\n",
    "final_figure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b861c2fe-dfa2-4c84-9a59-7be951e17e6c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "print(f\"Enrichment of BiG-SCAPE similarity network (cutoff {cutoff}) with KnownClusterBLAST, BiG-FAM, and ARTS models. (C) Blue polygons depicts GCF assigned through BiG-SCAPE similarity network. (D) Blue polygons depicts connected components except through ARTS model. BiG-FAM models {deleted_bigfam} were removed because of assigned top genus in the model are below {bigfam_taxa_cutoff:.0%}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ef2ee5f1-2607-4c52-9cc5-60c9a67921a7",
   "metadata": {},
   "source": [
    "A. Example of BGC similarity network using BiG-SCAPE default cutoffs. B. Example of the same BGCs with additional connection made using knownclusterblast predictions, BiG-SLICE query, and ARTS against the databases of MIBIG, BiGFAM-DB, and ARTS-DB, respectively. C. Overview of the similarity network of all 721 BGCs resulting in 330 connected components with 5 known BGCs and 209 singletons. D. Expanded network with information from knownclusterblast, BiG-SLICE and ARTS resulting in 166 connected components with 112 known BGCs and 32 singletons. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0329442c-dafb-4ef0-b6a7-6797a87fdf21",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "total_bgcs = len(df_bigscape_cluster)\n",
    "bgc_distribution = df_bigscape_cluster['fam_type_0.30'].value_counts().to_dict()\n",
    "family_distribution = df_bigscape_cluster['fam_known_compounds_0.30'].value_counts().to_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "657dd6bb-a66e-4015-8be5-a1eded932545",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "subgraph = [g for g in nx.connected_components(G_arts)]\n",
    "print(f\"Number of connected components: {len(subgraph)}\")\n",
    "\n",
    "subgraph_dict = {}\n",
    "for num, s in enumerate(subgraph):\n",
    "    container = []\n",
    "    for item in s:\n",
    "        if item in df_antismash.bgc_id.unique():\n",
    "            container.append(item)\n",
    "    subgraph_dict[num] = container"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "367378cb-4bf0-4be4-ae07-197713a66b06",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "df_antismash_annotated = df_antismash.copy()\n",
    "df_antismash_annotated  = df_antismash_annotated .set_index(\"bgc_id\")\n",
    "for k in subgraph_dict.keys():\n",
    "    for item in subgraph_dict[k]:\n",
    "        df_antismash_annotated .loc[subgraph_dict[k], \"subgraph\"] = f\"subgraph_{k}\"\n",
    "df_antismash_annotated['fam_known_compounds_0.30'] = df_bigscape_cluster.set_index(\"bgc_id\").loc[:, 'fam_known_compounds_0.30']\n",
    "df_antismash_annotated['fam_type_0.30'] = df_bigscape_cluster.set_index(\"bgc_id\").loc[:, 'fam_type_0.30']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b738f7c3-5bcf-4924-87fc-d8b9ef4f4d97",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "for s in df_antismash_annotated.subgraph.unique():\n",
    "    subset = df_antismash_annotated[df_antismash_annotated['subgraph'] == s]\n",
    "    if 'known_family' in subset[f'fam_type_{cutoff}'].unique():\n",
    "        df_antismash_annotated.loc[subset.index, \"subgraph_category\"] = \"known_cluster_bigscape\"\n",
    "    elif any(subset.similarity.unique() >= 0.8):\n",
    "        df_antismash_annotated.loc[subset.index, \"subgraph_category\"] = \"known_cluster_80%\"\n",
    "    elif all([any(subset.similarity.unique() < 0.8), any(subset.similarity.unique() >= 0.4)]):\n",
    "        df_antismash_annotated.loc[subset.index, \"subgraph_category\"] = \"unknown_cluster_40%_80%\"\n",
    "    elif any(subset.similarity.unique() < 0.4):\n",
    "        df_antismash_annotated.loc[subset.index, \"subgraph_category\"] = \"unknown_cluster_40%\"\n",
    "    elif len(subset) == 1:\n",
    "        df_antismash_annotated.loc[subset.index, \"subgraph_category\"] = \"unknown_cluster_singleton\"\n",
    "    else:\n",
    "        df_antismash_annotated.loc[subset.index, \"subgraph_category\"] = \"unknown_cluster\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1bf06464-f2e7-4fb3-bd67-aaeaaac69916",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "subgraph_dict = {}\n",
    "for s in df_antismash_annotated.subgraph_category.unique():\n",
    "    subset = df_antismash_annotated[df_antismash_annotated['subgraph_category'] == s]\n",
    "    subgraph_dict[s] = len(subset.subgraph.unique())\n",
    "\n",
    "subgraph_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a71ea3d5-ee85-4f56-8e91-a57bb5581436",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "bgc_dict = df_antismash_annotated.subgraph_category.value_counts().to_dict()\n",
    "bgc_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d01c2c02-e429-4136-b0a2-ddc0498fdac2",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "text = f\"\"\"\\\n",
    "A. Example of BGC similarity network using BiG-SCAPE {cutoff} cutoff.\n",
    "B. Example of the same BGCs with additional connection made using knownclusterblast predictions, BiG-SLICE query, and ARTS against the databases of MIBIG, BiGFAM-DB, and ARTS-DB, respectively.\n",
    "C. Overview of the similarity network of all {total_bgcs} BGCs resulting in {len(family_distribution)} connected components with {bgc_distribution['known_family']} BGCs assigned to \\\n",
    "{len([k for k in family_distribution.keys() if not k.startswith(\"u_\")])} known MIBIG entries and {len([(k, v) for k, v in family_distribution.items() if v == 1 and k.startswith(\"u_\")])} singletons.\n",
    "D. Expanded network with information from antiSMASH KnownClusterBlast and BiG-SLICE similarity to BiG-FAM models resulting in {len(subgraph)} connected components with {bgc_dict['known_cluster_bigscape']} BGCs assigned to \\\n",
    "{subgraph_dict['known_cluster_bigscape']} MIBIG entries, \\\n",
    "{bgc_dict['known_cluster_80%']} BGCs assigned to {subgraph_dict['known_cluster_80%']} MIBIG entries with >= 80% similarity, and {bgc_dict['unknown_cluster_singleton']} singletons.\n",
    "BiG-FAM models {deleted_bigfam} were removed because the assigned top genus in the model are below {bigfam_taxa_cutoff:.0%}\\\n",
    "\"\"\"\n",
    "print(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c363ef1d-5571-4b06-8ad3-c5e404afd717",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "df_antismash_annotated[df_antismash_annotated['subgraph_category'] == \"known_cluster_bigscape\"].most_similar_known_cluster_description.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "534c4dde-9138-427b-84c9-078fd3d3ca6a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "df_antismash_annotated[df_antismash_annotated['subgraph_category'] == \"known_cluster_80%\"].most_similar_known_cluster_description.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b4f452e3-a436-4f42-b3ed-4ab492b20357",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "df_antismash_annotated[df_antismash_annotated['subgraph_category'] == 'unknown_cluster_40%_80%'].most_similar_known_cluster_description.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c88c1f46-2d47-42a9-926b-a819cb8bb827",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "df_antismash_annotated[df_antismash_annotated['subgraph_category'] == 'unknown_cluster_40%'].most_similar_known_cluster_description.unique()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}