Fix issue clone repo results json

proteobench/github/gh.py

@@ -1,70 +1,30 @@
 import os
 from tempfile import TemporaryDirectory
 
+import pandas as pd
 from git import Repo
 
-from proteobench.modules.dda_quant.module import Module
 
-def write_json_local_development(
-    temporary_datapoints
-):  
-    t_dir = TemporaryDirectory().name
-    os.mkdir(t_dir)
-
-    current_datapoint = temporary_datapoints.iloc[-1]
-    current_datapoint["is_temporary"] = False
-    all_datapoints = Module().add_current_data_point(None, current_datapoint)
-
-    # TODO write below to logger instead of std.out
-    fname = os.path.join(t_dir, "results.json")
-    print(f"Writing the json to: {fname}")
-
-    f = open(os.path.join(t_dir, "results.json"), "w")
-
-    all_datapoints.to_json(
-        f,
-        orient="records",
-        indent=2
-    )
+def clone_repo_anon(
+    clone_dir="K:/pb/",
+    remote_git="https://github.com/Proteobench/Results_Module2_quant_DDA.git",
+):
+    repo = Repo.clone_from(remote_git, clone_dir)
+    return clone_dir
 
-    return os.path.join(t_dir, "results.json")
 
-def clone_pr(
-    temporary_datapoints,
-    token,
-    username="Proteobot",
-    remote_git="github.com/Proteobot/Results_Module2_quant_DDA.git",
-    branch_name="new_branch",
+def read_results_json_repo(
+    remote_git_repo= "https://github.com/Proteobench/Results_Module2_quant_DDA.git"
 ):
     t_dir = TemporaryDirectory().name
-
-    clone_repo(clone_dir=t_dir, token=token, remote_git=remote_git, username=username)
-    current_datapoint = temporary_datapoints.iloc[-1]
-    current_datapoint["is_temporary"] = False
-    all_datapoints = Module().add_current_data_point(None, current_datapoint)
-    branch_name = current_datapoint["id"]
-
-    # do the pd.write_json() here!!!
-    print(os.path.join(t_dir, "results.json"))
-    f = open(os.path.join(t_dir, "results.json"), "w")
-
-    all_datapoints.to_json(
-        f,
-        orient="records",
-        indent=2
-    )
-
-    f.close()
-    commit_message = "Added new run with id " + branch_name
-
-    pr_github(
-        clone_dir=t_dir,
-        token=token,
-        remote_git=remote_git,
-        username=username,
-        branch_name=branch_name,
-        commit_message=commit_message,
+    os.mkdir(t_dir)
+    clone_repo_anon(
+        t_dir,
+        remote_git_repo
     )
+    fname = os.path.join(t_dir, "results.json")
+    all_datapoints = pd.read_json(fname)
+    return all_datapoints
 
 
 def clone_repo(

proteobench/modules/dda_quant/module.py

@@ -2,17 +2,20 @@
 
 import datetime
 import itertools
+import os
 import re
 from dataclasses import asdict
+from tempfile import TemporaryDirectory
 
+import numpy as np
 import pandas as pd
 import streamlit as st
-import numpy as np
 
+from proteobench.github.gh import clone_repo, pr_github, read_results_json_repo
 from proteobench.modules.dda_quant.datapoint import Datapoint
 from proteobench.modules.dda_quant.parse import ParseInputs
 from proteobench.modules.dda_quant.parse_settings import (
-    DDA_QUANT_RESULTS_PATH, ParseSettings)
+    DDA_QUANT_RESULTS_REPO, ParseSettings)
 from proteobench.modules.interfaces import ModuleInterface
 
 
@@ -168,7 +171,8 @@ def load_input_file(self, input_csv: str, input_format: str) -> pd.DataFrame:
     def add_current_data_point(self, all_datapoints, current_datapoint):
         """Add current data point to all data points and load them from file if empty. TODO: Not clear why is the df transposed here."""
         if not isinstance(all_datapoints, pd.DataFrame):
-            all_datapoints = pd.read_json(DDA_QUANT_RESULTS_PATH)
+            #all_datapoints = pd.read_json(DDA_QUANT_RESULTS_PATH)
+            all_datapoints = read_results_json_repo(DDA_QUANT_RESULTS_REPO)
         all_datapoints = all_datapoints.T
         all_datapoints = pd.concat([all_datapoints, current_datapoint], axis=1)
         all_datapoints = all_datapoints.T.reset_index(drop=True)
@@ -200,3 +204,68 @@ def benchmarking(
 
         # TODO check why there are NA and inf/-inf values
         return intermediate_data_structure.fillna(0.0).replace([np.inf, -np.inf], 0), all_datapoints
+
+
+    def clone_pr(
+        self,
+        temporary_datapoints,
+        token,
+        username="Proteobot",
+        remote_git="github.com/Proteobot/Results_Module2_quant_DDA.git",
+        branch_name="new_branch",
+    ):
+        t_dir = TemporaryDirectory().name
+
+        clone_repo(clone_dir=t_dir, token=token, remote_git=remote_git, username=username)
+        current_datapoint = temporary_datapoints.iloc[-1]
+        current_datapoint["is_temporary"] = False
+        all_datapoints = self.add_current_data_point(None, current_datapoint)
+        branch_name = current_datapoint["id"]
+
+        # do the pd.write_json() here!!!
+        print(os.path.join(t_dir, "results.json"))
+        f = open(os.path.join(t_dir, "results.json"), "w")
+
+        all_datapoints.to_json(
+            f,
+            orient="records",
+            indent=2
+        )
+
+        f.close()
+        commit_message = "Added new run with id " + branch_name
+
+        pr_github(
+            clone_dir=t_dir,
+            token=token,
+            remote_git=remote_git,
+            username=username,
+            branch_name=branch_name,
+            commit_message=commit_message,
+        )
+
+
+    def write_json_local_development(
+        self, 
+        temporary_datapoints
+    ):  
+        t_dir = TemporaryDirectory().name
+        os.mkdir(t_dir)
+
+        current_datapoint = temporary_datapoints.iloc[-1]
+        current_datapoint["is_temporary"] = False
+        all_datapoints = self.add_current_data_point(None, current_datapoint)
+
+        # TODO write below to logger instead of std.out
+        fname = os.path.join(t_dir, "results.json")
+        print(f"Writing the json to: {fname}")
+
+        f = open(os.path.join(t_dir, "results.json"), "w")
+
+        all_datapoints.to_json(
+            f,
+            orient="records",
+            indent=2
+        )
+
+        return os.path.join(t_dir, "results.json")

proteobench/modules/dda_quant/parse_settings.py

@@ -35,6 +35,8 @@
 # For local development change below to the json and path, if you do not want to download it from github
 DDA_QUANT_RESULTS_PATH = "https://raw.githubusercontent.com/Proteobench/Results_Module2_quant_DDA/main/results.json" #e.g., K:/results.json
 
+DDA_QUANT_RESULTS_REPO = "https://github.com/Proteobench/Results_Module2_quant_DDA.git"
+
 class ParseSettings:
     """ Structure that contains all the parameters used to parse the given database search output. """
 

test/test_module_dda_quant.py

@@ -3,10 +3,11 @@
 
 import pandas as pd
 
+from proteobench.github.gh import read_results_json_repo
 from proteobench.modules.dda_quant.module import Module
 from proteobench.modules.dda_quant.parse import ParseInputs
 from proteobench.modules.dda_quant.parse_settings import (
-    DDA_QUANT_RESULTS_PATH, INPUT_FORMATS, ParseSettings)
+    DDA_QUANT_RESULTS_REPO, INPUT_FORMATS, ParseSettings)
 from proteobench.modules.dda_quant.plot import PlotDataPoint
 
 # genereate_input_field
@@ -126,7 +127,8 @@ class TestPlot(unittest.TestCase):
 
     def test_plot_metric(self):
 
-        all_datapoints = pd.read_json(DDA_QUANT_RESULTS_PATH)
+        #all_datapoints = pd.read_json(DDA_QUANT_RESULTS_PATH)
+        all_datapoints = read_results_json_repo(DDA_QUANT_RESULTS_REPO)
 
         fig = PlotDataPoint().plot_metric(all_datapoints)
         self.assertIsNotNone(fig)

webinterface/pages/DDA_Quant.py

@@ -6,10 +6,7 @@
 
 from proteobench.modules.dda_quant.module import Module
 from proteobench.modules.dda_quant.parse_settings import (
-    DDA_QUANT_RESULTS_PATH,
-    INPUT_FORMATS,
-    LOCAL_DEVELOPMENT,
-)
+    DDA_QUANT_RESULTS_PATH, INPUT_FORMATS, LOCAL_DEVELOPMENT)
 from proteobench.modules.dda_quant.plot import PlotDataPoint
 
 try:
@@ -21,7 +18,7 @@
 import streamlit_utils
 from streamlit_extras.let_it_rain import rain
 
-from proteobench.github.gh import clone_pr, write_json_local_development
+#from proteobench.github.gh import clone_pr, write_json_local_development
 
 logger = logging.getLogger(__name__)
 
@@ -259,15 +256,15 @@ def generate_results(
             if submit_pr:
                 st.session_state[SUBMIT] = True
                 if not LOCAL_DEVELOPMENT:
-                    clone_pr(
+                    Module().clone_pr(
                         st.session_state[ALL_DATAPOINTS],
                         st.secrets["gh"]["token"],
                         username="Proteobot",
                         remote_git="github.com/Proteobot/Results_Module2_quant_DDA.git",
                         branch_name="new_branch",
                     )
                 else:
-                    DDA_QUANT_RESULTS_PATH = write_json_local_development(
+                    DDA_QUANT_RESULTS_PATH = Module().write_json_local_development(
                         st.session_state[ALL_DATAPOINTS]
                     )
         if SUBMIT in st.session_state:

webinterface/pages/TEMPLATE.py

@@ -19,8 +19,6 @@
 import streamlit_utils
 from streamlit_extras.let_it_rain import rain
 
-from proteobench.github.gh import clone_pr, write_json_local_development
-
 logger = logging.getLogger(__name__)
 
 ## Different parts of the web application
@@ -225,15 +223,15 @@ def generate_results(
             if submit_pr:
                 st.session_state[SUBMIT] = True
                 if not LOCAL_DEVELOPMENT:
-                    clone_pr(
+                    Module().clone_pr(
                         st.session_state[ALL_DATAPOINTS],
                         st.secrets["gh"]["token"],
                         username="Proteobot",
                         remote_git="github.com/Proteobot/Results_Module2_quant_DDA.git",
                         branch_name="new_branch",
                     )
                 else:
-                    DDA_QUANT_RESULTS_PATH = write_json_local_development(
+                    DDA_QUANT_RESULTS_PATH = Module().write_json_local_development(
                         st.session_state[ALL_DATAPOINTS]
                     )
         if SUBMIT in st.session_state: