diff --git a/proteobench/github/gh.py b/proteobench/github/gh.py index e79bc43f..3ae613a2 100644 --- a/proteobench/github/gh.py +++ b/proteobench/github/gh.py @@ -1,70 +1,30 @@ import os from tempfile import TemporaryDirectory +import pandas as pd from git import Repo -from proteobench.modules.dda_quant.module import Module -def write_json_local_development( - temporary_datapoints -): - t_dir = TemporaryDirectory().name - os.mkdir(t_dir) - - current_datapoint = temporary_datapoints.iloc[-1] - current_datapoint["is_temporary"] = False - all_datapoints = Module().add_current_data_point(None, current_datapoint) - - # TODO write below to logger instead of std.out - fname = os.path.join(t_dir, "results.json") - print(f"Writing the json to: {fname}") - - f = open(os.path.join(t_dir, "results.json"), "w") - - all_datapoints.to_json( - f, - orient="records", - indent=2 - ) +def clone_repo_anon( + clone_dir="K:/pb/", + remote_git="https://github.com/Proteobench/Results_Module2_quant_DDA.git", +): + repo = Repo.clone_from(remote_git, clone_dir) + return clone_dir - return os.path.join(t_dir, "results.json") -def clone_pr( - temporary_datapoints, - token, - username="Proteobot", - remote_git="github.com/Proteobot/Results_Module2_quant_DDA.git", - branch_name="new_branch", +def read_results_json_repo( + remote_git_repo= "https://github.com/Proteobench/Results_Module2_quant_DDA.git" ): t_dir = TemporaryDirectory().name - - clone_repo(clone_dir=t_dir, token=token, remote_git=remote_git, username=username) - current_datapoint = temporary_datapoints.iloc[-1] - current_datapoint["is_temporary"] = False - all_datapoints = Module().add_current_data_point(None, current_datapoint) - branch_name = current_datapoint["id"] - - # do the pd.write_json() here!!! - print(os.path.join(t_dir, "results.json")) - f = open(os.path.join(t_dir, "results.json"), "w") - - all_datapoints.to_json( - f, - orient="records", - indent=2 - ) - - f.close() - commit_message = "Added new run with id " + branch_name - - pr_github( - clone_dir=t_dir, - token=token, - remote_git=remote_git, - username=username, - branch_name=branch_name, - commit_message=commit_message, + os.mkdir(t_dir) + clone_repo_anon( + t_dir, + remote_git_repo ) + fname = os.path.join(t_dir, "results.json") + all_datapoints = pd.read_json(fname) + return all_datapoints def clone_repo( diff --git a/proteobench/modules/dda_quant/module.py b/proteobench/modules/dda_quant/module.py index 938d0df0..12fa9170 100644 --- a/proteobench/modules/dda_quant/module.py +++ b/proteobench/modules/dda_quant/module.py @@ -2,17 +2,20 @@ import datetime import itertools +import os import re from dataclasses import asdict +from tempfile import TemporaryDirectory +import numpy as np import pandas as pd import streamlit as st -import numpy as np +from proteobench.github.gh import clone_repo, pr_github, read_results_json_repo from proteobench.modules.dda_quant.datapoint import Datapoint from proteobench.modules.dda_quant.parse import ParseInputs from proteobench.modules.dda_quant.parse_settings import ( - DDA_QUANT_RESULTS_PATH, ParseSettings) + DDA_QUANT_RESULTS_REPO, ParseSettings) from proteobench.modules.interfaces import ModuleInterface @@ -168,7 +171,8 @@ def load_input_file(self, input_csv: str, input_format: str) -> pd.DataFrame: def add_current_data_point(self, all_datapoints, current_datapoint): """Add current data point to all data points and load them from file if empty. TODO: Not clear why is the df transposed here.""" if not isinstance(all_datapoints, pd.DataFrame): - all_datapoints = pd.read_json(DDA_QUANT_RESULTS_PATH) + #all_datapoints = pd.read_json(DDA_QUANT_RESULTS_PATH) + all_datapoints = read_results_json_repo(DDA_QUANT_RESULTS_REPO) all_datapoints = all_datapoints.T all_datapoints = pd.concat([all_datapoints, current_datapoint], axis=1) all_datapoints = all_datapoints.T.reset_index(drop=True) @@ -200,3 +204,68 @@ def benchmarking( # TODO check why there are NA and inf/-inf values return intermediate_data_structure.fillna(0.0).replace([np.inf, -np.inf], 0), all_datapoints + + + def clone_pr( + self, + temporary_datapoints, + token, + username="Proteobot", + remote_git="github.com/Proteobot/Results_Module2_quant_DDA.git", + branch_name="new_branch", + ): + t_dir = TemporaryDirectory().name + + clone_repo(clone_dir=t_dir, token=token, remote_git=remote_git, username=username) + current_datapoint = temporary_datapoints.iloc[-1] + current_datapoint["is_temporary"] = False + all_datapoints = self.add_current_data_point(None, current_datapoint) + branch_name = current_datapoint["id"] + + # do the pd.write_json() here!!! + print(os.path.join(t_dir, "results.json")) + f = open(os.path.join(t_dir, "results.json"), "w") + + all_datapoints.to_json( + f, + orient="records", + indent=2 + ) + + f.close() + commit_message = "Added new run with id " + branch_name + + pr_github( + clone_dir=t_dir, + token=token, + remote_git=remote_git, + username=username, + branch_name=branch_name, + commit_message=commit_message, + ) + + + def write_json_local_development( + self, + temporary_datapoints + ): + t_dir = TemporaryDirectory().name + os.mkdir(t_dir) + + current_datapoint = temporary_datapoints.iloc[-1] + current_datapoint["is_temporary"] = False + all_datapoints = self.add_current_data_point(None, current_datapoint) + + # TODO write below to logger instead of std.out + fname = os.path.join(t_dir, "results.json") + print(f"Writing the json to: {fname}") + + f = open(os.path.join(t_dir, "results.json"), "w") + + all_datapoints.to_json( + f, + orient="records", + indent=2 + ) + + return os.path.join(t_dir, "results.json") diff --git a/proteobench/modules/dda_quant/parse_settings.py b/proteobench/modules/dda_quant/parse_settings.py index 7dd7154d..dfa37ab5 100644 --- a/proteobench/modules/dda_quant/parse_settings.py +++ b/proteobench/modules/dda_quant/parse_settings.py @@ -35,6 +35,8 @@ # For local development change below to the json and path, if you do not want to download it from github DDA_QUANT_RESULTS_PATH = "https://raw.githubusercontent.com/Proteobench/Results_Module2_quant_DDA/main/results.json" #e.g., K:/results.json +DDA_QUANT_RESULTS_REPO = "https://github.com/Proteobench/Results_Module2_quant_DDA.git" + class ParseSettings: """ Structure that contains all the parameters used to parse the given database search output. """ diff --git a/test/test_module_dda_quant.py b/test/test_module_dda_quant.py index c5b5d26a..ac6bda61 100644 --- a/test/test_module_dda_quant.py +++ b/test/test_module_dda_quant.py @@ -3,10 +3,11 @@ import pandas as pd +from proteobench.github.gh import read_results_json_repo from proteobench.modules.dda_quant.module import Module from proteobench.modules.dda_quant.parse import ParseInputs from proteobench.modules.dda_quant.parse_settings import ( - DDA_QUANT_RESULTS_PATH, INPUT_FORMATS, ParseSettings) + DDA_QUANT_RESULTS_REPO, INPUT_FORMATS, ParseSettings) from proteobench.modules.dda_quant.plot import PlotDataPoint # genereate_input_field @@ -126,7 +127,8 @@ class TestPlot(unittest.TestCase): def test_plot_metric(self): - all_datapoints = pd.read_json(DDA_QUANT_RESULTS_PATH) + #all_datapoints = pd.read_json(DDA_QUANT_RESULTS_PATH) + all_datapoints = read_results_json_repo(DDA_QUANT_RESULTS_REPO) fig = PlotDataPoint().plot_metric(all_datapoints) self.assertIsNotNone(fig) diff --git a/webinterface/pages/DDA_Quant.py b/webinterface/pages/DDA_Quant.py index 44a37dba..7f652bd4 100644 --- a/webinterface/pages/DDA_Quant.py +++ b/webinterface/pages/DDA_Quant.py @@ -6,10 +6,7 @@ from proteobench.modules.dda_quant.module import Module from proteobench.modules.dda_quant.parse_settings import ( - DDA_QUANT_RESULTS_PATH, - INPUT_FORMATS, - LOCAL_DEVELOPMENT, -) + DDA_QUANT_RESULTS_PATH, INPUT_FORMATS, LOCAL_DEVELOPMENT) from proteobench.modules.dda_quant.plot import PlotDataPoint try: @@ -21,7 +18,7 @@ import streamlit_utils from streamlit_extras.let_it_rain import rain -from proteobench.github.gh import clone_pr, write_json_local_development +#from proteobench.github.gh import clone_pr, write_json_local_development logger = logging.getLogger(__name__) @@ -259,7 +256,7 @@ def generate_results( if submit_pr: st.session_state[SUBMIT] = True if not LOCAL_DEVELOPMENT: - clone_pr( + Module().clone_pr( st.session_state[ALL_DATAPOINTS], st.secrets["gh"]["token"], username="Proteobot", @@ -267,7 +264,7 @@ def generate_results( branch_name="new_branch", ) else: - DDA_QUANT_RESULTS_PATH = write_json_local_development( + DDA_QUANT_RESULTS_PATH = Module().write_json_local_development( st.session_state[ALL_DATAPOINTS] ) if SUBMIT in st.session_state: diff --git a/webinterface/pages/TEMPLATE.py b/webinterface/pages/TEMPLATE.py index d17b94bb..9fa71861 100644 --- a/webinterface/pages/TEMPLATE.py +++ b/webinterface/pages/TEMPLATE.py @@ -19,8 +19,6 @@ import streamlit_utils from streamlit_extras.let_it_rain import rain -from proteobench.github.gh import clone_pr, write_json_local_development - logger = logging.getLogger(__name__) ## Different parts of the web application @@ -225,7 +223,7 @@ def generate_results( if submit_pr: st.session_state[SUBMIT] = True if not LOCAL_DEVELOPMENT: - clone_pr( + Module().clone_pr( st.session_state[ALL_DATAPOINTS], st.secrets["gh"]["token"], username="Proteobot", @@ -233,7 +231,7 @@ def generate_results( branch_name="new_branch", ) else: - DDA_QUANT_RESULTS_PATH = write_json_local_development( + DDA_QUANT_RESULTS_PATH = Module().write_json_local_development( st.session_state[ALL_DATAPOINTS] ) if SUBMIT in st.session_state: