Merge pull request #15 from george0st/transaction

UC101 finish

main.py

@@ -3,18 +3,21 @@
 from qgate.uc.uc101 import UC101
 from qgate.uc import ucsetup, ucoutput
 
-
-if __name__ == '__main__':
-
+def test():
     setup= ucsetup.UCSetup("0-size-100",
                          ["qgate-sln-mlrun-private.env", "qgate-sln-mlrun.env"])
 
     ucoutput.UCOutput(setup)
     aa=UC101(setup)
+    aa.exec()
 
-    # sln = qgate.Solution("0-size-100",
-    #                      ["qgate-sln-mlrun-private.env", "qgate-sln-mlrun.env"])
-    # try:
-    #     sln.create(force=True)
-    # finally:
-    #     sln.delete()
+if __name__ == '__main__':
+
+#    test()
+
+    sln = qgate.Solution("0-size-100",
+                         ["qgate-sln-mlrun-private.env", "qgate-sln-mlrun.env"])
+    try:
+        sln.create(force=True)
+    finally:
+        sln.delete()

qgate/__init__.py

@@ -1 +1,2 @@
-# info
+# info
+from .version import __version__

qgate/uc/__init__.py

@@ -1 +1,2 @@
-# info
+# info
+from ..version import __version__

qgate/uc/uc101.py

@@ -22,7 +22,7 @@ def desc(self):
     def exec(self):
 
         # create projects
-        dir=os.path.join(os.getcwd(), self._model_definition, "01-model", "01-project", "*.json")
+        dir=os.path.join(os.getcwd(), self.setup.model_definition, "01-model", "01-project", "*.json")
         for file in glob.glob(dir):
             with open(file, "r") as json_file:
                 json_content = json.load(json_file)

qgate/uc/ucbase.py

@@ -22,10 +22,21 @@ def __init__(self, setup: UCSetup, name):
         self._setup=setup
         self._name=name
 
+    @property
+    def setup(self):
+        return self._setup
+
     @property
     def desc(self):
         raise NotImplemented()
 
+    @property
+    def name(self):
+        return self._name
+
+    def exec(self):
+        raise NotImplemented()
+
     def _get_json_header(self, json_content):
         """ Get common header from config files
 
@@ -40,10 +51,6 @@ def _get_json_header(self, json_content):
         lbls = None if json_content.get('labels') is None else json_content.get('labels')
         return name, desc, lbls, kind
 
-    @property
-    def name(self):
-        return self._name
-
     @staticmethod
     def str2bool(v):
         return v.lower() in ("yes", "true", "t", "1")

qgate/uc/ucoutput.py

@@ -1,7 +1,10 @@
 import datetime
+import multiprocessing
 import os, platform, sys
 import mlrun
 from qgate.uc.ucsetup import UCSetup
+from contextlib import suppress
+from qgate.version import __version__
 
 class Singleton (type):
     _instances = {}
@@ -26,24 +29,53 @@ def __init__(self, setup: UCSetup):
         self._file = open(os.path.join(self._setup.model_output, "qgate-sln-mlrun.txt"), 'w+t')
 
         self._headr()
+        self._footer()
 
     def __del__(self):
+        self._footer()
         self._file.close()
 
     def _headr(self):
-        self._print(datetime.datetime.now().isoformat())
-        self._print()
-
-        self._print("MLRun:    " + mlrun.get_version() + " (https://docs.mlrun.org/en/latest/change-log/index.html)")
-        self._print("Python:   " + sys.version)
-        self._print("System:   " + platform.system() + " " + platform.version() + " (" + platform.platform() + ")")
+        self._print("QGate version: " + __version__)
+        self._print(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
+
+    def _footer(self):
+        total, free = self._memory()
+        self._print("Host: " + self._host())
+        self._print("RAM total/free: " + total + "/" + free)
+        self._print("CPU: " + str(multiprocessing.cpu_count()))
+        self._print("-----------------------")
+        self._print("MLRun: " + mlrun.get_version() + " (https://docs.mlrun.org/en/latest/change-log/index.html)")
+        self._print("Python: " + sys.version)
+        self._print("System: " + platform.system() + " " + platform.version() + " (" + platform.platform() + ")")
         self._print("Platform: " + platform.machine()+ " (" + platform.processor() + ")")
+        self._print("-----------------------")
 
-        self._print()
-
-        self._print("DIR = " + os.getcwd())
+        self._print("DIR: '" + os.getcwd() + "'")
         self._print(str(self._setup).replace('\n',"\n" + UCOutput.COMMENT))
 
+    def _memory(self):
+
+        mem_total, mem_free = "", ""
+        with suppress(Exception):
+            import psutil
+
+            values = psutil.virtual_memory()
+            mem_total = f"{round(values.total / (1073741824), 1)} GB"
+            mem_free = f"{round(values.free / (1073741824), 1)} GB"
+        return mem_total, mem_free
+
+    def _host(self):
+        """ Return information about the host in format (host_name/ip addr)"""
+
+        host = ""
+        with suppress(Exception):
+            import socket
+
+            host_name = socket.gethostname()
+            ip = socket.gethostbyname(host_name)
+            host = f"{host_name}/{ip}"
+        return host
 
     def print(self, uc_name, *args, **kwargs):
         self._print(uc_name + str.format(args, kwargs), False)

qgate/uc/ucsetup.py

@@ -25,9 +25,13 @@ def __init__(self, data_size, mlrun_env_file: list[str]):
     def __str__(self):
         ret=""
         for key in self._variables.keys():
-            ret+=key+ " = "+ "'" + self._variables[key] + "'\n"
-        return ret
+            ret+=key+ ": "+ "'" + self._variables[key] + "'\n"
+        return ret[:-1]
 
     @property
     def model_output(self):
-        return self._model_output
+        return self._model_output
+
+    @property
+    def model_definition(self):
+        return self._model_definition

qgate/version.py

@@ -0,0 +1,3 @@
+# Store the version here so:
+
+__version__ = '0.0.4'