change CExtension modules imports;

cleaning

mimircache/CExtension/pyBindings/pyHeatmap.c

@@ -788,7 +788,7 @@ static PyMethodDef c_heatmap_funcs[] = {
         METH_VARARGS | METH_KEYWORDS, "heatmap pixel computation"},
     {"differential_heatmap_with_Optimal", (PyCFunction)differential_heatmap_with_Optimal,
         METH_VARARGS | METH_KEYWORDS, "differential heatmap pixel computation compared with Optimal"},
-    {"diffHeatmap", (PyCFunction)differential_heatmap_py,
+    {"diff_heatmap", (PyCFunction)differential_heatmap_py,
         METH_VARARGS | METH_KEYWORDS, "differential heatmap pixel computation"},
     {"heatmap_rd_distribution", (PyCFunction)heatmap_rd_distribution_py,
         METH_VARARGS | METH_KEYWORDS, "reuse distance distribution heatmap"},
@@ -798,7 +798,7 @@ static PyMethodDef c_heatmap_funcs[] = {
         METH_VARARGS | METH_KEYWORDS, "reuse distance distribution heatmap"},
     {"heatmap_reuse_time_distribution", (PyCFunction)heatmap_rt_distribution_py,
         METH_VARARGS | METH_KEYWORDS, "reuse distance distribution heatmap"},
-    {"getBreakpoints", (PyCFunction)heatmap_get_break_points,
+    {"get_breakpoints", (PyCFunction)heatmap_get_break_points,
         METH_VARARGS | METH_KEYWORDS, "generate virtual/real break points"},
     {NULL, NULL, 0, NULL}
 };

mimircache/__init__.py

@@ -18,13 +18,13 @@
 sys.path.extend([CWD, os.path.join(CWD, "..")])
 
 
+from mimircache.const import *
 from mimircache.profiler.LRUProfiler import LRUProfiler as LRUProfiler
 from mimircache.profiler.cGeneralProfiler import cGeneralProfiler
 from mimircache.profiler.generalProfiler import generalProfiler as generalProfiler
 from mimircache.profiler.cHeatmap import cHeatmap
 from mimircache.profiler.heatmap import heatmap as heatmap
 from mimircache.top.cachecow import cachecow as cachecow
-from mimircache.const import *
 
 from version import __version__ as __version__
 

mimircache/cache/Optimal.py

@@ -4,8 +4,8 @@
 from mimircache.cache.abstractCache import cache
 from mimircache.const import CExtensionMode
 if CExtensionMode:
-    import mimircache.c_LRUProfiler as c_LRUProfiler
-    import mimircache.c_heatmap as c_heatmap
+    import mimircache.c_LRUProfiler
+    import mimircache.c_heatmap
 from heapdict import heapdict
 
 
@@ -15,15 +15,15 @@ def __init__(self, cache_size, reader):
         # reader.reset()
         self.reader = reader
         self.reader.lock.acquire()
-        self.next_access = c_heatmap.get_next_access_dist(self.reader.cReader)
+        self.next_access = mimircache.c_heatmap.get_next_access_dist(self.reader.cReader)
         self.reader.lock.release()
         self.pq = heapdict()
 
 
         self.ts = 0
 
     def get_reversed_reuse_dist(self):
-        return c_LRUProfiler.get_reversed_reuse_dist(self.reader.cReader)
+        return mimircache.c_LRUProfiler.get_reversed_reuse_dist(self.reader.cReader)
 
     def checkElement(self, element):
         """

mimircache/cacheReader/abstractReader.py

@@ -3,7 +3,9 @@
 import os
 from multiprocessing import Lock
 from collections import defaultdict
-import mimircache.c_cacheReader as c_cacheReader
+from mimircache.const import CExtensionMode
+if CExtensionMode:
+    import mimircache.c_cacheReader
 
 
 class cacheReaderAbstract(metaclass=abc.ABCMeta):
@@ -44,7 +46,7 @@ def reset(self):
         self.counter = 0
         self.trace_file.seek(0, 0)
         if self.cReader:
-            c_cacheReader.reset_reader(self.cReader)
+            mimircache.c_cacheReader.reset_reader(self.cReader)
 
     def get_num_of_req(self):
         """
@@ -58,7 +60,7 @@ def get_num_of_req(self):
         # clear before counting
         self.num_of_req = 0
         if self.cReader:
-            self.num_of_req = c_cacheReader.get_num_of_req(self.cReader)
+            self.num_of_req = mimircache.c_cacheReader.get_num_of_req(self.cReader)
         else:
             while self.read_one_element() is not None:
                 self.num_of_req += 1
@@ -119,12 +121,12 @@ def close(self):
                 if self.trace_file:
                     self.trace_file.close()
                     self.trace_file = None
-                if self.cReader and c_cacheReader is not None:
-                    c_cacheReader.close_reader(self.cReader)
+                if self.cReader and mimircache.c_cacheReader is not None:
+                    mimircache.c_cacheReader.close_reader(self.cReader)
                     self.cReader = None
         except Exception as e:
             # return
-            print("Exception during close reader: {}, ccacheReader={}".format(e, c_cacheReader))
+            print("Exception during close reader: {}, ccacheReader={}".format(e, mimircache.c_cacheReader))
 
     @abc.abstractmethod
     def __next__(self):  # Python 3

mimircache/cacheReader/binaryReader.py

@@ -5,7 +5,9 @@
 """
 
 import io, os, struct
-import mimircache.c_cacheReader as c_cacheReader
+from mimircache.const import CExtensionMode
+if CExtensionMode:
+    import mimircache.c_cacheReader
 from mimircache.cacheReader.abstractReader import cacheReaderAbstract
 
 
@@ -44,7 +46,7 @@ def __init__(self, file_loc, init_params, data_type='c',
 
         if open_c_reader:
             # the data type here is not real data type, it will auto correct in C
-            self.cReader = c_cacheReader.setup_reader(file_loc, 'b', data_type=self.data_type,
+            self.cReader = mimircache.c_cacheReader.setup_reader(file_loc, 'b', data_type=self.data_type,
                                                       block_unit_size=block_unit_size,
                                                       disk_sector_size=disk_sector_size, 
                                                       init_params=init_params)

mimircache/cacheReader/csvReader.py

@@ -2,7 +2,7 @@
 import string
 from mimircache.const import CExtensionMode
 if CExtensionMode:
-    import mimircache.c_cacheReader as c_cacheReader
+    import mimircache.c_cacheReader
 from mimircache.cacheReader.abstractReader import cacheReaderAbstract
 
 
@@ -47,7 +47,7 @@ def __init__(self, file_loc, data_type='c', init_params=None,
             self.read_one_element()
 
         if open_c_reader:
-            self.cReader = c_cacheReader.setup_reader(file_loc, 'c', data_type=data_type,
+            self.cReader = mimircache.c_cacheReader.setup_reader(file_loc, 'c', data_type=data_type,
                                                       block_unit_size=block_unit_size,
                                                       disk_sector_size=disk_sector_size, 
                                                       init_params=init_params)

mimircache/cacheReader/multiReader.py

@@ -1,8 +1,6 @@
 # coding=utf-8
 
 
-# from mimircache import
-from heapdict import heapdict
 import heapq
 
 

mimircache/cacheReader/plainReader.py

@@ -1,6 +1,8 @@
 # coding=utf-8
 from mimircache.cacheReader.abstractReader import cacheReaderAbstract
-import mimircache.c_cacheReader as c_cacheReader
+from mimircache.const import CExtensionMode
+if CExtensionMode:
+    import mimircache.c_cacheReader
 
 
 class plainReader(cacheReaderAbstract):
@@ -13,7 +15,7 @@ def __init__(self, file_loc, data_type='c', open_c_reader=True):
         super(plainReader, self).__init__(file_loc, data_type, 0, 0)
         self.trace_file = open(file_loc, 'r')
         if open_c_reader:
-            self.cReader = c_cacheReader.setup_reader(file_loc, 'p', data_type=data_type, block_unit_size=0)
+            self.cReader = mimircache.c_cacheReader.setup_reader(file_loc, 'p', data_type=data_type, block_unit_size=0)
 
     def read_one_element(self):
         """

mimircache/cacheReader/vscsiReader.py

@@ -2,7 +2,9 @@
 
 
 from mimircache.cacheReader.abstractReader import cacheReaderAbstract
-import mimircache.c_cacheReader as c_cacheReader
+from mimircache.const import CExtensionMode
+if CExtensionMode:
+    import mimircache.c_cacheReader
 
 
 class vscsiReader(cacheReaderAbstract):
@@ -25,7 +27,7 @@ def __init__(self, file_loc, data_type='l',
         super().__init__(file_loc, data_type='l', block_unit_size=block_unit_size,
                          disk_sector_size=disk_sector_size)
         if open_c_reader:
-            self.cReader = c_cacheReader.setup_reader(file_loc, 'v', data_type=data_type,
+            self.cReader = mimircache.c_cacheReader.setup_reader(file_loc, 'v', data_type=data_type,
                                                       block_unit_size=block_unit_size,
                                                       disk_sector_size=disk_sector_size)
         self.support_size = True
@@ -36,14 +38,14 @@ def __init__(self, file_loc, data_type='l',
 
     def reset(self):
         if self.cReader:
-            c_cacheReader.reset_reader(self.cReader)
+            mimircache.c_cacheReader.reset_reader(self.cReader)
 
     def read_one_element(self):
         """
         read one request, return only block number
         :return:
         """
-        r = c_cacheReader.read_one_element(self.cReader)
+        r = mimircache.c_cacheReader.read_one_element(self.cReader)
         if r and self.block_unit_size != 0 and self.disk_sector_size != 0:
             r = r * self.disk_sector_size // self.block_unit_size
         return r
@@ -53,7 +55,7 @@ def read_time_request(self):
         return real_time information for the request in the form of (time, request)
         :return:
         """
-        r = c_cacheReader.read_time_request(self.cReader)
+        r = mimircache.c_cacheReader.read_time_request(self.cReader)
         if r and self.block_unit_size != 0 and self.disk_sector_size != 0:
             r[1] = r[1] * self.disk_sector_size // self.block_unit_size
         return r
@@ -63,7 +65,7 @@ def read_one_request_full_info(self):
         obtain more info for the request in the form of (time, request, size)
         :return:
         """
-        r = c_cacheReader.read_one_request_full_info(self.cReader)
+        r = mimircache.c_cacheReader.read_one_request_full_info(self.cReader)
         if r and self.block_unit_size != 0 and self.disk_sector_size != 0:
             r = list(r)
             r[1] = r[1] * self.disk_sector_size // self.block_unit_size
@@ -92,10 +94,10 @@ def get_timestamp_list(self):
         :return:
         """
         ts_list = []
-        r = c_cacheReader.read_time_request(self.cReader)
+        r = mimircache.c_cacheReader.read_time_request(self.cReader)
         while r:
             ts_list.append(r[0])
-            r = c_cacheReader.read_time_request(self.cReader)
+            r = mimircache.c_cacheReader.read_time_request(self.cReader)
         return ts_list
 
 

mimircache/const.py

@@ -17,11 +17,28 @@
 DEFAULT_NUM_OF_THREADS = os.cpu_count()
 
 
+failed_components = []
 try:
     import mimircache.c_cacheReader
 except:
+    failed_components.append("cacheReader")
+try:
+    import mimircache.c_LRUProfiler
+except:
+    failed_components.append("LRUProfiler")
+try:
+    import mimircache.c_generalProfiler
+except:
+    failed_components.append("generalProfiler")
+try:
+    import mimircache.c_heatmap
+except:
+    failed_components.append("heatmap")
+
+if len(failed_components):
     CExtensionMode = False
-    print("C extension import failed, which will hurt performance by 10*", file=sys.stderr)
+    print("C extension {} import failed, which will hurt performance by 10*".
+            format(", ".join(failed_components)), file=sys.stderr)
 
 
 from mimircache.cache.ARC import ARC

mimircache/profiler/LRUProfiler.py

@@ -11,7 +11,9 @@
 import os
 import socket
 from mimircache.const import INTERNAL_USE
-import mimircache.c_LRUProfiler as c_LRUProfiler
+from mimircache.const import CExtensionMode
+if CExtensionMode:
+    import mimircache.c_LRUProfiler
 from mimircache.cacheReader.binaryReader import binaryReader
 from mimircache.cacheReader.abstractReader import cacheReaderAbstract
 import matplotlib.pyplot as plt
@@ -80,7 +82,7 @@ def save_reuse_dist(self, file_loc, rd_type):
         """
         assert rd_type == 'rd' or rd_type == 'frd', \
             "please provide a valid reuse distance type, currently support rd and frd"
-        c_LRUProfiler.save_reuse_dist(self.reader.cReader, file_loc, rd_type)
+        mimircache.c_LRUProfiler.save_reuse_dist(self.reader.cReader, file_loc, rd_type)
 
     def load_reuse_dist(self, file_loc, rd_type):
         """
@@ -96,7 +98,7 @@ def load_reuse_dist(self, file_loc, rd_type):
             "please provide a valid reuse distance type, currently support rd and frd"
         if not os.path.exists(file_loc):
             WARNING("pre-computed reuse distance file does not exist")
-        c_LRUProfiler.load_reuse_dist(self.reader.cReader, file_loc, rd_type)
+        mimircache.c_LRUProfiler.load_reuse_dist(self.reader.cReader, file_loc, rd_type)
         self.reader.already_load_rd = True
 
     def _del_reuse_dist_file(self):
@@ -150,7 +152,7 @@ def get_hit_count(self, **kargs):
             print("not supported yet")
             return None
         else:
-            hit_count = c_LRUProfiler.get_hit_count_seq(self.reader.cReader, **kargs)
+            hit_count = mimircache.c_LRUProfiler.get_hit_count_seq(self.reader.cReader, **kargs)
         return hit_count
 
     def get_hit_ratio(self, **kwargs):
@@ -170,10 +172,10 @@ def get_hit_ratio(self, **kwargs):
             kargs['end'] = kwargs['end']
 
         if self.block_unit_size != 0 :
-            hit_ratio = c_LRUProfiler.get_hit_ratio_with_size(self.reader.cReader,
+            hit_ratio = mimircache.c_LRUProfiler.get_hit_ratio_with_size(self.reader.cReader,
                                                             block_unit_size=self.block_unit_size, **kargs)
         else:
-            hit_ratio = c_LRUProfiler.get_hit_ratio_seq(self.reader.cReader, **kargs)
+            hit_ratio = mimircache.c_LRUProfiler.get_hit_ratio_seq(self.reader.cReader, **kargs)
         return hit_ratio
 
 
@@ -202,7 +204,7 @@ def get_hit_ratio_shards(self, sample_ratio=0.01, **kwargs):
             print("not supported yet")
             return None
         else:
-            hit_ratio = c_LRUProfiler.get_hit_ratio_seq_shards(tempReader.cReader, sample_ratio=sample_ratio,
+            hit_ratio = mimircache.c_LRUProfiler.get_hit_ratio_seq_shards(tempReader.cReader, sample_ratio=sample_ratio,
                                                        correction=correction, **kargs)
         return hit_ratio
 
@@ -216,7 +218,7 @@ def get_miss_ratio(self, **kargs):
             print("not supported yet")
             return None
         else:
-            miss_ratio = c_LRUProfiler.get_miss_ratio_seq(self.reader.cReader, **kargs)
+            miss_ratio = mimircache.c_LRUProfiler.get_miss_ratio_seq(self.reader.cReader, **kargs)
         return miss_ratio
 
     def get_reuse_distance(self, **kargs):
@@ -228,7 +230,7 @@ def get_reuse_distance(self, **kargs):
         if self.block_unit_size != 0:
             WARNING("reuse distance calculation does not support variable obj size, "
                     "calculating without considering size")
-        rd = c_LRUProfiler.get_reuse_dist_seq(self.reader.cReader, **kargs)
+        rd = mimircache.c_LRUProfiler.get_reuse_dist_seq(self.reader.cReader, **kargs)
         return rd
 
     def get_future_reuse_distance(self, **kargs):
@@ -240,7 +242,7 @@ def get_future_reuse_distance(self, **kargs):
         if self.block_unit_size != 0:
             WARNING("future reuse distance calculation does not support variable obj size, "
                 "calculating without considering size")
-        frd = c_LRUProfiler.get_future_reuse_dist(self.reader.cReader, **kargs)
+        frd = mimircache.c_LRUProfiler.get_future_reuse_dist(self.reader.cReader, **kargs)
         return frd
 
     def plotMRC(self, figname="MRC.png", auto_resize=False, threshold=0.98, **kwargs):