diff --git a/.travis.yml b/.travis.yml index 6f71795..cca97f4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,71 @@ +#language: python +# +## os: +## - linux +## - osx +# +##python: +## - "3.4" +## - "3.5" +## - "3.6" +## - "nightly" +# +#addons: +# apt: +## sources: +## - sourceline: 'deb http://us.archive.ubuntu.com/ubuntu/ trusty main restricted universe multiverse' +## - sourceline: 'deb-src http://us.archive.ubuntu.com/ubuntu/ trusty main restricted universe multiverse' +# packages: +# libglib2.0-dev +# python3-pip +# python3-matplotlib +# python3-dev +# git +# gcc +# g++ +# +#before_install: +## - sudo apt-get -qq update +## - sudo apt-get install -y libglib2.0-dev python3-pip python3-matplotlib python3-dev git gcc g++ +# - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew update ; fi +# - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install glib python3 openssl readline ; fi +# - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew outdated pyenv || brew upgrade pyenv ; fi +# - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install pyenv-virtualenv ; fi +# - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then export PATH="/Users/travis/.pyenv/shims:${PATH}" ; fi +# - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then pyenv-virtualenv venv; source venv/bin/activate ; fi +# - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then python --version ; fi +# - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then python -m pip install -U pip ; fi +# - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then python -m easy_install -U setuptools ; fi +# +# +## command to install dependencies +#install: +# - pip3 install --upgrade pip setuptools wheel +# - pip3 install -r requirements.txt +# - python3 setup.py build_ext -i +# - which python3 +# - python3 --version +# +#matrix: +# include: +# - os: linux +# python: 3.4 +# - os: linux +# python: 3.5 +# - os: linux +# python: 3.6 +# allow_failures: +# - os: linux +# python: "nightly" +## - os: osx +## language: generic +# +# +# +## command to run tests +#script: nosetests + + diff --git a/docs/User/API.rst b/docs/User/API.rst index 51ed3c1..450181f 100644 --- a/docs/User/API.rst +++ b/docs/User/API.rst @@ -2,8 +2,21 @@ API === +The architecture of mimircache is shown in the diagram below, it contains three parts, profiler, cache and cacheReader, which provides the detailed functions. cachecow is an upper level API that provides most common basic operations. To perform detailed operations, you will need to deal with the three components, so check the API below. +.. image:: ../images/architecture.png + :width: 100% -If you want to know more mimircache and even advanced usage, please check this section. +.. toctree:: + :maxdepth: 3 -Please read source code. \ No newline at end of file + API/cachecow + API/profiler + API/profiler/LRUProfiler + API/profiler/cGeneralProfiler + API/profiler/twoDPlots + API/profiler/cHeatmap + API/profiler/heatmap + API/profiler/generalProfiler + API/cache + API/cacheReader diff --git a/docs/User/API/cache.rst b/docs/User/API/cache.rst new file mode 100644 index 0000000..0f127e8 --- /dev/null +++ b/docs/User/API/cache.rst @@ -0,0 +1,4 @@ +.. _API_cache: + +API-cache +========= \ No newline at end of file diff --git a/docs/User/API/cacheReader.rst b/docs/User/API/cacheReader.rst new file mode 100644 index 0000000..52bb3bf --- /dev/null +++ b/docs/User/API/cacheReader.rst @@ -0,0 +1,4 @@ +.. _API_cacheReader: + +API-cacheReader +=============== \ No newline at end of file diff --git a/docs/User/API/cachecow.rst b/docs/User/API/cachecow.rst new file mode 100644 index 0000000..5ff5707 --- /dev/null +++ b/docs/User/API/cachecow.rst @@ -0,0 +1,15 @@ +.. _API_cachecow: + +API-cachecow +============ + +.. automodule:: mimircache.top.cachecow + :members: + +.. autoclass:: mimircache.top.cachecow.cachecow + :members: + + +.. :inherited-members: +.. :show-inheritance: +.. :undoc-members: diff --git a/docs/User/API/profiler.rst b/docs/User/API/profiler.rst new file mode 100644 index 0000000..057fe79 --- /dev/null +++ b/docs/User/API/profiler.rst @@ -0,0 +1,4 @@ +.. _API_profiler: + +API-profiler +============ \ No newline at end of file diff --git a/docs/User/heatmap_plotting.rst b/docs/User/heatmap_plotting.rst index 1e32cdf..f283b37 100644 --- a/docs/User/heatmap_plotting.rst +++ b/docs/User/heatmap_plotting.rst @@ -69,7 +69,7 @@ Ploting Examples ^^^^^^^^^^^^^^^^ >>> c.heatmap('r', "hit_ratio_start_time_end_time", num_of_pixels=200, cache_size=2000, figname="heatmap1.png", num_of_threads=8) -.. figure:: ../images/example_heatmap.png +.. figure:: ../images/github_heatmap.png :width: 50% :align: center :alt: example hit_ratio_start_time_end_time @@ -100,7 +100,23 @@ You are in the right place! Differential heatmaps allow you to compare cache rep Currently we only support differential heatmap of hit_ratio_start_time_end_time, each pixel (x, y) in the plot implies the hit ratio difference between algorithm1 and algorithm2 and divided by hit ratio of algorithm 1 from time x to y. The function to plot is shown below: ->>> c.diffHeatmap(mode, plot_type, algorithm1, time_interval=-1, num_of_pixels=-1, +>>> c.diff_heatmap(mode, plot_type, algorithm1, time_interval=-1, num_of_pixels=-1, + algorithm2="Optimal", cache_params1=None, cache_params2=None, cache_size=-1, **kwargs) + +The arguments here are similar to plotting heatmaps, the only difference is that we have one more algorithm, which is used for comparison, + + +Example: +>>> c.diffHeatmap('r', "hit_ratio_start_time_end_time", time_interval=1000000, algorithm1="LRU", cache_size=2000) + + +You are in the right place! Differential heatmaps allow you to compare cache replacement algorithms with respect to time. + + +Currently we only support differential heatmap of hit_ratio_start_time_end_time, each pixel (x, y) in the plot implies the hit ratio difference between algorithm1 and algorithm2 and divided by hit ratio of algorithm 1 from time x to y. +The function to plot is shown below: + +>>> c.diff_heatmap(mode, plot_type, algorithm1, time_interval=-1, num_of_pixels=-1, algorithm2="Optimal", cache_params1=None, cache_params2=None, cache_size=-1, **kwargs) The arguments here are similar to plotting heatmaps, the only difference is that we have one more algorithm, which is used for comparison, @@ -109,6 +125,54 @@ The arguments here are similar to plotting heatmaps, the only difference is that Example: >>> c.diffHeatmap('r', "hit_ratio_start_time_end_time", time_interval=1000000, algorithm1="LRU", cache_size=2000) + +You are in the right place! Differential heatmaps allow you to compare cache replacement algorithms with respect to time. + + +Currently we only support differential heatmap of hit_ratio_start_time_end_time, each pixel (x, y) in the plot implies the hit ratio difference between algorithm1 and algorithm2 and divided by hit ratio of algorithm 1 from time x to y. +The function to plot is shown below: + +>>> c.diffHeatmap(mode, plot_type, algorithm1, time_interval=-1, num_of_pixels=-1, + algorithm2="Optimal", cache_params1=None, cache_params2=None, cache_size=-1, **kwargs) + +The arguments here are similar to plotting heatmaps, the only difference is that we have one more algorithm, which is used for comparison, + + +Example: +>>> c.diff_heatmap('r', "hit_ratio_start_time_end_time", time_interval=1000000, algorithm1="LRU", cache_size=2000) + + +You are in the right place! Differential heatmaps allow you to compare cache replacement algorithms with respect to time. + + +Currently we only support differential heatmap of hit_ratio_start_time_end_time, each pixel (x, y) in the plot implies the hit ratio difference between algorithm1 and algorithm2 and divided by hit ratio of algorithm 1 from time x to y. +The function to plot is shown below: + +>>> c.diffHeatmap(mode, plot_type, algorithm1, time_interval=-1, num_of_pixels=-1, + algorithm2="Optimal", cache_params1=None, cache_params2=None, cache_size=-1, **kwargs) + +The arguments here are similar to plotting heatmaps, the only difference is that we have one more algorithm, which is used for comparison, + + +Example: +>>> c.diff_heatmap('r', "hit_ratio_start_time_end_time", time_interval=1000000, algorithm1="LRU", cache_size=2000) + + +You are in the right place! Differential heatmaps allow you to compare cache replacement algorithms with respect to time. + + +Currently we only support differential heatmap of hit_ratio_start_time_end_time, each pixel (x, y) in the plot implies the hit ratio difference between algorithm1 and algorithm2 and divided by hit ratio of algorithm 1 from time x to y. +The function to plot is shown below: + +>>> c.diff_heatmap(mode, plot_type, algorithm1, time_interval=-1, num_of_pixels=-1, + algorithm2="Optimal", cache_params1=None, cache_params2=None, cache_size=-1, **kwargs) + +The arguments here are similar to plotting heatmaps, the only difference is that we have one more algorithm, which is used for comparison, + + +Example: +>>> c.diff_heatmap('r', "hit_ratio_start_time_end_time", time_interval=1000000, algorithm1="LRU", cache_size=2000) + .. figure:: ../images/example_differential_heatmap.png :width: 50% :align: center diff --git a/docs/User/quick_start.rst b/docs/User/quick_start.rst index 4942bb2..aa38a3d 100644 --- a/docs/User/quick_start.rst +++ b/docs/User/quick_start.rst @@ -30,22 +30,19 @@ Get Basic Statistics -------------------- You can get some statistics about the trace, for example how many request, how may unique requests. -+-------------------------+------------------------+------------------------------------------------------------+ -| Functions | Parameters | Description | -+-------------------------+------------------------+------------------------------------------------------------+ -| num_of_req | None | return the number of requests in the trace | -+-------------------------+------------------------+------------------------------------------------------------+ -| num_of_uniq_req | None | return the number of unique requests in the trace | -+-------------------------+------------------------+------------------------------------------------------------+ -|get_reuse_distance | None | return an array of reuse distance of the trace | -+-------------------------+------------------------+------------------------------------------------------------+ -|get_hit_ratio_dict | algorithm, cache_size, | return a dict of mapping cache size to hit ratio, | -| | bin_size, cache_params | all params except algorithm is optional | -+-------------------------+------------------------+------------------------------------------------------------+ -| reset | None | reset reader to the beginning of the trace | -+-------------------------+------------------------+------------------------------------------------------------+ -| len | None | return the number of requests in the trace | -+-------------------------+------------------------+------------------------------------------------------------+ ++-------------------------+--------------------------------+------------------------------------------------------------+ +| Functions | Parameters | Description | ++-------------------------+--------------------------------+------------------------------------------------------------+ +| num_of_req | None | return the number of requests in the trace | ++-------------------------+--------------------------------+------------------------------------------------------------+ +| num_of_uniq_req | None | return the number of unique requests in the trace | ++-------------------------+--------------------------------+------------------------------------------------------------+ +| stat | None | get a list of statistical information about the trace | ++-------------------------+--------------------------------+------------------------------------------------------------+ +| characterize | type (short/medium/long/all) | plot a series of fig, type indicates run time | ++-------------------------+--------------------------------+------------------------------------------------------------+ +| len | None | return the number of requests in the trace | ++-------------------------+--------------------------------+------------------------------------------------------------+ If you want to read your data from cachecow, you can simply use cachecow as an iterator, for example, doing the following: @@ -55,31 +52,83 @@ If you want to read your data from cachecow, you can simply use cachecow as an i Profiler and Profiling -______________________ +----------------------- +Now cachecow supports basic profiling, to conduct complex profiling, you still need to get a profiler. +With a profiler, you can obtain the reuse distance of a request, the hit count and hit ratio of at a certain size, you can even directly plot the hit ratio curve (HRC). See :ref:`here` for details. -With a profiler, you can obtain the reuse distance of a request, the hit count and hit ratio of at a certain size, you can even directly plot the hit ratio curve(HRC). -See :ref:`here` for details. +cachecow supports two type of profiling right now, calculate reuse distance and calculate hit ratio, the syntax are listed below. +>>> # get an array of reuse distance +>>> c.get_reuse_distance() +>>> # get a dictionary of cache size -> hit ratio +>>> c.get_hit_ratio_dict(algorithm, cache_size=-1, cache_params=None, bin_size=-1) +See :ref:`API-cachecow` section for details. -Basic Plotting -______________ -Basic plotting describes how to use mimircache to plot cold_miss, cold_miss_ratio, request_num, obj_mapping plot and hit ratio curve (miss ratio curve) plots. -See :ref:`here` for details. +Two Dimensional Plotting +------------------------ +cachecow supports the following two dimensional figures, + ======================== ============================ ================================================= + plot type required parameters Description + ======================== ============================ ================================================= + cold_miss_count time_mode, time_interval cold miss count VS time + cold_miss_ratio time_mode, time_interval coid miss ratio VS time + request_rate time_mode, time_interval num of requests VS time + popularity NA Percentage of obj VS frequency + rd_popularity NA Num of req VS reuse distance + rt_popularity NA Num of req VS reuse time + mapping NA mapping from original objID to sequential number + interval_hit_ratio cache_size hit ratio of interval VS time + ======================== ============================ ================================================= +The basic syntax for plotting the two dimensional figures is here +>>> # see table for plot_type names +>>> c.twoDPlot(plot_type, **kwargs) -Heatmap Plotting -________________ -Heatmap plotting section describes how to use mimircache to plot heatmaps. -See :ref:`here` for details. +See :ref:`API-twoDPlots` section and :ref:`basic plotting` for details. + + +Hit Ratio Curve Plotting +------------------------ + +cachecow supports plotting against a list of cache replacement algorithms, the syntax is here. +>>> plotHRCs(algorithm_list, cache_params=(), cache_size=-1, bin_size=-1, auto_resize=True, figname="HRC.png", **kwargs) +See :ref:`API-LRUProfiler and API-cGeneralProfiler` section and :ref:`basic plotting` for details. +Heatmap Plotting +---------------- + +cachecow supports basic heatmap plotting, and supported plot type is listed below. + +>>> # plot heatmaps +>>> heatmap(time_mode, plot_type, time_interval=-1, num_of_pixels=-1, algorithm="LRU", cache_params=None, cache_size=-1, **kwargs) +>>> # plot differential heatmaps +>>> diff_heatmap(time_mode, plot_type, algorithm1, time_interval=-1, num_of_pixels=-1, algorithm2="Optimal", cache_params1=None, cache_params2=None, cache_size=-1, **kwargs) + +================================================================ ======================================================================== + plot type Description +================================================================ ======================================================================== + * hit_ratio_start_time_end_time Hit ratio heatmap of given start time and end time + * hit_ratio_start_time_cache_size (python only) Hit ratio heatmap of given start time and cache size + * avg_rd_start_time_end_time (python only) Average reuse distance of start time and end time + * cold_miss_count_start_time_end_time (python only) deprecated + * rd_distribution Heatmap of reuse distance distribution over time + * rd_distribution_CDF Heatmap (CDF) of reuse distance distribution over time + * future_rd_distribution Heatmap of future reuse distribution over time + * dist_distribution Heatmap of distance distribution over time + * reuse_time_distribution Heatmap of reuse time distribution over time +================================================================ ======================================================================== + +Heatmap plotting section describes how to use mimircache to plot heatmaps. +See :ref:`API-cHeatmap` section and :ref:`here` for details. + diff --git a/docs/_build/doctrees/User/API.doctree b/docs/_build/doctrees/User/API.doctree index c2403ba..dc10f60 100644 Binary files a/docs/_build/doctrees/User/API.doctree and b/docs/_build/doctrees/User/API.doctree differ diff --git a/docs/_build/doctrees/User/heatmap_plotting.doctree b/docs/_build/doctrees/User/heatmap_plotting.doctree index 5286b62..ed263a5 100644 Binary files a/docs/_build/doctrees/User/heatmap_plotting.doctree and b/docs/_build/doctrees/User/heatmap_plotting.doctree differ diff --git a/docs/_build/doctrees/User/quick_start.doctree b/docs/_build/doctrees/User/quick_start.doctree index 998b6b2..0cd91e5 100644 Binary files a/docs/_build/doctrees/User/quick_start.doctree and b/docs/_build/doctrees/User/quick_start.doctree differ diff --git a/docs/_build/doctrees/environment.pickle b/docs/_build/doctrees/environment.pickle index 569ce9b..905157b 100644 Binary files a/docs/_build/doctrees/environment.pickle and b/docs/_build/doctrees/environment.pickle differ diff --git a/docs/_build/doctrees/index.doctree b/docs/_build/doctrees/index.doctree index f0385d6..e7ec6ac 100644 Binary files a/docs/_build/doctrees/index.doctree and b/docs/_build/doctrees/index.doctree differ diff --git a/docs/conf.py b/docs/conf.py index 9feeb84..283db41 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,24 +1,21 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# -# mimircache documentation build configuration file, created by -# sphinx-quickstart on Sun May 29 09:40:45 2016. -# -# This file is execfile()d with the current directory set to its -# containing dir. -# -# Note that not all possible configuration values are present in this -# autogenerated file. -# -# All configuration values have a default; values that are commented out -# serve to show the default. + import sys import os +# import sphinx_bootstrap_theme + +# Insert mimircache' path into the system. +sys.path.insert(0, os.path.abspath('..')) +sys.path.insert(0, os.path.abspath('_themes')) + +from version import __version__ MOCK_MODULES = ['mimircache.c_generalProfiler', 'mimircache.c_heatmap', 'mimircache.c_LRUProfiler', - 'mimircache.c_cacheReader', 'c_generalProfiler', 'c_heatmap', 'c_LRUProfiler', 'c_cacheReader'] + 'mimircache.c_cacheReader', "mimircache.c_eviction_stat"] + #, 'c_generalProfiler', 'c_heatmap', 'c_LRUProfiler', 'c_cacheReader'] @@ -46,15 +43,6 @@ def __getattr__(cls, name): sys.modules[mod_name] = Mock() - - -# Insert mimircache' path into the system. -sys.path.insert(0, os.path.abspath('..')) -sys.path.insert(0, os.path.abspath('_themes')) - - -from version import __version__ - # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. @@ -156,9 +144,11 @@ def __getattr__(cls, name): # html_theme = 'alabaster' # html_theme = 'default' # html_theme = 'nature' -# html_theme = 'sphinxdoc' +html_theme = 'sphinxdoc' # html_theme = 'scrolls' -html_theme = 'pyramid' +# html_theme = 'pyramid' +# html_theme = 'bootstrap' +# html_theme_path = sphinx_bootstrap_theme.get_html_theme_path() # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the @@ -223,7 +213,7 @@ def __getattr__(cls, name): # html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -html_show_sphinx = False +html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. # html_show_copyright = True @@ -334,3 +324,5 @@ def __getattr__(cls, name): # If true, do not generate a @detailmenu in the "Top" node's menu. # texinfo_no_detailmenu = False + +autodoc_member_order = "bysource" diff --git a/docs/images/architecture.png b/docs/images/architecture.png new file mode 100644 index 0000000..cb51ea5 Binary files /dev/null and b/docs/images/architecture.png differ diff --git a/docs/index.rst b/docs/index.rst index 31d023f..7889bcd 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -5,7 +5,7 @@ Release v\ |version|. Welcome to the documentation of mimircache, a Python3 cache trace analysis platform. -**The power of mimircache**:: +**The power of mimircache**: >>> import mimircache as m >>> c = m.cachecow() >>> c.vscsi("trace.vscsi") # find this data under data folder, other type of data supported too @@ -20,12 +20,12 @@ Welcome to the documentation of mimircache, a Python3 cache trace analysis platf >>> c.heatmap('r', "hit_ratio_start_time_end_time", time_interval=10000000) -.. image:: images/example_MRC.png +.. image:: images/github_HRC.png :width: 45% .. image:: images/github_heatmap.png :width: 48% -An example of MRC plot and hit ratio heatmap. +An example of hit ratio curve plot and hit ratio heatmap. The User Guide @@ -80,6 +80,4 @@ Now you can customize mimircache to fit your own need, you can Indices and tables ================== -* :ref:`genindex` -* :ref:`modindex` * :ref:`search` diff --git a/setup.py b/setup.py index e018d4f..fa66899 100644 --- a/setup.py +++ b/setup.py @@ -225,6 +225,7 @@ def getNumpyHeader(): extra_link_args=extra_link_args, language="c")) + extensions.append(Extension( 'mimircache.c_eviction_stat', glob("mimircache/CExtension/profiler/*.c") +