diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ddc93d9b..155a78d3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -44,17 +44,17 @@ jobs: working-directory: ${{github.workspace}}/build run: ctest -C ${{env.BUILD_TYPE}} - selfhosted: - runs-on: self-hosted - steps: - - uses: actions/checkout@v2 - - name: Configure CMake - run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DENABLE_GLCACHE=on -DENABLE_LRB=on - - name: Build - run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} - - name: Test - working-directory: ${{github.workspace}}/build - run: ctest -C ${{env.BUILD_TYPE}} -j4 + # selfhosted: + # runs-on: self-hosted + # steps: + # - uses: actions/checkout@v2 + # - name: Configure CMake + # run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DENABLE_GLCACHE=on -DENABLE_LRB=on + # - name: Build + # run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} + # - name: Test + # working-directory: ${{github.workspace}}/build + # run: ctest -C ${{env.BUILD_TYPE}} -j4 diff --git a/.gitignore b/.gitignore index 4deb53b8..dc00d77f 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,5 @@ example/cacheSimulatorC/cmake-build-debug .vscode/* *.log fig/ +# Chaos +sftp-config.json diff --git a/CMakeLists.txt b/CMakeLists.txt index d802d6a0..3aaa37d3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.2) +cmake_minimum_required(VERSION 3.12) project(libCacheSim) set(DESCRIPTION "a high performance cache simulation library") set(PROJECT_WEB "http://cachesim.com") @@ -9,6 +9,9 @@ set(${PROJECT_NAME}_VERSION_PATCH 0) set(${PROJECT_NAME}_RELEASE_VERSION ${${PROJECT_NAME}_VERSION_MAJOR}.${${PROJECT_NAME}_VERSION_MINOR}) set(${PROJECT_NAME}_VERSION ${${PROJECT_NAME}_RELEASE_VERSION}.${${PROJECT_NAME}_VERSION_PATCH}) +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED On) +set(CMAKE_CXX_EXTENSIONS Off) ######################################## # define options # @@ -19,7 +22,7 @@ option(USE_HUGEPAGE "use transparent hugepage" ON) option(ENABLE_TESTS "whether enable test" ON) option(ENABLE_GLCACHE "enable group-learned cache" OFF) option(SUPPORT_TTL "whether support TTL" OFF) -option(SUPPORT_ZSTD_TRACE "whether support zstd trace" ON) +option(OPT_SUPPORT_ZSTD_TRACE "whether support zstd trace" ON) option(ENABLE_LRB "enable LRB" OFF) set(LOG_LEVEL NONE CACHE STRING "change the logging level") set_property(CACHE LOG_LEVEL PROPERTY STRINGS INFO WARN ERROR DEBUG VERBOSE VVERBOSE VVVERBOSE) @@ -29,10 +32,10 @@ set_property(CACHE LOG_LEVEL PROPERTY STRINGS INFO WARN ERROR DEBUG VERBOSE VVER # detect platform # ######################################## if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") + message(STATUS "Mac OS X detected, version ${CMAKE_SYSTEM_VERSION}") add_definitions(-DOS_DARWIN) - if (NOT CMAKE_BUILD_TYPE) - set(CMAKE_BUILD_TYPE Debug) - endif() + set(CMAKE_INCLUDE_PATH ${CMAKE_INCLUDE_PATH} "/opt/homebrew/include/") + set(CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH} "/opt/homebrew/") elseif (${CMAKE_SYSTEM_NAME} MATCHES "Linux") # set(CFLAGS "$ENV{CFLAGS} " "-Wl,--export-dynamic ") @@ -102,7 +105,7 @@ endif() message(STATUS "CMAKE_CXX_FLAGS_DEBUG ${CMAKE_CXX_FLAGS_DEBUG} CMAKE_CXX_FLAGS_RELWITHDEBINFO ${CMAKE_CXX_FLAGS_RELWITHDEBINFO} CMAKE_CXX_FLAGS_RELEASE ${CMAKE_CXX_FLAGS_RELEASE}") # string( REPLACE "/DNDEBUG" "" CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO}") -message(STATUS "SUPPORT TTL ${SUPPORT_TTL}, USE_HUGEPAGE ${USE_HUGEPAGE}, LOGLEVEL ${LOG_LEVEL}, ENABLE_GLCACHE ${ENABLE_GLCACHE}, ENABLE_LRB ${ENABLE_LRB}, SUPPORT_ZSTD_TRACE ${SUPPORT_ZSTD_TRACE}") +message(STATUS "SUPPORT TTL ${SUPPORT_TTL}, USE_HUGEPAGE ${USE_HUGEPAGE}, LOGLEVEL ${LOG_LEVEL}, ENABLE_GLCACHE ${ENABLE_GLCACHE}, ENABLE_LRB ${ENABLE_LRB}, OPT_SUPPORT_ZSTD_TRACE ${OPT_SUPPORT_ZSTD_TRACE}") # add_compile_options(-fsanitize=address) # add_link_options(-fsanitize=address) @@ -114,13 +117,11 @@ message(STATUS "SUPPORT TTL ${SUPPORT_TTL}, USE_HUGEPAGE ${USE_HUGEPAGE}, LOGLEV set(CFLAGS "$ENV{CFLAGS} " "-Wall -Wshadow -Winline " "-Wno-unused " - "-Wstrict-prototypes -Wmissing-prototypes " + "-Wstrict-prototypes " + # "-Wmissing-prototypes " "-Wmissing-declarations " "-Wredundant-decls " - "-Wunused-value -Wunused-variable " - "-std=c11 " "-fno-strict-aliasing " - "-O0 " ) @@ -140,24 +141,25 @@ set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/Modules/") find_package(GLib REQUIRED) include_directories(${GLib_INCLUDE_DIRS}) set(LIBS ${LIBS} ${GLib_LIBRARY}) -#if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") -# pkg_check_modules(GLib2 REQUIRED Glib-2.0) -# include_directories(${GLib2_INCLUDE_DIRS}) -# # add_compile_options("${LDFLAGS}") -# link_libraries(${GLib2_LDFLAGS}) -#endif() - -if (SUPPORT_ZSTD_TRACE) + + +find_package(argp REQUIRED) +include_directories(${ARGP_INCLUDE_DIRS}) +set(LIBS ${LIBS} ${ARGP_LIBRARY}) + +if (OPT_SUPPORT_ZSTD_TRACE) add_compile_definitions(SUPPORT_ZSTD_TRACE=1) find_package(ZSTD) + # https://stackoverflow.com/questions/61377055/cannot-find-gflags-gflags-h-while-building-library-osx/61379123#61379123 + include_directories(${ZSTD_INCLUDE_DIR}) if ("${ZSTD_LIBRARIES}" STREQUAL "") message(FATAL_ERROR "zstd not found") endif() link_libraries(${ZSTD_LIBRARIES}) - message(STATUS "ZSTD_INCLUDE_DIRS ${ZSTD_INCLUDE_DIRS}, ZSTD_LIBRARIES ${ZSTD_LIBRARIES}") + message(STATUS "ZSTD_INCLUDE_DIR ${ZSTD_INCLUDE_DIR}, ZSTD_LIBRARIES ${ZSTD_LIBRARIES}") else() remove_definitions(SUPPORT_ZSTD_TRACE) -endif(SUPPORT_ZSTD_TRACE) +endif(OPT_SUPPORT_ZSTD_TRACE) # libgoogle-perftools-dev google-perftools @@ -222,7 +224,7 @@ message(STATUS "<<++=====------------------/\\------------------=====++>>") message(STATUS "================== dependency related ==================") message(STATUS "glib found? ${GLib_FOUND} - LIBS=${GLib_LIBRARY}, header =${GLib_INCLUDE_DIRS}") message(STATUS "tcmalloc found? ${Tcmalloc_FOUND} - LIBS=${Tcmalloc_LIBRARIES}, header=${Tcmalloc_INCLUDE_DIRS}") -message(STATUS "ZSTD found? ${ZSTD_FOUND} - LIBS=${ZSTD_LIBRARIES}, header=${ZSTD_INCLUDE_DIRS}") +message(STATUS "ZSTD found? ${ZSTD_FOUND} - LIBS=${ZSTD_LIBRARIES}, header=${ZSTD_INCLUDE_DIR}") message(STATUS "==================== CMake related =====================") message(STATUS "platform = ${CMAKE_SYSTEM_NAME} ${CMAKE_SYSTEM_VERSION}") @@ -311,14 +313,24 @@ if (ENABLE_LRB) ) endif(ENABLE_LRB) -file(GLOB reader_source - ${PROJECT_SOURCE_DIR}/libCacheSim/traceReader/*.c - ${PROJECT_SOURCE_DIR}/libCacheSim/traceReader/generalReader/*.c +set(reader_source + ${PROJECT_SOURCE_DIR}/libCacheSim/traceReader/reader.c + ${PROJECT_SOURCE_DIR}/libCacheSim/traceReader/generalReader/binary.c + ${PROJECT_SOURCE_DIR}/libCacheSim/traceReader/generalReader/csv.c + ${PROJECT_SOURCE_DIR}/libCacheSim/traceReader/generalReader/lcs.c + ${PROJECT_SOURCE_DIR}/libCacheSim/traceReader/generalReader/libcsv.c + ${PROJECT_SOURCE_DIR}/libCacheSim/traceReader/generalReader/txt.c ) +if (OPT_SUPPORT_ZSTD_TRACE) + set (reader_source + ${reader_source} ${PROJECT_SOURCE_DIR}/libCacheSim/traceReader/generalReader/zstdReader.c + ) +endif(OPT_SUPPORT_ZSTD_TRACE) + file(GLOB dataStructure_source ${PROJECT_SOURCE_DIR}/libCacheSim/dataStructure/*.c ${PROJECT_SOURCE_DIR}/libCacheSim/dataStructure/hashtable/*.c - ${PROJECT_SOURCE_DIR}/libCacheSim/dataStructure/hash/*.c + ${PROJECT_SOURCE_DIR}/libCacheSim/dataStructure/hash/murmur3.c ) file(GLOB profiler_source @@ -335,7 +347,8 @@ set(LIB_SOURCE ${LIB_SOURCE} ${cache_source} ${reader_source} ${dataStructure_so # # https://stackoverflow.com/questions/32469953/why-is-cmake-designed-so-that-it-removes-runtime-path-when-installing SET(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) -add_library(${PROJECT_NAME} SHARED ${LIB_SOURCE}) +add_library(${PROJECT_NAME} ${LIB_SOURCE}) +# add_library(${PROJECT_NAME} SHARED ${LIB_SOURCE}) set_target_properties(${PROJECT_NAME} PROPERTIES VERSION ${${PROJECT_NAME}_VERSION}) set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE C) set_target_properties(${PROJECT_NAME} PROPERTIES PUBLIC_HEADER ${PROJECT_SOURCE_DIR}/libCacheSim/include/libCacheSim.h) @@ -354,12 +367,6 @@ install(TARGETS ${PROJECT_NAME} LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) - - -# #target_compile_definitions(binBenchmark PRIVATE HASH=0xb2) -# #add_executable(binBenchmark libCacheSim/benchmark/main.c) -# #target_link_libraries(binBenchmark benchmark cachelib evictionAlgos traceReader profiler dataStructure utils -# # ${LIBS}) ############################# diff --git a/README.md b/README.md index e9834377..56c0edaa 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,39 @@ # libCacheSim - building and running cache simulations -[![build](https://github.com/1a1a11a/libCacheSimPrv/actions/workflows/build.yml/badge.svg)](https://github.com/1a1a11a/libCacheSimPrv/actions/workflows/build.yml) - -**The stable development of libCacheSim has moved to [https://github.com/cacheMon/libCacheSim](https://github.com/cacheMon/libCacheSim)**. - - +[![build](https://github.com/1a1a11a/libCacheSim/actions/workflows/build.yml/badge.svg)](https://github.com/1a1a11a/libCacheSim/actions/workflows/build.yml) + +#### The main development of libCacheSim is at [https://github.com/1a1a11a/libCacheSim](https://github.com/1a1a11a/libCacheSim), the [cachemon](https://github.com/cachemon/libCacheSim) repo is a mirror of the stable branch. Please fork and submit PR to this repo. + + +- [What is libCacheSim](#what-is-libcachesim) +- [libCacheSim features](#libcachesim-features) +- [Supported algorithms](#supported-algorithms) +- [Build and Install libCacheSim](#build-and-install-libcachesim) + - [One-line install](#one-line-install) + - [Install dependency](#install-dependency) + - [Build libCacheSim](#build-libcachesim) +- [Usage](#usage) + - [cachesim (a high-performance cache simulator)](#cachesim-a-high-performance-cache-simulator) + - [Trace analysis](#trace-analysis) + - [Using libCacheSim as a library](#using-libcachesim-as-a-library) + - [Extending libCacheSim (new algorithms and trace types)](#extending-libcachesim-new-algorithms-and-trace-types) +- [Open source cache traces](#open-source-cache-traces) +- [Questions?](#questions) +- [Contributions](#contributions) +- [Reference](#reference) +- [License](#license) +- [Related](#related) + + + + ## News -* **2023 June**: **QD-LP** is available now, see [our paper](https://dl.acm.org/doi/10.1145/3593856.3595887) for details. -* **2023 Oct**: **S3-FIFO** and **Sieve** are generally available! These are simple algorithms that are very effective in reducing cache misses. Try them out in libCacheSim and your production! +* **2023 June**: **QDLP** is available now, see [our paper](https://dl.acm.org/doi/10.1145/3593856.3595887) for details. +* **2023 Oct**: **[S3-FIFO](https://dl.acm.org/doi/10.1145/3600006.3613147)** and **SIEVE(https://sievecache.com)** are available! These are very simple algorithms that are very effective in reducing cache misses. Try them out in libCacheSim and your production! +* **2024 Jan**: We compiled a list of open-source cache datasets at the bottom of this page --- + ## What is libCacheSim * a high-performance **cache simulator** for running cache simulations. * a high-performance and versatile trace analyzer for **analyzing different cache traces**. @@ -17,18 +41,21 @@ --- + ## libCacheSim features * **High performance** - over 20M requests/sec for a realistic trace replay. * **High memory efficiency** - predictable and small memory footprint. -* **State-of-the-art algorithms** - eviction algorithms, admission algorithms, sampling techniques, approximate miss ratio computation, see [here](/doc/quickstart_cachesim.md). +* **State-of-the-art algorithms** - eviction algorithms, admission algorithms, prefetching algorithms, sampling techniques, approximate miss ratio computation, see [here](/doc/quickstart_cachesim.md). * Parallelism out-of-the-box - uses the many CPU cores to speed up trace analysis and cache simulations. * **The ONLY feature-rich trace analyzer** - all types of trace analysis you need, see [here](/doc/quickstart_traceAnalyzer.md). -* **Simple API** - easy to build cache clusters, multi-layer caching, etc, see [here](/doc/API.md). -* **Extensible** - easy to support new trace types or eviction algorithms, see [here](/doc/advanced_lib_extend.md). +* **Simple API** - easy to build cache clusters, multi-layer caching, etc.; see [here](/doc/API.md). +* **Extensible** - easy to support new trace types or eviction algorithms; see [here](/doc/advanced_lib_extend.md). --- + ## Supported algorithms cachesim supports the following algorithms: +### Eviction algorithms * [FIFO](/libCacheSim/cache/eviction/FIFO.c), [LRU](/libCacheSim/cache/eviction/LRU.c), [Clock](/libCacheSim/cache/eviction/Clock.c), [SLRU](/libCacheSim/cache/eviction/SLRU.c) * [LFU](/libCacheSim/cache/eviction/LFU.c), [LFU with dynamic aging](/libCacheSim/cache/eviction/LFUDA.c) * [ARC](/libCacheSim/cache/eviction/ARC.c), [TwoQ](/libCacheSim/cache/eviction/TwoQ.c) @@ -40,14 +67,25 @@ cachesim supports the following algorithms: * [LHD](/libCacheSim/cache/eviction/LHD/LHD_Interface.cpp) * [LRB](/libCacheSim/cache/eviction/LRB/LRB_Interface.cpp) * [GLCache](/libCacheSim/cache/eviction/GLCache/GLCache.c) -* [TinyLFU](/libCacheSim/cache/eviction/TinyLFU.c) +* [WTinyLFU](/libCacheSim/cache/eviction/WTinyLFU.c) * [QD-LP](/libCacheSim/cache/eviction/QDLP.c) * [S3-FIFO](/libCacheSim/cache/eviction/S3FIFO.c) * [Sieve](/libCacheSim/cache/eviction/Sieve.c) +### Admission algorithms +* [Adaptsize](/libCacheSim/cache/admission/adaptsize.c) +* [Bloomfilter](/libCacheSim/cache/admission/bloomfilter.c) +* [Prob](/libCacheSim/cache/admission/prob.c) +* [Size](/libCacheSim/cache/admission/size.c) +### Prefetching algorithms +* [OBL](/libCacheSim/cache/prefetch/OBL.c) +* [Mithril](/libCacheSim/cache/prefetch/Mithril.c) +* [PG](/libCacheSim/cache/prefetch/PG.c) --- + ## Build and Install libCacheSim + ### One-line install We provide some scripts for quick installation of libCacheSim. ```bash @@ -57,16 +95,18 @@ If this does not work, please 1. let us know what system you are using and what error you get 2. read the following sections for self-installation. + ### Install dependency libCacheSim uses [cmake](https://cmake.org/) build system and has a few dependencies: [glib](https://developer.gnome.org/glib/), [tcmalloc](https://github.com/google/tcmalloc), [zstd](https://github.com/facebook/zstd). -Please see [install.md](/doc/install.md) for how to install the dependencies. +Please see [install.md](/doc/install.md) for instructions on how to install the dependencies. + ### Build libCacheSim cmake recommends **out-of-source build**, so we do it in a new directory: ``` -git clone https://github.com/1a1a11a/libCacheSimPrv -pushd libCachesimPrv; +git clone https://github.com/1a1a11a/libCacheSim +pushd libCachesim; mkdir _build && cd _build; cmake .. && make -j; [sudo] make install; @@ -74,9 +114,12 @@ popd; ``` --- + ## Usage + ### cachesim (a high-performance cache simulator) After building and installing libCacheSim, `cachesim` should be in the `_build/bin/` directory. + #### basic usage ``` ./bin/cachesim trace_path trace_type eviction_algo cache_size [OPTION...] @@ -84,20 +127,22 @@ After building and installing libCacheSim, `cachesim` should be in the `_build/b use `./bin/cachesim --help` to get more information. + #### Run a single cache simulation Run the example traces with LRU eviction algorithm and 1GB cache size. ```bash -# Note that no space between the cache size and the unit, unit is not case sensitive +# Note that no space between the cache size and the unit, and the unit is not case-sensitive ./bin/cachesim ../data/trace.vscsi vscsi lru 1gb ``` + #### Run multiple cache simulations with different cache sizes ```bash -# Note that no space between the cache sizes +# Note that there is no space between the cache sizes ./bin/cachesim ../data/trace.vscsi vscsi lru 1mb,16mb,256mb,8gb -# besides absolute cache size, you can also use fraction of working set size +# Besides absolute cache size, you can also use a fraction of the working set size ./bin/cachesim ../data/trace.vscsi vscsi lru 0.001,0.01,0.1,0.2 # besides using byte as the unit, you can also treat all objects having the same size, and the size is the number of objects @@ -113,6 +158,7 @@ Run the example traces with LRU eviction algorithm and 1GB cache size. See [quick start cachesim](/doc/quickstart_cachesim.md) for more usages. + #### Plot miss ratio curve You can plot miss ratios of different algorithms and sizes, and plot the miss ratios over time. @@ -127,6 +173,7 @@ python3 plot_mrc_time.py --tracepath ../data/twitter_cluster52.csv --trace-forma --- + ### Trace analysis libCacheSim also has a trace analyzer that provides a lot of useful information about the trace. And it is very fast, designed to work with billions of requests. @@ -138,9 +185,10 @@ See [trace analysis](/doc/quickstart_traceAnalyzer.md) for more details. --- + ### Using libCacheSim as a library libCacheSim can be used as a library for building cache simulators. -For example, you can build a cache cluster with consistent hashing, or a multi-layer cache simulator. +For example, you can build a cache cluster with consistent hashing or a multi-layer cache simulator. Here is a simplified example showing the basic APIs. ```c @@ -149,7 +197,7 @@ Here is a simplified example showing the basic APIs. /* open trace, see quickstart_lib.md for opening csv and binary trace */ reader_t *reader = open_trace("../data/trace.vscsi", VSCSI_TRACE, NULL); -/* craete a container for reading from trace */ +/* create a container for reading from trace */ request_t *req = new_request(); /* create a LRU cache */ @@ -175,7 +223,7 @@ free_request(req); cache->cache_free(cache); ``` -save this to `test.c` and compile with +save this to `test.c` and compile it with ``` bash gcc test.c $(pkg-config --cflags --libs libCacheSim glib-2.0) -o test.out ``` @@ -185,6 +233,7 @@ See [here](/doc/advanced_lib.md) for more details, and see [example folder](/exa --- + ### Extending libCacheSim (new algorithms and trace types) libCacheSim supports *txt*, *csv*, and *binary* traces. We prefer binary traces because it allows libCacheSim to run faster, and the traces are more compact. @@ -194,21 +243,55 @@ If you need to add a new trace type or a new algorithm, please see [here](/doc/a --- -### Questions? + +## Open source cache traces +In the [repo](/data/), there are sample (one from cloudphysics and one from twitter) traces in different formats (csv, txt, vscsi, and oracleGeneral). Note that the provided traces are **very small** samples and __should not be used for evaluating different algorithms' miss ratios__. The full traces can be found either with the original release or the processed oracleGeneral format. + +Note that the oracleGeneral traces are compressed with [zstd](https://github.com/facebook/zstd) and have the following format: + +``` +struct { + uint32_t timestamp; + uint64_t obj_id; + uint32_t obj_size; + int64_t next_access_vtime; // -1 if no next access +} +``` +The compressed traces can be used with libCacheSim without decompression. And libCacheSim provides a `tracePrint` tool to print the trace in human-readable format. + + +| Dataset | Year | Type | Original release | OracleGeneral format | +|---------------|------|:---------:|:-----------------------------------------------------------------------------------------:|:----------------------------------------------------------------------------------:| +| Tencent Photo | 2018 | object | [link](http://iotta.snia.org/traces/parallel?only=27476) | [link](https://ftp.pdl.cmu.edu/pub/datasets/twemcacheWorkload/cacheDatasets/tencentPhoto/) | +| WikiCDN | 2019 | object | [link](https://wikitech.wikimedia.org/wiki/Analytics/Data_Lake/Traffic/Caching) | [link](https://ftp.pdl.cmu.edu/pub/datasets/twemcacheWorkload/cacheDatasets/wiki/) | +| Tencent CBS | 2020 | block | [link](http://iotta.snia.org/traces/parallel?only=27917) | [link](https://ftp.pdl.cmu.edu/pub/datasets/twemcacheWorkload/cacheDatasets/tencentBlock/) | +| Alibaba Block | 2020 | block | [link](https://github.com/alibaba/block-traces) | [link](https://ftp.pdl.cmu.edu/pub/datasets/twemcacheWorkload/cacheDatasets/alibabaBlock/) | +| Twitter | 2020 | key-value | [link](https://github.com/twitter/cache-traces) | [link](https://ftp.pdl.cmu.edu/pub/datasets/twemcacheWorkload/cacheDatasets/twitter/) | +| MetaKV | 2022 | key-value | [link](https://cachelib.org/docs/Cache_Library_User_Guides/Cachebench_FB_HW_eval/#list-of-traces) | [link](https://ftp.pdl.cmu.edu/pub/datasets/twemcacheWorkload/cacheDatasets/metaKV/) | +| MetaCDN | 2023 | object | [link](https://cachelib.org/docs/Cache_Library_User_Guides/Cachebench_FB_HW_eval/#list-of-traces) | [link](https://ftp.pdl.cmu.edu/pub/datasets/twemcacheWorkload/cacheDatasets/metaCDN/) | + +Among the large number of traces, I recommend using the newer traces from Twitter (cluster52), Wiki, and Meta. + + +--- + +## Questions? Please join the Google group https://groups.google.com/g/libcachesim and ask questions. --- -### Contributions + +## Contributions We gladly welcome pull requests. -Before making any changes, we recommend opening an issue and discussing your proposed changes. -This will let us give you advice on the proposed changes. If the changes are minor, then feel free to make them without discussion. +Before making any large changes, we recommend opening an issue and discussing your proposed changes. +If the changes are minor, then feel free to make them without discussion. This project adheres to Google's coding style. By participating, you are expected to uphold this code. --- -### Reference + +## Reference ``` -@inproceedings{yang20-workload, +@inproceedings{yang2020-workload, author = {Juncheng Yang and Yao Yue and K. V. Rashmi}, title = {A large scale analysis of hundreds of in-memory cache clusters at Twitter}, booktitle = {14th USENIX Symposium on Operating Systems Design and Implementation (OSDI 20)}, @@ -217,26 +300,42 @@ This project adheres to Google's coding style. By participating, you are expecte pages = {191--208}, url = {https://www.usenix.org/conference/osdi20/presentation/yang}, publisher = {USENIX Association}, - month = nov, } @inproceedings{yang2023-s3fifo, - title={FIFO queues are all you need for cache eviction}, - author={Yang, Juncheng and Zhang, Yazhuo and Qiu, Ziyue and Yue, Yao and Rashmi, K.V.}, - booktitle={Symposium on Operating Systems Principles (SOSP'23)}, + title = {FIFO Queues Are All You Need for Cache Eviction}, + author = {Juncheng Yang and Yazhuo Zhang and Ziyue Qiu and Yao Yue and K.V. Rashmi}, + isbn = {9798400702297}, + publisher = {Association for Computing Machinery}, + booktitle = {Symposium on Operating Systems Principles (SOSP'23)}, + pages = {130–149}, + numpages = {20}, year={2023} } + +@inproceedings{yang2023-qdlp, + author = {Juncheng Yang and Ziyue Qiu and Yazhuo Zhang and Yao Yue and K.V. Rashmi}, + title = {FIFO Can Be Better than LRU: The Power of Lazy Promotion and Quick Demotion}, + year = {2023}, + isbn = {9798400701955}, + publisher = {Association for Computing Machinery}, + doi = {10.1145/3593856.3595887}, + booktitle = {Proceedings of the 19th Workshop on Hot Topics in Operating Systems (HotOS23)}, + pages = {70–79}, + numpages = {10}, +} ``` -If you used libCacheSim in your research, please cite the above papers. And we welcome you to send us a link to your paper and add reference to (references.md)[references.md]. +If you used libCacheSim in your research, please cite the above papers. And we welcome you to send us a link to your paper and add a reference to [references.md](references.md). --- -### License + +## License See [LICENSE](LICENSE) for details. -### Related + +## Related * [PyMimircache](https://github.com/1a1a11a/PyMimircache): a python based cache trace analysis platform, now deprecated --- - diff --git a/cmake/Modules/Findargp.cmake b/cmake/Modules/Findargp.cmake new file mode 100644 index 00000000..0d780260 --- /dev/null +++ b/cmake/Modules/Findargp.cmake @@ -0,0 +1,84 @@ +# - try to find the argp library/component of glibc +# +# Users may optionally supply: +# ARGP_ROOT_DIR - a prefix to start searching. +# +# Cache Variables: (probably not for direct use in your scripts) +# ARGP_INCLUDE_DIR +# ARGP_LIBRARY, only defined if linking to an extra library is required +# +# Non-cache variables you might use in your CMakeLists.txt: +# ARGP_FOUND +# ARGP_INCLUDE_DIRS +# ARGP_LIBRARIES +# +# Requires these CMake modules: +# FindPackageHandleStandardArgs (known included with CMake >=2.6.2) +# +# Original Author: +# 2009-2010 Rylie Pavlik +# https://ryliepavlik.com/ +# Iowa State University HCI Graduate Program/VRAC +# +# Copyright 2009-2010, Iowa State University +# +# Distributed under the Boost Software License, Version 1.0. +# (See accompanying file LICENSE_1_0.txt or copy at +# http://www.boost.org/LICENSE_1_0.txt) +# +# SPDX-License-Identifier: BSL-1.0 + +set(ARGP_ROOT_DIR + "${ARGP_ROOT_DIR}" + CACHE + PATH + "Path to search for ARGP library") + +### +# Configure ARGP +### +set(_check ARGP_INCLUDE_DIR) + +find_path(ARGP_INCLUDE_DIR + NAMES + argp.h + HINTS + "${ARGP_ROOT_DIR}" + PATHS + /usr/local + /opt/local + /sw) +mark_as_advanced(ARGP_INCLUDE_DIR) + +include(CheckFunctionExists) +check_function_exists(argp_parse ARGP_BUILTIN) + +if(NOT ARGP_BUILTIN) + find_library(ARGP_LIBRARY + NAMES + argp + HINTS + "${ARGP_ROOT_DIR}" + PATH_SUFFIXES + lib + lib64 + PATHS + /usr/local + /opt/local + /sw) + list(APPEND _check ARGP_LIBRARY) + mark_as_advanced(ARGP_LIBRARY) +endif() + +# handle the QUIETLY and REQUIRED arguments and set xxx_FOUND to TRUE if +# all listed variables are TRUE +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(argp + DEFAULT_MSG + ${_check}) + +if(ARGP_FOUND) + set(ARGP_INCLUDE_DIRS "${ARGP_INCLUDE_DIR}") + set(ARGP_LIBRARIES "${ARGP_LIBRARY}") + mark_as_advanced(ARGP_ROOT_DIR) +endif() \ No newline at end of file diff --git a/data/trace.csv b/data/cloudPhysicsIO.csv similarity index 100% rename from data/trace.csv rename to data/cloudPhysicsIO.csv diff --git a/data/trace.oracleGeneral.bin b/data/cloudPhysicsIO.oracleGeneral.bin similarity index 100% rename from data/trace.oracleGeneral.bin rename to data/cloudPhysicsIO.oracleGeneral.bin diff --git a/data/trace.txt b/data/cloudPhysicsIO.txt similarity index 100% rename from data/trace.txt rename to data/cloudPhysicsIO.txt diff --git a/data/trace.vscsi b/data/cloudPhysicsIO.vscsi similarity index 100% rename from data/trace.vscsi rename to data/cloudPhysicsIO.vscsi diff --git a/doc/quickstart_cachesim.md b/doc/quickstart_cachesim.md index d22556d0..6f22f605 100644 --- a/doc/quickstart_cachesim.md +++ b/doc/quickstart_cachesim.md @@ -71,7 +71,7 @@ cachesim supports the following algorithms: * [ARC](/libCacheSim/cache/eviction/ARC.c) * [SLRU](/libCacheSim/cache/eviction/SLRU.c) * [GDSF](/libCacheSim/cache/eviction/GDSF.c) -* [TinyLFU](/libCacheSim/cache/eviction/TinyLFU.c) +* [WTinyLFU](/libCacheSim/cache/eviction/WTinyLFU.c) * [LeCaR](/libCacheSim/cache/eviction/LeCaR.c) * [Cacheus](/libCacheSim/cache/eviction/Cacheus.c) * [Hyperbolic](/libCacheSim/cache/eviction/Hyperbolic.c) @@ -148,12 +148,19 @@ Some eviction algorithms have parameters, you can set the parameters by using `- ### Admission algorithm cachesim supports the following admission algorithms: size, probabilistic, bloomFilter, adaptSize. -You can use `-a` or `--admission-algo` to set the admission algorithm. +You can use `-a` or `--admission` to set the admission algorithm. ```bash # add a bloom filter to filter out objects on first access ./cachesim ../data/trace.vscsi vscsi lru 1gb -a bloomFilter ``` +### Prefetching algorithm +cachesim supports the following prefetching algorithms: OBL, Mithril, PG (and AMP is on the way). +You can use `-p` or `--prefetch` to set the prefetching algorithm. +```bash +# add a mithril to record object association information and fetch objests that are likely to be accessed in the future +./cachesim ../data/trace.vscsi vscsi lru 1gb -p Mithril +``` ### Advanced features ```bash diff --git a/doc/quickstart_traceUtils.md b/doc/quickstart_traceUtils.md index 774239f7..e837772d 100644 --- a/doc/quickstart_traceUtils.md +++ b/doc/quickstart_traceUtils.md @@ -7,21 +7,22 @@ Print requests from a trace. ```bash # print 10 requests from a trace -./tracePrint ../data/trace.vscsi vscsi -n 10 +./bin/tracePrint ../data/cloudPhysicsIO.vscsi vscsi -n 10 ``` ### traceConv -Convert a trace to libCacheSim format so it has a smaller size, contains next request time (oracle information) and runs faster. +Convert a trace to oracleGeneral format so you can run it faster (10x speedup) using less memory. Meanwhile, the generated trace has a smaller size, contains next request time. ```bash -./traceConv ../data/trace.vscsi vscsi ../data/trace.vscsi.bin +# the first parameter is the input trace, the second parameter is trace type, the third parameter is the output trace path +./bin/traceConv ../data/cloudPhysicsIO.txt txt cloudPhysicsIO.oracleGeneral.bin ``` -Note that the conversion supports all trace types including csv trace. Note that if the object id is numeric, add the option -e `obj_id_is_num=1` to the command line, which reduces memory usage. +Note that the conversion supports all trace types including csv trace. Moreover, if the object id is numeric, add the option `-e obj_id_is_num=1` to the command line, which reduces memory usage. We can also sample a trace to reduce the size of the trace. ```bash # sample 1% of the trace -./traceConv ../data/trace.vscsi vscsi ../data/trace.vscsi.bin -s 0.01 +./bin/traceConv ../data/cloudPhysicsIO.vscsi vscsi ../data/cloudPhysicsIO.oracleGeneral -s 0.01 ``` diff --git a/dockerfile b/dockerfile index 17af5b3f..9cb5f708 100644 --- a/dockerfile +++ b/dockerfile @@ -1,33 +1,27 @@ FROM ubuntu:22.04 as base MAINTAINER Juncheng -LABEL version="0.0.1" +LABEL version="0.0.2" WORKDIR / # install dependency -RUN apt update && apt-get install -yqq libglib2.0-dev libgoogle-perftools-dev cmake git sudo wget gcc g++ - -# install dependency zstd -RUN wget https://github.com/facebook/zstd/releases/download/v1.5.0/zstd-1.5.0.tar.gz > /dev/null 2>&1 && \ -tar xvf zstd-1.5.0.tar.gz >/dev/null && \ -cd zstd-1.5.0/build/cmake/ && mkdir _build && cd _build/ && \ -cmake .. && make -j && make install && \ -cd .. +# RUN apt update && apt-get install -yqq libglib2.0-dev libgoogle-perftools-dev cmake git sudo wget gcc g++ xxhash +RUN apt update && apt-get install -yqq cmake git sudo wget # clone repo RUN git clone https://github.com/1a1a11a/libCacheSim -b develop # build libCacheSim WORKDIR /libCacheSim/ -RUN mkdir _build; -# RUN bash ./scripts/install_dependency.sh -# RUN bash ./scripts/install_libcachesim.sh +# RUN mkdir _build; +RUN bash ./scripts/install_dependency.sh +RUN bash ./scripts/install_libcachesim.sh WORKDIR /libCacheSim/_build/ -RUN cmake -DSUPPORT_ZSTD_TRACE=on .. && make -j && sudo make install +# RUN cmake -DSUPPORT_ZSTD_TRACE=on .. && make -j && sudo make install WORKDIR /libCacheSim/_build/bin/ @@ -38,8 +32,9 @@ WORKDIR /libCacheSim/_build/bin/ # sudo docker build -t 1a1a11a/libcachesim -f dockerfile . # push to docker hub -# sudo docker tag 1a1a11a/libcachesim:latest 1a1a11a/libcachesim:0.0.1 +# sudo docker tag 1a1a11a/libcachesim:latest 1a1a11a/libcachesim:0.0.2 # sudo docker push 1a1a11a/libcachesim:latest +# sudo docker push 1a1a11a/libcachesim:0.0.2 # use the container # sudo docker run -v /local/data/path:/data -it 1a1a11a/libcachesim:latest bash @@ -58,5 +53,3 @@ WORKDIR /libCacheSim/_build/bin/ # sudo apt update && sudo apt-get install -yqq docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin - - diff --git a/example/cacheCluster/CMakeLists.txt b/example/cacheCluster/CMakeLists.txt index 40d6c36f..3bf99e2d 100644 --- a/example/cacheCluster/CMakeLists.txt +++ b/example/cacheCluster/CMakeLists.txt @@ -1,29 +1,29 @@ cmake_minimum_required (VERSION 3.2) project (cacheCluster) -set(CMAKE_BUILD_TYPE Debug) -set(CMAKE_CXX_STANDARD 11) +# set(CMAKE_BUILD_TYPE Debug) +set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) -# clear an error when using cmake in macOS -# if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") -# set(CMAKE_MACOSX_RPATH 1) -# # Fix linking on 10.14+. See https://stackoverflow.com/questions/54068035 -# LINK_DIRECTORIES(/usr/local/lib) -# # a temporary fix for mac -# SET(CMAKE_C_FLAGS_DEBUG " -L/usr/local/lib -lglib-2.0 -lintl -Wno-unused-command-line-argument") -# endif() - set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/../../cmake/Modules/") find_package(GLib "2.40" REQUIRED) include_directories(${GLib_INCLUDE_DIRS}) message(STATUS "glib found? " ${GLib_FOUND} ", GLIB = " ${GLib_LIBRARY} ", header = " ${GLib_INCLUDE_DIRS}) + +find_package(ZSTD) +# https://stackoverflow.com/questions/61377055/cannot-find-gflags-gflags-h-while-building-library-osx/61379123#61379123 +include_directories(${ZSTD_INCLUDE_DIR}) +if ("${ZSTD_LIBRARIES}" STREQUAL "") + message(FATAL_ERROR "zstd not found") +endif() + + ################ this mode compiles the external cache replacement algorithm together with the simulator ############ message(STATUS "project_source dir = " ${PROJECT_SOURCE_DIR}) file(GLOB ALL_CXX_SRC ${PROJECT_SOURCE_DIR}/*.cpp) file(GLOB ALL_C_SRC ${PROJECT_SOURCE_DIR}/*.c) add_executable(cacheCluster ${ALL_CXX_SRC} ${ALL_C_SRC}) -target_link_libraries(cacheCluster libCacheSim m ${GLib_LIBRARY} pthread dl) +target_link_libraries(cacheCluster libCacheSim m ${GLib_LIBRARY} pthread dl ${ZSTD_LIBRARIES}) diff --git a/example/cacheCluster/main.cpp b/example/cacheCluster/main.cpp index 008d2882..747e1ac7 100644 --- a/example/cacheCluster/main.cpp +++ b/example/cacheCluster/main.cpp @@ -13,9 +13,11 @@ namespace CDNSimulator { void simulate(int argc, char *argv[]) { - const char *data_path = "../../../data/trace.csv"; + const char *data_path = "../../../data/twitter_cluster52.csv"; if (argc > 1) { data_path = argv[1]; + } else { + printf("use default data at ../../../data/twitter_cluster52.csv\n"); } if (access(data_path, F_OK) == -1) { @@ -25,9 +27,9 @@ void simulate(int argc, char *argv[]) { /* setup a csv reader */ reader_init_param_t init_params = default_reader_init_params(); - init_params.obj_id_field = 5; - init_params.obj_size_field = 4; - init_params.time_field = 2; + init_params.obj_id_field = 2; + init_params.obj_size_field = 3; + init_params.time_field = 1; init_params.has_header_set = true; init_params.has_header = true; init_params.delimiter = ','; @@ -36,12 +38,17 @@ void simulate(int argc, char *argv[]) { reader_t *reader = open_trace(data_path, CSV_TRACE, &init_params); const uint64_t n_server = 10; - const uint64_t server_dram_cache_size = 8 * GiB; - const uint64_t server_disk_cache_size = 100 * GiB; + const uint64_t server_dram_cache_size = 1 * MiB; + const uint64_t server_disk_cache_size = 10 * MiB; const std::string algo = "lru"; // each cache holds 2 ** 20 objects, this is for performance optimization // you can specify a smaller number to save memory const uint32_t hashpower = 20; + printf( + "setting up a cluster of %lu servers, each server has %lu MB DRAM cache " + "and %lu MB disk cache, using %s as cache replacement algorithm\n", + (unsigned long)n_server, (unsigned long)(server_dram_cache_size / MiB), + (unsigned long)(server_disk_cache_size / MiB), algo.c_str()); CacheCluster cluster(0); @@ -68,10 +75,8 @@ void simulate(int argc, char *argv[]) { n_req_byte += req->obj_size; } - std::cout << n_req << " requests, " << n_miss - << " misses, miss ratio: " << (double)n_miss / n_req - << ", byte miss ratio: " << (double)n_miss_byte / n_req_byte - << std::endl; + std::cout << n_req << " requests, " << n_miss << " misses, miss ratio: " << (double)n_miss / n_req + << ", byte miss ratio: " << (double)n_miss_byte / n_req_byte << std::endl; close_trace(reader); } diff --git a/example/cacheHierarchy/CMakeLists.txt b/example/cacheHierarchy/CMakeLists.txt index cb4fa113..19a62c31 100644 --- a/example/cacheHierarchy/CMakeLists.txt +++ b/example/cacheHierarchy/CMakeLists.txt @@ -1,33 +1,28 @@ cmake_minimum_required (VERSION 3.2) project (layeredCache) - -set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) message("****************** this example only works after libCacheSim has been installed ******************") -#### -## export C_INCLUDE_PATH=/usr/local/include -## export CPLUS_INCLUDE_PATH=/usr/local/include - -# clear an error when using cmake in macOS -if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") - set(CMAKE_MACOSX_RPATH 1) - # Fix linking on 10.14+. See https://stackoverflow.com/questions/54068035 - LINK_DIRECTORIES(/usr/local/lib) - # a temporary fix for mac - SET(CMAKE_C_FLAGS_DEBUG " -L/usr/local/lib -lglib-2.0 -lintl -Wno-unused-command-line-argument") -endif() - - -set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/Modules/") -# find libglib2.0-dev +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/../../cmake/Modules/") find_package(GLib "2.40" REQUIRED) include_directories(${GLib_INCLUDE_DIRS}) message(STATUS "glib found? " ${GLib_FOUND} ", GLIB = " ${GLib_LIBRARY} ", header = " ${GLib_INCLUDE_DIRS}) -# set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "/usr/local/share/cmake/yaml-cpp/") +find_package(ZSTD) +include_directories(${ZSTD_INCLUDE_DIR}) +if ("${ZSTD_LIBRARIES}" STREQUAL "") + message(FATAL_ERROR "zstd not found") +endif() + + +# https://github.com/jbeder/yaml-cpp +# wget https://github.com/jbeder/yaml-cpp/archive/refs/tags/0.8.0.tar.gz +# tar -xzf 0.8.0.tar.gz; mkdir yaml-cpp-0.8.0/_build; cd yaml-cpp-0.8.0/_build; +# cmake -DBUILD_SHARED_LIBS=ON ..; +# make -j; sudo make install find_package(YAML-CPP REQUIRED) # set(YAML_CPP_INCLUDE_DIR /usr/local/include/) # set(YAML_CPP_LIBRARIES /usr/local/lib/) @@ -38,10 +33,5 @@ message(STATUS ${YAML_CPP_INCLUDE_DIR} ";" ${YAML_CPP_LIBRARIES}) file(GLOB ALL_SRC ${PROJECT_SOURCE_DIR}/*.cpp) message(STATUS "all sources " ${ALL_SRC}) add_executable(layeredCache ${ALL_SRC}) -target_link_libraries(layeredCache yaml-cpp libCacheSim m ${GLib_LIBRARY}) - - - - - +target_link_libraries(layeredCache yaml-cpp libCacheSim dl m ${GLib_LIBRARY} ${ZSTD_LIBRARIES} pthread) diff --git a/example/cacheHierarchy/cmake/Modules/FindGLib.cmake b/example/cacheHierarchy/cmake/Modules/FindGLib.cmake deleted file mode 100644 index 8939d857..00000000 --- a/example/cacheHierarchy/cmake/Modules/FindGLib.cmake +++ /dev/null @@ -1,83 +0,0 @@ -# FindGLib.cmake -# -# -# CMake support for GLib/GObject/GIO. -# -# License: -# -# Copyright (c) 2016 Evan Nemerson -# -# Permission is hereby granted, free of charge, to any person -# obtaining a copy of this software and associated documentation -# files (the "Software"), to deal in the Software without -# restriction, including without limitation the rights to use, copy, -# modify, merge, publish, distribute, sublicense, and/or sell copies -# of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -# DEALINGS IN THE SOFTWARE. - -find_package(PkgConfig) - -if(PKG_CONFIG_FOUND) - pkg_search_module(GLib_PKG glib-2.0) -endif() - -find_library(GLib_LIBRARY glib-2.0 HINTS ${GLib_PKG_LIBRARY_DIRS}) -set(GLib glib-2.0) - -if(GLib_LIBRARY AND NOT GLib_FOUND) - add_library(${GLib} SHARED IMPORTED) - set_property(TARGET ${GLib} PROPERTY IMPORTED_LOCATION "${GLib_LIBRARY}") - set_property(TARGET ${GLib} PROPERTY INTERFACE_COMPILE_OPTIONS "${GLib_PKG_CFLAGS_OTHER}") - - find_path(GLib_INCLUDE_DIRS "glib.h" - HINTS ${GLib_PKG_INCLUDE_DIRS} - PATH_SUFFIXES "glib-2.0") - - get_filename_component(GLib_LIBDIR "${GLib}" DIRECTORY) - find_path(GLib_CONFIG_INCLUDE_DIR "glibconfig.h" - HINTS - ${GLib_LIBDIR} - ${GLib_PKG_INCLUDE_DIRS} - PATHS - "${CMAKE_LIBRARY_PATH}" - PATH_SUFFIXES - "glib-2.0/include" - "glib-2.0") - unset(GLib_LIBDIR) - - if(GLib_CONFIG_INCLUDE_DIR) - file(STRINGS "${GLib_CONFIG_INCLUDE_DIR}/glibconfig.h" GLib_MAJOR_VERSION REGEX "^#define GLIB_MAJOR_VERSION +([0-9]+)") - string(REGEX REPLACE "^#define GLIB_MAJOR_VERSION ([0-9]+)$" "\\1" GLib_MAJOR_VERSION "${GLib_MAJOR_VERSION}") - file(STRINGS "${GLib_CONFIG_INCLUDE_DIR}/glibconfig.h" GLib_MINOR_VERSION REGEX "^#define GLIB_MINOR_VERSION +([0-9]+)") - string(REGEX REPLACE "^#define GLIB_MINOR_VERSION ([0-9]+)$" "\\1" GLib_MINOR_VERSION "${GLib_MINOR_VERSION}") - file(STRINGS "${GLib_CONFIG_INCLUDE_DIR}/glibconfig.h" GLib_MICRO_VERSION REGEX "^#define GLIB_MICRO_VERSION +([0-9]+)") - string(REGEX REPLACE "^#define GLIB_MICRO_VERSION ([0-9]+)$" "\\1" GLib_MICRO_VERSION "${GLib_MICRO_VERSION}") - set(GLib_VERSION "${GLib_MAJOR_VERSION}.${GLib_MINOR_VERSION}.${GLib_MICRO_VERSION}") - unset(GLib_MAJOR_VERSION) - unset(GLib_MINOR_VERSION) - unset(GLib_MICRO_VERSION) - - list(APPEND GLib_INCLUDE_DIRS ${GLib_CONFIG_INCLUDE_DIR}) - set_property(TARGET ${GLib} PROPERTY INTERFACE_INCLUDE_DIRECTORIES "${GLib_INCLUDE_DIRS}") - endif() -endif() - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(GLib - REQUIRED_VARS - GLib_LIBRARY - GLib_INCLUDE_DIRS - VERSION_VAR - GLib_VERSION) diff --git a/example/cacheSimulator/CMakeLists.txt b/example/cacheSimulator/CMakeLists.txt index 49775ead..fbf63fcb 100644 --- a/example/cacheSimulator/CMakeLists.txt +++ b/example/cacheSimulator/CMakeLists.txt @@ -4,27 +4,23 @@ project (cacheSimulator) message("****************** this example only works after libCacheSim has been installed ******************") - -# clear an error when using cmake in macOS -# if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") -# set(CMAKE_MACOSX_RPATH 1) -# # Fix linking on 10.14+. See https://stackoverflow.com/questions/54068035 -# LINK_DIRECTORIES(/usr/local/lib) -# # a temporary fix for mac -# SET(CMAKE_C_FLAGS_DEBUG " -L/usr/local/lib -lglib-2.0 -lintl -Wno-unused-command-line-argument") -# endif() - find_library(libCacheSim libCacheSim) find_path(libCacheSimInclude libCacheSim) message(STATUS "libCacheSim found? ${libCacheSim} ${libCacheSimInclude}") include_directories(${libCacheSimInclude}) - -set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/Modules/") +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/../../cmake/Modules/") find_package(GLib "2.40" REQUIRED) include_directories(${GLib_INCLUDE_DIRS}) message(STATUS "glib found? ${GLib_FOUND}, lib = ${GLib_LIBRARY}, header = ${GLib_INCLUDE_DIRS}") +find_package(ZSTD) +# https://stackoverflow.com/questions/61377055/cannot-find-gflags-gflags-h-while-building-library-osx/61379123#61379123 +include_directories(${ZSTD_INCLUDE_DIR}) +if ("${ZSTD_LIBRARIES}" STREQUAL "") + message(FATAL_ERROR "zstd not found") +endif() + add_executable(cacheSimulator main.c) -target_link_libraries(cacheSimulator libCacheSim m ${GLib_LIBRARY}) +target_link_libraries(cacheSimulator libCacheSim m ${GLib_LIBRARY} ${ZSTD_LIBRARIES}) diff --git a/example/cacheSimulator/cmake/Modules/FindGLib.cmake b/example/cacheSimulator/cmake/Modules/FindGLib.cmake deleted file mode 100644 index 8939d857..00000000 --- a/example/cacheSimulator/cmake/Modules/FindGLib.cmake +++ /dev/null @@ -1,83 +0,0 @@ -# FindGLib.cmake -# -# -# CMake support for GLib/GObject/GIO. -# -# License: -# -# Copyright (c) 2016 Evan Nemerson -# -# Permission is hereby granted, free of charge, to any person -# obtaining a copy of this software and associated documentation -# files (the "Software"), to deal in the Software without -# restriction, including without limitation the rights to use, copy, -# modify, merge, publish, distribute, sublicense, and/or sell copies -# of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -# DEALINGS IN THE SOFTWARE. - -find_package(PkgConfig) - -if(PKG_CONFIG_FOUND) - pkg_search_module(GLib_PKG glib-2.0) -endif() - -find_library(GLib_LIBRARY glib-2.0 HINTS ${GLib_PKG_LIBRARY_DIRS}) -set(GLib glib-2.0) - -if(GLib_LIBRARY AND NOT GLib_FOUND) - add_library(${GLib} SHARED IMPORTED) - set_property(TARGET ${GLib} PROPERTY IMPORTED_LOCATION "${GLib_LIBRARY}") - set_property(TARGET ${GLib} PROPERTY INTERFACE_COMPILE_OPTIONS "${GLib_PKG_CFLAGS_OTHER}") - - find_path(GLib_INCLUDE_DIRS "glib.h" - HINTS ${GLib_PKG_INCLUDE_DIRS} - PATH_SUFFIXES "glib-2.0") - - get_filename_component(GLib_LIBDIR "${GLib}" DIRECTORY) - find_path(GLib_CONFIG_INCLUDE_DIR "glibconfig.h" - HINTS - ${GLib_LIBDIR} - ${GLib_PKG_INCLUDE_DIRS} - PATHS - "${CMAKE_LIBRARY_PATH}" - PATH_SUFFIXES - "glib-2.0/include" - "glib-2.0") - unset(GLib_LIBDIR) - - if(GLib_CONFIG_INCLUDE_DIR) - file(STRINGS "${GLib_CONFIG_INCLUDE_DIR}/glibconfig.h" GLib_MAJOR_VERSION REGEX "^#define GLIB_MAJOR_VERSION +([0-9]+)") - string(REGEX REPLACE "^#define GLIB_MAJOR_VERSION ([0-9]+)$" "\\1" GLib_MAJOR_VERSION "${GLib_MAJOR_VERSION}") - file(STRINGS "${GLib_CONFIG_INCLUDE_DIR}/glibconfig.h" GLib_MINOR_VERSION REGEX "^#define GLIB_MINOR_VERSION +([0-9]+)") - string(REGEX REPLACE "^#define GLIB_MINOR_VERSION ([0-9]+)$" "\\1" GLib_MINOR_VERSION "${GLib_MINOR_VERSION}") - file(STRINGS "${GLib_CONFIG_INCLUDE_DIR}/glibconfig.h" GLib_MICRO_VERSION REGEX "^#define GLIB_MICRO_VERSION +([0-9]+)") - string(REGEX REPLACE "^#define GLIB_MICRO_VERSION ([0-9]+)$" "\\1" GLib_MICRO_VERSION "${GLib_MICRO_VERSION}") - set(GLib_VERSION "${GLib_MAJOR_VERSION}.${GLib_MINOR_VERSION}.${GLib_MICRO_VERSION}") - unset(GLib_MAJOR_VERSION) - unset(GLib_MINOR_VERSION) - unset(GLib_MICRO_VERSION) - - list(APPEND GLib_INCLUDE_DIRS ${GLib_CONFIG_INCLUDE_DIR}) - set_property(TARGET ${GLib} PROPERTY INTERFACE_INCLUDE_DIRECTORIES "${GLib_INCLUDE_DIRS}") - endif() -endif() - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(GLib - REQUIRED_VARS - GLib_LIBRARY - GLib_INCLUDE_DIRS - VERSION_VAR - GLib_VERSION) diff --git a/example/cacheSimulatorConcurrent/CMakeLists.txt b/example/cacheSimulatorConcurrent/CMakeLists.txt index d5f60a93..1494124d 100644 --- a/example/cacheSimulatorConcurrent/CMakeLists.txt +++ b/example/cacheSimulatorConcurrent/CMakeLists.txt @@ -1,18 +1,27 @@ -cmake_minimum_required (VERSION 2.8) +cmake_minimum_required (VERSION 3.2) project (cacheSimulatorMultiSize) -# set(CMAKE_BUILD_TYPE Debug) +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) -set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/Modules/") find_library(libCacheSim libCacheSim) find_path(libCacheSimInclude libCacheSim) include_directories(${libCacheSimInclude}) message(STATUS "libCacheSim found? ${libCacheSim}, ${libCacheSimInclude}") + +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/../../cmake/Modules/") find_package(GLib "2.40" REQUIRED) include_directories(${GLib_INCLUDE_DIRS}) message(STATUS "glib found? ${GLib_FOUND}, lib = ${GLib_LIBRARY}, header = ${GLib_INCLUDE_DIRS}") +find_package(ZSTD) +# https://stackoverflow.com/questions/61377055/cannot-find-gflags-gflags-h-while-building-library-osx/61379123#61379123 +include_directories(${ZSTD_INCLUDE_DIR}) +if ("${ZSTD_LIBRARIES}" STREQUAL "") + message(FATAL_ERROR "zstd not found") +endif() add_executable(cacheSimulatorMultiSize main.cpp) -target_link_libraries(cacheSimulatorMultiSize libCacheSim) +target_link_libraries(cacheSimulatorMultiSize libCacheSim m ${GLib_LIBRARY} ${ZSTD_LIBRARIES}) + diff --git a/example/cacheSimulatorConcurrent/README.md b/example/cacheSimulatorConcurrent/README.md index 73619b35..4093815f 100644 --- a/example/cacheSimulatorConcurrent/README.md +++ b/example/cacheSimulatorConcurrent/README.md @@ -1,7 +1,5 @@ ## Building a concurrent cache simulator -> WARN -> this has not been updated This tutorial shows how to build a concurrent cache simulator using libCacheSim. diff --git a/example/cacheSimulatorConcurrent/cmake/Modules/FindGLib.cmake b/example/cacheSimulatorConcurrent/cmake/Modules/FindGLib.cmake deleted file mode 100644 index 8939d857..00000000 --- a/example/cacheSimulatorConcurrent/cmake/Modules/FindGLib.cmake +++ /dev/null @@ -1,83 +0,0 @@ -# FindGLib.cmake -# -# -# CMake support for GLib/GObject/GIO. -# -# License: -# -# Copyright (c) 2016 Evan Nemerson -# -# Permission is hereby granted, free of charge, to any person -# obtaining a copy of this software and associated documentation -# files (the "Software"), to deal in the Software without -# restriction, including without limitation the rights to use, copy, -# modify, merge, publish, distribute, sublicense, and/or sell copies -# of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -# DEALINGS IN THE SOFTWARE. - -find_package(PkgConfig) - -if(PKG_CONFIG_FOUND) - pkg_search_module(GLib_PKG glib-2.0) -endif() - -find_library(GLib_LIBRARY glib-2.0 HINTS ${GLib_PKG_LIBRARY_DIRS}) -set(GLib glib-2.0) - -if(GLib_LIBRARY AND NOT GLib_FOUND) - add_library(${GLib} SHARED IMPORTED) - set_property(TARGET ${GLib} PROPERTY IMPORTED_LOCATION "${GLib_LIBRARY}") - set_property(TARGET ${GLib} PROPERTY INTERFACE_COMPILE_OPTIONS "${GLib_PKG_CFLAGS_OTHER}") - - find_path(GLib_INCLUDE_DIRS "glib.h" - HINTS ${GLib_PKG_INCLUDE_DIRS} - PATH_SUFFIXES "glib-2.0") - - get_filename_component(GLib_LIBDIR "${GLib}" DIRECTORY) - find_path(GLib_CONFIG_INCLUDE_DIR "glibconfig.h" - HINTS - ${GLib_LIBDIR} - ${GLib_PKG_INCLUDE_DIRS} - PATHS - "${CMAKE_LIBRARY_PATH}" - PATH_SUFFIXES - "glib-2.0/include" - "glib-2.0") - unset(GLib_LIBDIR) - - if(GLib_CONFIG_INCLUDE_DIR) - file(STRINGS "${GLib_CONFIG_INCLUDE_DIR}/glibconfig.h" GLib_MAJOR_VERSION REGEX "^#define GLIB_MAJOR_VERSION +([0-9]+)") - string(REGEX REPLACE "^#define GLIB_MAJOR_VERSION ([0-9]+)$" "\\1" GLib_MAJOR_VERSION "${GLib_MAJOR_VERSION}") - file(STRINGS "${GLib_CONFIG_INCLUDE_DIR}/glibconfig.h" GLib_MINOR_VERSION REGEX "^#define GLIB_MINOR_VERSION +([0-9]+)") - string(REGEX REPLACE "^#define GLIB_MINOR_VERSION ([0-9]+)$" "\\1" GLib_MINOR_VERSION "${GLib_MINOR_VERSION}") - file(STRINGS "${GLib_CONFIG_INCLUDE_DIR}/glibconfig.h" GLib_MICRO_VERSION REGEX "^#define GLIB_MICRO_VERSION +([0-9]+)") - string(REGEX REPLACE "^#define GLIB_MICRO_VERSION ([0-9]+)$" "\\1" GLib_MICRO_VERSION "${GLib_MICRO_VERSION}") - set(GLib_VERSION "${GLib_MAJOR_VERSION}.${GLib_MINOR_VERSION}.${GLib_MICRO_VERSION}") - unset(GLib_MAJOR_VERSION) - unset(GLib_MINOR_VERSION) - unset(GLib_MICRO_VERSION) - - list(APPEND GLib_INCLUDE_DIRS ${GLib_CONFIG_INCLUDE_DIR}) - set_property(TARGET ${GLib} PROPERTY INTERFACE_INCLUDE_DIRECTORIES "${GLib_INCLUDE_DIRS}") - endif() -endif() - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(GLib - REQUIRED_VARS - GLib_LIBRARY - GLib_INCLUDE_DIRS - VERSION_VAR - GLib_VERSION) diff --git a/example/cacheSimulatorConcurrent/main.cpp b/example/cacheSimulatorConcurrent/main.cpp index f05f0673..1b583102 100644 --- a/example/cacheSimulatorConcurrent/main.cpp +++ b/example/cacheSimulatorConcurrent/main.cpp @@ -3,7 +3,7 @@ #include #define NUM_SIZES 8 -const char *TRACE_PATH = "../../../data/trace.csv"; +const char *TRACE_PATH = "../../../data/cloudPhysicsIO.csv"; void run_one_cache_multiple_sizes(cache_t *cache, reader_t *reader) { uint64_t cache_sizes[NUM_SIZES] = {100, 200, 400, 800, @@ -44,7 +44,7 @@ void run_multiple_caches(reader_t *reader) { cache_t *caches[8] = { LRU_init(cc_params, nullptr), LFU_init(cc_params, nullptr), - FIFO_init(cc_params, nullptr), SLRU_init(cc_params, nullptr), + FIFO_init(cc_params, nullptr), Sieve_init(cc_params, nullptr), LHD_init(cc_params, nullptr), LeCaR_init(cc_params, nullptr), ARC_init(cc_params, nullptr), NULL}; @@ -55,7 +55,7 @@ void run_multiple_caches(reader_t *reader) { cache_stat_t *result = simulate_with_multi_caches( reader, caches, 8, nullptr, 0.0, 0, - static_cast(std::thread::hardware_concurrency())); + static_cast(std::thread::hardware_concurrency()), 0); printf( " cache name cache size num_miss num_req" diff --git a/libCacheSim/bin/cachesim/cache_init.h b/libCacheSim/bin/cachesim/cache_init.h index b4c2668b..6c372117 100644 --- a/libCacheSim/bin/cachesim/cache_init.h +++ b/libCacheSim/bin/cachesim/cache_init.h @@ -39,6 +39,10 @@ static inline cache_t *create_cache(const char *trace_path, cache = ARCv0_init(cc_params, eviction_params); } else if (strcasecmp(eviction_algo, "lhd") == 0) { cache = LHD_init(cc_params, eviction_params); + } else if (strcasecmp(eviction_algo, "random") == 0) { + cache = Random_init(cc_params, eviction_params); + } else if (strcasecmp(eviction_algo, "randomTwo") == 0) { + cache = RandomTwo_init(cc_params, eviction_params); } else if (strcasecmp(eviction_algo, "lfu") == 0) { cache = LFU_init(cc_params, eviction_params); } else if (strcasecmp(eviction_algo, "gdsf") == 0) { @@ -83,6 +87,10 @@ static inline cache_t *create_cache(const char *trace_path, } else if (strcasecmp(eviction_algo, "belady") == 0) { if (strcasestr(trace_path, "oracleGeneral") == NULL) { WARN("belady is only supported for oracleGeneral trace\n"); + WARN("to convert a trace to oracleGeneral format\n"); + WARN("./bin/traceConv input_trace trace_format output_trace\n"); + WARN("./bin/traceConv ../data/cloudPhysicsIO.txt txt cloudPhysicsIO.oracleGeneral.bin\n"); + exit(1); } cache = Belady_init(cc_params, eviction_params); } else if (strcasecmp(eviction_algo, "nop") == 0) { @@ -102,8 +110,8 @@ static inline cache_t *create_cache(const char *trace_path, } else if (strcasecmp(eviction_algo, "fifomerge") == 0 || strcasecmp(eviction_algo, "fifo-merge") == 0) { cache = FIFO_Merge_init(cc_params, eviction_params); - // } else if (strcasecmp(eviction_algo, "fifo-reinsertion") == 0) { - // cache = FIFO_Reinsertion_init(cc_params, eviction_params); + // } else if (strcasecmp(eviction_algo, "fifo-reinsertion") == 0) { + // cache = FIFO_Reinsertion_init(cc_params, eviction_params); } else if (strcasecmp(eviction_algo, "flashProb") == 0) { // used to measure application level write amp cache = flashProb_init(cc_params, eviction_params); @@ -121,7 +129,8 @@ static inline cache_t *create_cache(const char *trace_path, cache = Sieve_Belady_init(cc_params, eviction_params); } else if (strcasecmp(eviction_algo, "s3lru") == 0) { cache = S3LRU_init(cc_params, eviction_params); - } else if (strcasecmp(eviction_algo, "s3fifo") == 0) { + } else if (strcasecmp(eviction_algo, "s3fifo") == 0 || + strcasecmp(eviction_algo, "s3-fifo") == 0) { cache = S3FIFO_init(cc_params, eviction_params); } else if (strcasecmp(eviction_algo, "s3fifod") == 0) { cache = S3FIFOd_init(cc_params, eviction_params); @@ -139,8 +148,6 @@ static inline cache_t *create_cache(const char *trace_path, cache = LRB_init(cc_params, eviction_params); #endif #ifdef INCLUDE_PRIV - } else if (strcasecmp(eviction_algo, "myclock") == 0) { - cache = MyClock_init(cc_params, eviction_params); } else if (strcasecmp(eviction_algo, "mclock") == 0) { cache = MClock_init(cc_params, eviction_params); } else if (strcasecmp(eviction_algo, "lp-sfifo") == 0) { diff --git a/libCacheSim/bin/cachesim/cli_parser.c b/libCacheSim/bin/cachesim/cli_parser.c index a2299bfc..caf0a0d8 100644 --- a/libCacheSim/bin/cachesim/cli_parser.c +++ b/libCacheSim/bin/cachesim/cli_parser.c @@ -72,7 +72,7 @@ static struct argp_option options[] = { {"admission-params", OPTION_ADMISSION_PARAMS, "\"prob=0.8\"", 0, "params for admission algorithm", 4}, {"prefetch", OPTION_PREFETCH_ALGO, "Mithril", 0, - "Prefetching algorithm: Mithril", 4}, + "Prefetching algorithm: Mithril/OBL/PG/AMP", 4}, {"prefetch-params", OPTION_PREFETCH_PARAMS, "\"block-size=65536\"", 0, "optional params for each prefetching algorithm, e.g., block-size=65536", 4}, @@ -107,7 +107,8 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { case OPTION_NUM_THREAD: arguments->n_thread = atoi(arg); if (arguments->n_thread == 0 || arguments->n_thread == -1) { - arguments->n_thread = n_cores(); + // arguments->n_thread = n_cores(); + arguments->n_thread = 1; } break; case OPTION_TRACE_TYPE_PARAMS: @@ -221,7 +222,8 @@ static void init_arg(struct arguments *args) { args->ignore_obj_size = false; args->consider_obj_metadata = false; args->report_interval = 3600 * 24; - args->n_thread = n_cores(); + // args->n_thread = n_cores(); + args->n_thread = 1; args->warmup_sec = -1; memset(args->ofilepath, 0, OFILEPATH_LEN); args->n_req = -1; @@ -477,7 +479,7 @@ static void set_cache_size(struct arguments *args, reader_t *reader) { args->n_cache_size = n_cache_sizes; if (args->n_cache_size == 0) { - printf("working set %ld too small\n", wss); + printf("working set %ld too small\n", (long) wss); exit(0); } } @@ -494,7 +496,7 @@ void print_parsed_args(struct arguments *args) { output_str, OUTPUT_STR_LEN - 1, "trace path: %s, trace_type %s, ofilepath " "%s, %d threads, warmup %d sec, total %d algo x %d size = %d caches", - args->trace_path, trace_type_str[args->trace_type], args->ofilepath, + args->trace_path, g_trace_type_name[args->trace_type], args->ofilepath, args->n_thread, args->warmup_sec, args->n_eviction_algo, args->n_cache_size, args->n_eviction_algo * args->n_cache_size); diff --git a/libCacheSim/bin/cachesim/internal.h b/libCacheSim/bin/cachesim/internal.h index 115f473d..a40bc89f 100644 --- a/libCacheSim/bin/cachesim/internal.h +++ b/libCacheSim/bin/cachesim/internal.h @@ -56,7 +56,7 @@ void parse_cmd(int argc, char *argv[], struct arguments *args); void free_arg(struct arguments *args); -void simulate(reader_t *reader, cache_t *cache, int report_interval, +void simulate(reader_t *reader, cache_t *cache, int report_interval, int num_of_threads, int warmup_sec, char *ofilepath); void print_parsed_args(struct arguments *args); diff --git a/libCacheSim/bin/cachesim/main.c b/libCacheSim/bin/cachesim/main.c index e83bf76a..8b3d1f0b 100644 --- a/libCacheSim/bin/cachesim/main.c +++ b/libCacheSim/bin/cachesim/main.c @@ -19,14 +19,14 @@ int main(int argc, char **argv) { if (args.n_cache_size == 0) { ERROR("no cache size found\n"); } + // if (args.n_cache_size * args.n_eviction_algo == 1) { + // INFO("simulate with single cache\n"); + // simulate(args.reader, args.caches[0], args.report_interval, args.warmup_sec, args.n_thread, + // args.ofilepath); - if (args.n_cache_size * args.n_eviction_algo == 1) { - simulate(args.reader, args.caches[0], args.report_interval, args.warmup_sec, - args.ofilepath); - - free_arg(&args); - return 0; - } + // free_arg(&args); + // return 0; + // } // cache_stat_t *result = simulate_at_multi_sizes( // args.reader, args.cache, args.n_cache_size, args.cache_sizes, NULL, 0, @@ -61,12 +61,13 @@ int main(int argc, char **argv) { for (int i = 0; i < args.n_cache_size * args.n_eviction_algo; i++) { snprintf(output_str, 1024, "%s %32s cache size %8ld%s, %lld req, miss ratio %.4lf, byte miss " - "ratio %.4lf\n", + "ratio %.4lf, throughput %.2lf MQPS\n", output_filename, result[i].cache_name, - (long)result[i].cache_size / size_unit, size_unit_str, + (long)(result[i].cache_size / size_unit), size_unit_str, (long long)result[i].n_req, (double)result[i].n_miss / (double)result[i].n_req, - (double)result[i].n_miss_byte / (double)result[i].n_req_byte); + (double)result[i].n_miss_byte / (double)result[i].n_req_byte, + (double)result[i].n_req / 1000000.0 / result[i].runtime); printf("%s", output_str); fprintf(output_file, "%s", output_str); } diff --git a/libCacheSim/bin/cachesim/sim.c b/libCacheSim/bin/cachesim/sim.c index 56519748..15a5c13e 100644 --- a/libCacheSim/bin/cachesim/sim.c +++ b/libCacheSim/bin/cachesim/sim.c @@ -89,7 +89,8 @@ void simulate(reader_t *reader, cache_t *cache, int report_interval, #if defined(TRACK_EVICTION_V_AGE) while (cache->get_occupied_byte(cache) > 0) { - cache->evict(cache, req); + cache_obj_t* obj_evicted = cache->evict(cache, req); + my_free(sizeof(cache_obj_t), obj_evicted); } #endif diff --git a/libCacheSim/bin/cli_reader_utils.c b/libCacheSim/bin/cli_reader_utils.c index 748ad548..a38cde0c 100644 --- a/libCacheSim/bin/cli_reader_utils.c +++ b/libCacheSim/bin/cli_reader_utils.c @@ -1,13 +1,12 @@ #define _GNU_SOURCE -#include "cli_reader_utils.h" - #include #include #include "../include/libCacheSim/reader.h" #include "../utils/include/mystr.h" +#include "cli_reader_utils.h" #ifdef __cplusplus extern "C" { @@ -60,11 +59,12 @@ trace_type_e trace_type_str_to_enum(const char *trace_type_str, return ORACLE_CF1_TRACE; } else if (strcasecmp(trace_type_str, "oracleSysTwrNS") == 0) { return ORACLE_SYS_TWRNS_TRACE; - } else if (strcasecmp(trace_type_str, "valpinTrace") == 0){ + } else if (strcasecmp(trace_type_str, "valpinTrace") == 0) { return VALPIN_TRACE; } else { ERROR("unsupported trace type: %s\n", trace_type_str); } + return UNKNOWN_TRACE; } bool is_true(const char *arg) { @@ -215,7 +215,7 @@ trace_type_e detect_trace_type(const char *trace_path) { trace_type = UNKNOWN_TRACE; } - INFO("detecting trace type: %s\n", trace_type_str[trace_type]); + INFO("detecting trace type: %s\n", g_trace_type_name[trace_type]); return trace_type; } @@ -276,7 +276,8 @@ void cal_working_set_size(reader_t *reader, int64_t *wss_obj, } *wss_obj *= scaling_factor; *wss_byte *= scaling_factor; - INFO("working set size: %ld object %ld byte\n", *wss_obj, *wss_byte); + INFO("working set size: %ld object %ld byte\n", (long)*wss_obj, + (long)*wss_byte); free_request(req); reset_reader(reader); diff --git a/libCacheSim/bin/traceAnalyzer/cli_parser.cpp b/libCacheSim/bin/traceAnalyzer/cli_parser.cpp index d7a59940..a694bcb9 100644 --- a/libCacheSim/bin/traceAnalyzer/cli_parser.cpp +++ b/libCacheSim/bin/traceAnalyzer/cli_parser.cpp @@ -61,9 +61,9 @@ enum argp_option_short { static struct argp_option options[] = { {NULL, 0, NULL, 0, "trace reader related parameters:", 0}, {"trace-type-params", OPTION_TRACE_TYPE_PARAMS, - "time-col=1,obj-id-col=2,obj-size-col=3,delimiter=,", NULL, + "time-col=1,obj-id-col=2,obj-size-col=3,delimiter=,", 0, "Parameters used for csv trace", 1}, - {"num-req", OPTION_NUM_REQ, 0, NULL, + {"num-req", OPTION_NUM_REQ, 0, 0, "Num of requests to process, default -1 means all requests in the trace", 1}, @@ -112,7 +112,7 @@ static struct argp_option options[] = { {NULL, 0, NULL, 0, "common parameters:", 0}, - {"output", OPTION_OUTPUT_PATH, "", NULL, "Output path", 8}, + {"output", OPTION_OUTPUT_PATH, "", OPTION_ARG_OPTIONAL, "Output path", 8}, {"verbose", OPTION_VERBOSE, NULL, OPTION_ARG_OPTIONAL, "Produce verbose output", 8}, {0}}; diff --git a/libCacheSim/bin/traceUtils/cli_parser.cpp b/libCacheSim/bin/traceUtils/cli_parser.cpp index de885268..39233c45 100644 --- a/libCacheSim/bin/traceUtils/cli_parser.cpp +++ b/libCacheSim/bin/traceUtils/cli_parser.cpp @@ -30,6 +30,9 @@ enum argp_option_short { // trace print OPTION_NUM_REQ = 'n', + OPTION_FIELD_DELIMITER = 0x201, + OPTION_OBJ_ID_ONLY = 0x202, + OPTION_OBJ_ID_32bit = 0x204, // trace filter OPTION_FILTER_TYPE = 0x301, @@ -63,6 +66,11 @@ static struct argp_option options[] = { {0, 0, 0, 0, "tracePrint options:"}, {"num-req", OPTION_NUM_REQ, "-1", 0, "Number of requests to process, -1 means all requests in the trace", 6}, + {"field-delimiter", OPTION_FIELD_DELIMITER, ",", 0, + "The delimiter formatting the trace", 6}, + {"obj-id-only", OPTION_OBJ_ID_ONLY, "0", 0, "Only to print object id", 6}, + {"obj-id-32bit", OPTION_OBJ_ID_32bit, "0", 0, + "Print object id as 32-bit int", 6}, {0, 0, 0, 0, "traceFilter options:"}, {"filter-type", OPTION_FILTER_TYPE, "FIFO", 0, @@ -106,6 +114,15 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { case OPTION_NUM_REQ: arguments->n_req = atoll(arg); break; + case OPTION_FIELD_DELIMITER: + arguments->delimiter = arg[0]; + break; + case OPTION_OBJ_ID_ONLY: + arguments->print_obj_id_only = atol(arg); + break; + case OPTION_OBJ_ID_32bit: + arguments->print_obj_id_32bit = atol(arg); + break; case OPTION_FILTER_TYPE: arguments->cache_name = arg; break; @@ -146,6 +163,8 @@ static char doc[] = "tracePrint: utility to print binary trace in human-readable format\n" "traceConv: utility to convert a trace to oracleGeneral format\n\n" "traceFilter: utility to filter a trace\n\n" + "example usage: ./tracePrint /trace/path oracleGeneral -n 20 " + "--obj-id-only=1\n\n" "example usage: ./traceConv /trace/path csv -o " "/path/new_trace.oracleGeneral -t " "\"obj-id-col=5,time-col=2,obj-size-col=4\"\n\n" @@ -171,6 +190,9 @@ static void init_arg(struct arguments *args) { args->remove_size_change = false; args->cache_name = NULL; args->cache_size = 0; + args->delimiter = ','; + args->print_obj_id_only = false; + args->print_obj_id_32bit = false; } static void print_parsed_arg(struct arguments *args) { @@ -178,7 +200,7 @@ static void print_parsed_arg(struct arguments *args) { int n = 0; char output_str[OUTPUT_STR_LEN]; n = snprintf(output_str, OUTPUT_STR_LEN - 1, "trace path: %s, trace_type %s", - args->trace_path, trace_type_str[args->trace_type]); + args->trace_path, g_trace_type_name[args->trace_type]); if (args->trace_type_params != NULL) n += snprintf(output_str + n, OUTPUT_STR_LEN - n - 1, @@ -229,7 +251,8 @@ void parse_cmd(int argc, char *argv[], struct arguments *args) { assert(N_ARGS == 2); args->reader = create_reader(args->trace_type_str, args->trace_path, - args->trace_type_params, args->n_req, args->ignore_obj_size, 0); + args->trace_type_params, args->n_req, + args->ignore_obj_size, 0); print_parsed_arg(args); } diff --git a/libCacheSim/bin/traceUtils/internal.hpp b/libCacheSim/bin/traceUtils/internal.hpp index f3715231..b2ef443d 100644 --- a/libCacheSim/bin/traceUtils/internal.hpp +++ b/libCacheSim/bin/traceUtils/internal.hpp @@ -31,6 +31,9 @@ struct arguments { /* trace print */ int64_t num_req; /* number of requests to print */ + char delimiter; + bool print_obj_id_only; + bool print_obj_id_32bit; /* trace filter */ char *cache_name; diff --git a/libCacheSim/bin/traceUtils/traceConv.cpp b/libCacheSim/bin/traceUtils/traceConv.cpp index 404324c1..26236177 100644 --- a/libCacheSim/bin/traceUtils/traceConv.cpp +++ b/libCacheSim/bin/traceUtils/traceConv.cpp @@ -96,7 +96,7 @@ void convert_to_oracleGeneral(reader_t *reader, std::string ofilepath, "%s: %ld M requests (%.2lf GB), trace time %ld, working set %lld " "object, %lld B (%.2lf GB)\n", reader->trace_path, (long)(n_req_curr / 1e6), - (double)total_bytes / GiB, start_ts - req->clock_time, + (double)total_bytes / GiB, (long)(start_ts - req->clock_time), (long long)n_obj, (long long)unique_bytes, (double)unique_bytes / GiB); } @@ -118,8 +118,8 @@ void convert_to_oracleGeneral(reader_t *reader, std::string ofilepath, "%s: %ld M requests (%.2lf GB), trace time %ld, working set %lld " "object, %lld B (%.2lf GB), reversing output...\n", reader->trace_path, (long)(n_req_curr / 1e6), (double)total_bytes / GiB, - start_ts - req->clock_time, (long long)n_obj, (long long)unique_bytes, - (double)unique_bytes / GiB); + (long)(start_ts - req->clock_time), (long long)n_obj, + (long long)unique_bytes, (double)unique_bytes / GiB); struct trace_stat stat; stat.n_req = n_req_curr; @@ -251,6 +251,6 @@ static void _reverse_file(std::string ofilepath, struct trace_stat stat, remove((ofilepath + ".reverse").c_str()); INFO("trace conversion finished, %ld requests %ld objects, output %s\n", - n_req, stat.n_obj, ofilepath.c_str()); + (long) n_req, (long) stat.n_obj, ofilepath.c_str()); } } // namespace traceConv \ No newline at end of file diff --git a/libCacheSim/bin/traceUtils/traceConvMain.cpp b/libCacheSim/bin/traceUtils/traceConvMain.cpp index 23198212..a2883c11 100644 --- a/libCacheSim/bin/traceUtils/traceConvMain.cpp +++ b/libCacheSim/bin/traceUtils/traceConvMain.cpp @@ -30,7 +30,7 @@ int main(int argc, char *argv[]) { struct arguments args; cli::parse_cmd(argc, argv, &args); - if (args.ofilepath == NULL) { + if (strlen(args.ofilepath) == 0) { snprintf(args.ofilepath, OFILEPATH_LEN, "%s.oracleGeneral", args.trace_path); } diff --git a/libCacheSim/bin/traceUtils/traceFilterMain.cpp b/libCacheSim/bin/traceUtils/traceFilterMain.cpp index a6eb46c0..e601d268 100644 --- a/libCacheSim/bin/traceUtils/traceFilterMain.cpp +++ b/libCacheSim/bin/traceUtils/traceFilterMain.cpp @@ -53,7 +53,7 @@ void filter(reader_t *reader, cache_t *cache, std::string ofilepath) { read_one_req(reader, req); } - INFO("write %ld/%ld %.4lf requests to file %s\n", n_written_req, n_req, + INFO("write %ld/%ld %.4lf requests to file %s\n", (long) n_written_req, (long) n_req, (double)n_written_req / n_req, ofilepath.c_str()); free_request(req); output_file.close(); diff --git a/libCacheSim/bin/traceUtils/tracePrintMain.cpp b/libCacheSim/bin/traceUtils/tracePrintMain.cpp index 1c41c009..733fb867 100644 --- a/libCacheSim/bin/traceUtils/tracePrintMain.cpp +++ b/libCacheSim/bin/traceUtils/tracePrintMain.cpp @@ -8,7 +8,6 @@ #include "../../include/libCacheSim/reader.h" #include "internal.hpp" - int main(int argc, char *argv[]) { struct arguments args; @@ -16,22 +15,32 @@ int main(int argc, char *argv[]) { request_t *req = new_request(); read_one_req(args.reader, req); - + bool trace_has_next_access_vtime = req->next_access_vtime != -2; - if (trace_has_next_access_vtime) { - printf("# time, object, size, next_access_vtime\n"); - } else { - printf("# time, object, size\n"); + if (!args.print_obj_id_only) { + if (trace_has_next_access_vtime) { + printf("# time,object,size,next_access_vtime\n"); + } else { + printf("# time,object,size\n"); + } } while (req->valid) { - printf("%ld, %lu, %d", (long)req->clock_time, (unsigned long)req->obj_id, - (int)req->obj_size); - if (trace_has_next_access_vtime) { - printf(", %ld\n", (long)req->next_access_vtime); + if (args.print_obj_id_32bit) { + req->obj_id = (uint32_t)req->obj_id; + } + + if (args.print_obj_id_only) { + printf("%lu\n", (unsigned long)req->obj_id); } else { - printf("\n"); + printf("%ld%c%lu%c%d", (long)req->clock_time, args.delimiter, + (unsigned long)req->obj_id, args.delimiter, (int)req->obj_size); + if (trace_has_next_access_vtime) { + printf("%c%ld\n", args.delimiter, (long)req->next_access_vtime); + } else { + printf("\n"); + } } read_one_req(args.reader, req); } @@ -40,4 +49,3 @@ int main(int argc, char *argv[]) { return 0; } - diff --git a/libCacheSim/cache/admission/adaptsize.cpp b/libCacheSim/cache/admission/adaptsize.cpp index ad5de582..6fe3fadd 100644 --- a/libCacheSim/cache/admission/adaptsize.cpp +++ b/libCacheSim/cache/admission/adaptsize.cpp @@ -23,7 +23,7 @@ static void adaptsize_admissioner_parse_params( const char *init_params, adaptsize_admission_params_t *pa) { if (init_params == NULL) { pa->adaptsize_threshold = INT64_MAX; - INFO("use default adaptsize admission: %ld", pa->adaptsize_threshold); + INFO("use default adaptsize admission: %ld", (long) pa->adaptsize_threshold); } else { char *params_str = strdup(init_params); char *old_params_str = params_str; diff --git a/libCacheSim/cache/admission/size.c b/libCacheSim/cache/admission/size.c index a868e77b..e3f80022 100644 --- a/libCacheSim/cache/admission/size.c +++ b/libCacheSim/cache/admission/size.c @@ -26,7 +26,7 @@ static void size_admissioner_parse_params(const char *init_params, size_admission_params_t *pa) { if (init_params == NULL) { pa->size_threshold = INT64_MAX; - INFO("use default size admission: %ld", pa->size_threshold); + INFO("use default size admission: %ld", (long)pa->size_threshold); } else { char *params_str = strdup(init_params); char *old_params_str = params_str; diff --git a/libCacheSim/cache/cache.c b/libCacheSim/cache/cache.c index 3d527592..bbbc064e 100644 --- a/libCacheSim/cache/cache.c +++ b/libCacheSim/cache/cache.c @@ -2,11 +2,10 @@ // Created by Juncheng Yang on 6/20/20. // -#include "../include/libCacheSim/cache.h" - #include "../dataStructure/hashtable/hashtable.h" +#include "../include/libCacheSim/cache.h" #include "../include/libCacheSim/prefetchAlgo.h" - +#include "../utils/include/mymutex.h" /** this file contains both base function, which should be called by all *eviction algorithms, and the queue related functions, which should be called *by algorithm that uses only one queue and needs to update the queue such as @@ -78,7 +77,7 @@ void cache_struct_free(cache_t *cache) { } /** - * @brief create a new cache with the same size as the old cache + * @brief create a new cache with the same size as the old cache. Not thread-safe. * * @param old_cache * @param new_size @@ -102,7 +101,7 @@ cache_t *clone_cache(const cache_t *old_cache) { return cache; } /** - * @brief this function is called by all eviction algorithms to clone old cache + * @brief this function is called by all eviction algorithms to clone old cache. Not thread-safe. * with new size * * @param old_cache @@ -145,7 +144,7 @@ bool cache_can_insert_default(cache_t *cache, const request_t *req) { if (admissioner->admit(admissioner, req) == false) { DEBUG_ONCE( "admission algorithm does not admit: req %ld, obj %lu, size %lu\n", - cache->n_req, (unsigned long)req->obj_id, + (long)cache->n_req, (unsigned long)req->obj_id, (unsigned long)req->obj_size); return false; } @@ -153,7 +152,7 @@ bool cache_can_insert_default(cache_t *cache, const request_t *req) { if (req->obj_size + cache->obj_md_size > cache->cache_size) { WARN_ONCE("%ld req, obj %lu, size %lu larger than cache size %lu\n", - cache->n_req, (unsigned long)req->obj_id, + (long)cache->n_req, (unsigned long)req->obj_id, (unsigned long)req->obj_size, (unsigned long)cache->cache_size); return false; } @@ -176,7 +175,6 @@ bool cache_can_insert_default(cache_t *cache, const request_t *req) { cache_obj_t *cache_find_base(cache_t *cache, const request_t *req, const bool update_cache) { cache_obj_t *cache_obj = hashtable_find(cache->hashtable, req); - // "update_cache = true" means that it is a real user request, use handle_find // to update prefetcher's state if (cache->prefetcher && cache->prefetcher->handle_find && update_cache) { @@ -197,10 +195,10 @@ cache_obj_t *cache_find_base(cache_t *cache, const request_t *req, if (update_cache) { cache_obj->misc.next_access_vtime = req->next_access_vtime; - cache_obj->misc.freq += 1; + // cache_obj->misc.freq += 1; + fetch_add(&cache_obj->misc.freq, 1); } } - return cache_obj; } @@ -213,8 +211,6 @@ cache_obj_t *cache_find_base(cache_t *cache, const request_t *req, * update_metadata * return true * else: - * if cache does not have enough space: - * evict until it has space to insert * insert the object * return false * ``` @@ -225,32 +221,26 @@ cache_obj_t *cache_find_base(cache_t *cache, const request_t *req, */ bool cache_get_base(cache_t *cache, const request_t *req) { cache->n_req += 1; - VERBOSE("******* %s req %ld, obj %ld, obj_size %ld, cache size %ld/%ld\n", cache->cache_name, cache->n_req, req->obj_id, req->obj_size, cache->get_occupied_byte(cache), cache->cache_size); cache_obj_t *obj = cache->find(cache, req, true); bool hit = (obj != NULL); - if (hit) { VVERBOSE("req %ld, obj %ld --- cache hit\n", cache->n_req, req->obj_id); - } else if (!cache->can_insert(cache, req)) { - VVERBOSE("req %ld, obj %ld --- cache miss cannot insert\n", cache->n_req, - req->obj_id); } else { - while (cache->get_occupied_byte(cache) + req->obj_size + - cache->obj_md_size > - cache->cache_size) { - cache->evict(cache, req); - } + // while (cache->get_occupied_byte(cache) + req->obj_size + + // cache->obj_md_size > + // cache->cache_size) { + // cache->evict(cache, req); + // } cache->insert(cache, req); } - if (cache->prefetcher && cache->prefetcher->prefetch) { cache->prefetcher->prefetch(cache, req); } - + // INFO("req %ld, obj %ld --- cache hit %d\n", cache->n_req, req->obj_id, hit); return hit; } @@ -260,37 +250,65 @@ bool cache_get_base(cache_t *cache, const request_t *req) { * this function assumes the cache has enough space * and eviction is not part of this function * + * ``` + * if cache not full: + * create an object; + * else: + * evict an object; + * reassign the object; + * insert the object; + * return false + * ``` + * * @param cache * @param req * @return the inserted object */ cache_obj_t *cache_insert_base(cache_t *cache, const request_t *req) { - cache_obj_t *cache_obj = hashtable_insert(cache->hashtable, req); - cache->occupied_byte += - (int64_t)cache_obj->obj_size + (int64_t)cache->obj_md_size; - cache->n_obj += 1; + cache_obj_t *new_cache_obj; + if(cache->get_occupied_byte(cache) + req->obj_size + + cache->obj_md_size < cache->cache_size){ + new_cache_obj = create_cache_obj_from_request(req); + } + else{ + new_cache_obj = cache->to_evict(cache, req); + copy_request_to_cache_obj(new_cache_obj, req); + } + + + // cache_obj_t *cache_obj = hashtable_insert(cache->hashtable, req); + hashtable_insert_obj(cache->hashtable, new_cache_obj); + // cache->occupied_byte += + // (int64_t)cache_obj->obj_size + (int64_t)cache->obj_md_size; + fetch_add(&cache->occupied_byte, + (int64_t)new_cache_obj->obj_size + (int64_t)cache->obj_md_size); + + // cache->n_obj += 1; + fetch_add(&cache->n_obj, 1); + #ifdef SUPPORT_TTL if (cache->default_ttl != 0 && req->ttl == 0) { - cache_obj->exp_time = (int32_t)cache->default_ttl + req->clock_time; + uint32_t exp_time = (int32_t)cache->default_ttl + req->clock_time; + new_cache_obj->exp_time = exp_time; } #endif #if defined(TRACK_EVICTION_V_AGE) || defined(TRACK_DEMOTION) || \ defined(TRACK_CREATE_TIME) - cache_obj->create_time = CURR_TIME(cache, req); + new_cache_obj->create_time = CURR_TIME(cache, req); #endif - cache_obj->misc.next_access_vtime = req->next_access_vtime; - cache_obj->misc.freq = 0; + new_cache_obj->misc.next_access_vtime = req->next_access_vtime; + new_cache_obj->misc.freq = 0; - return cache_obj; + return new_cache_obj; } /** * @brief this function is called by all eviction algorithms in the eviction - * function, it updates the cache metadata. Because it frees the object struct, - * it needs to be called at the end of the eviction function. + * function, it updates the cache metadata. It needs to be called at the end + * of the eviction function. * * @param cache the cache * @param obj the object to be removed @@ -314,24 +332,24 @@ void cache_evict_base(cache_t *cache, cache_obj_t *obj, } cache_remove_obj_base(cache, obj, remove_from_hashtable); + // cache->occupied_byte -= (obj->obj_size + cache->obj_md_size); + fetch_sub(&cache->occupied_byte, + obj->obj_size + cache->obj_md_size); } /** * @brief this function is called by all eviction algorithms that * need to remove an object from the cache, it updates the cache metadata, - * because it frees the object struct, it needs to be called at the end of - * the eviction function. + * because it not frees the object struct, while it does not update the + * eviction data structure. This stale item will be removed in the next eviction. * * @param cache the cache * @param obj the object to be removed */ void cache_remove_obj_base(cache_t *cache, cache_obj_t *obj, bool remove_from_hashtable) { - DEBUG_ASSERT(cache->occupied_byte >= obj->obj_size + cache->obj_md_size); - cache->occupied_byte -= (obj->obj_size + cache->obj_md_size); - cache->n_obj -= 1; - if (remove_from_hashtable) { - hashtable_delete(cache->hashtable, obj); + if(hashtable_try_delete(cache->hashtable, obj)){ + fetch_sub(&cache->n_obj, 1); } } diff --git a/libCacheSim/cache/cacheObj.c b/libCacheSim/cache/cacheObj.c index 07380b02..651628ae 100644 --- a/libCacheSim/cache/cacheObj.c +++ b/libCacheSim/cache/cacheObj.c @@ -17,6 +17,7 @@ void copy_cache_obj_to_request(request_t *req_dest, req_dest->obj_id = cache_obj->obj_id; req_dest->obj_size = cache_obj->obj_size; req_dest->next_access_vtime = cache_obj->misc.next_access_vtime; + // req_dest->valid = verify_cache_obj_fingerprint(cache_obj); req_dest->valid = true; } @@ -34,6 +35,7 @@ void copy_request_to_cache_obj(cache_obj_t *cache_obj, const request_t *req) { cache_obj->exp_time = 0; #endif cache_obj->obj_id = req->obj_id; + // set_cache_obj_fingerprint(cache_obj); } /** @@ -48,6 +50,26 @@ cache_obj_t *create_cache_obj_from_request(const request_t *req) { return cache_obj; } +/** + * [create a cache_obj from obj_id] + * @method create_cache_obj_from_obj_id + * @author Chaos + * @date 2023-11-22 + * @param obj_id [Given obj_id] + * @return [cache_obj] + */ +cache_obj_t *create_cache_obj_from_obj_id(const obj_id_t obj_id) { + cache_obj_t *cache_obj = my_malloc(cache_obj_t); + memset(cache_obj, 0, sizeof(cache_obj_t)); + cache_obj->obj_id = obj_id; + cache_obj->obj_size = 0; +#ifdef SUPPORT_TTL + cache_obj->exp_time = 0; +#endif + // set_cache_obj_fingerprint(cache_obj); + return cache_obj; +} + /** remove the object from the built-in doubly linked list * * @param head @@ -226,4 +248,16 @@ void append_obj_to_tail(cache_obj_t **head, cache_obj_t **tail, *tail = cache_obj; +} + + +/** + * free the the doubly linked list + * @param head + * @param tail + */ +void free_list(cache_obj_t **head, cache_obj_t **tail) { + cache_obj_t *oldHead = *head; + remove_obj_from_list(head, tail, oldHead); + free_cache_obj(oldHead); } \ No newline at end of file diff --git a/libCacheSim/cache/eviction/ARC.c b/libCacheSim/cache/eviction/ARC.c index ec7b6a1e..a719f0fb 100644 --- a/libCacheSim/cache/eviction/ARC.c +++ b/libCacheSim/cache/eviction/ARC.c @@ -180,7 +180,6 @@ static void ARC_free(cache_t *cache) { * @return true if cache hit, false if cache miss */ static bool ARC_get(cache_t *cache, const request_t *req) { - ARC_params_t *params = (ARC_params_t *)(cache->eviction_params); #ifdef DEBUG_MODE return ARC_get_debug(cache, req); #else @@ -520,7 +519,6 @@ static void _ARC_evict_L2_ghost(cache_t *cache, const request_t *req) { static void _ARC_replace(cache_t *cache, const request_t *req) { ARC_params_t *params = (ARC_params_t *)(cache->eviction_params); - cache_obj_t *obj = NULL; bool cond1 = params->L1_data_size > 0; bool cond2 = params->L1_data_size > params->p; @@ -613,13 +611,12 @@ static void ARC_parse_params(cache_t *cache, char *params_str = strdup(cache_specific_params); char *old_params_str = params_str; - char *end; while (params_str != NULL && params_str[0] != '\0') { /* different parameters are separated by comma, * key and value are separated by = */ char *key = strsep((char **)¶ms_str, "="); - char *value = strsep((char **)¶ms_str, ","); + // char *value = strsep((char **)¶ms_str, ","); // skip the white space while (params_str != NULL && *params_str == ' ') { @@ -649,7 +646,7 @@ static void print_cache(cache_t *cache) { cache_obj_t *obj = params->L1_data_head; printf("T1: "); while (obj != NULL) { - printf("%ld ", obj->obj_id); + printf("%ld ", (long)obj->obj_id); obj = obj->queue.next; } printf("\n"); @@ -657,7 +654,7 @@ static void print_cache(cache_t *cache) { obj = params->L1_ghost_head; printf("B1: "); while (obj != NULL) { - printf("%ld ", obj->obj_id); + printf("%ld ", (long)obj->obj_id); obj = obj->queue.next; } printf("\n"); @@ -665,7 +662,7 @@ static void print_cache(cache_t *cache) { obj = params->L2_data_head; printf("T2: "); while (obj != NULL) { - printf("%ld ", obj->obj_id); + printf("%ld ", (long)obj->obj_id); obj = obj->queue.next; } printf("\n"); @@ -673,7 +670,7 @@ static void print_cache(cache_t *cache) { obj = params->L2_ghost_head; printf("B2: "); while (obj != NULL) { - printf("%ld ", obj->obj_id); + printf("%ld ", (long)obj->obj_id); obj = obj->queue.next; } printf("\n"); @@ -773,7 +770,6 @@ static inline void _ARC_sanity_check_full(cache_t *cache, } static bool ARC_get_debug(cache_t *cache, const request_t *req) { - ARC_params_t *params = (ARC_params_t *)(cache->eviction_params); cache->n_req += 1; diff --git a/libCacheSim/cache/eviction/ARCv0.c b/libCacheSim/cache/eviction/ARCv0.c index 19fb5e0f..1c3c6488 100644 --- a/libCacheSim/cache/eviction/ARCv0.c +++ b/libCacheSim/cache/eviction/ARCv0.c @@ -30,8 +30,6 @@ extern "C" { // #define LAZY_PROMOTION // #define QUICK_DEMOTION -// #define USE_MYCLOCK - typedef struct ARCv0_params { // L1_data is T1 in the paper, L1_ghost is B1 in the paper cache_t *T1; @@ -93,7 +91,8 @@ static bool ARCv0_get_debug(cache_t *cache, const request_t *req); */ cache_t *ARCv0_init(const common_cache_params_t ccache_params, const char *cache_specific_params) { - cache_t *cache = cache_struct_init("ARCv0", ccache_params, cache_specific_params); + cache_t *cache = + cache_struct_init("ARCv0", ccache_params, cache_specific_params); cache->cache_init = ARCv0_init; cache->cache_free = ARCv0_free; cache->get = ARCv0_get; @@ -140,12 +139,6 @@ cache_t *ARCv0_init(const common_cache_params_t ccache_params, snprintf(cache->cache_name, CACHE_NAME_ARRAY_LEN, "ARCv0-QD"); #endif -#ifdef USE_MYCLOCK - params->T2->cache_free(params->T2); - params->T2 = MyClock_init(ccache_params_local, NULL); - snprintf(cache->cache_name, CACHE_NAME_ARRAY_LEN, "ARCv0-myclock"); -#endif - return cache; } @@ -186,7 +179,7 @@ static void ARCv0_free(cache_t *cache) { * @return true if cache hit, false if cache miss */ static bool ARCv0_get(cache_t *cache, const request_t *req) { - ARCv0_params_t *params = (ARCv0_params_t *)(cache->eviction_params); + // ARCv0_params_t *params = (ARCv0_params_t *)(cache->eviction_params); #ifdef DEBUG_MODE return ARCv0_get_debug(cache, req); #else @@ -257,7 +250,7 @@ static cache_obj_t *ARCv0_find(cache_t *cache, const request_t *req, } #ifdef QUICK_DEMOTION // params->p = MIN(params->p, cache->cache_size/10); - params->p = cache->cache_size/10; + params->p = cache->cache_size / 10; #endif } else { // cache hit, case I: x in L1_data or L2_data @@ -534,13 +527,12 @@ static void ARCv0_parse_params(cache_t *cache, char *params_str = strdup(cache_specific_params); char *old_params_str = params_str; - char *end; while (params_str != NULL && params_str[0] != '\0') { /* different parameters are separated by comma, * key and value are separated by = */ char *key = strsep((char **)¶ms_str, "="); - char *value = strsep((char **)¶ms_str, ","); + // char *value = strsep((char **)¶ms_str, ","); // skip the white space while (params_str != NULL && *params_str == ' ') { @@ -581,7 +573,8 @@ static bool ARCv0_get_debug(cache_t *cache, const request_t *req) { cache->n_req += 1; - printf("%ld obj_id %ld: p %.2lf\n", cache->n_req, req->obj_id, params->p); + printf("%ld obj_id %ld: p %.2lf\n", (long)cache->n_req, (long)req->obj_id, + params->p); print_cache(cache); printf("==================================\n"); diff --git a/libCacheSim/cache/eviction/CMakeLists.txt b/libCacheSim/cache/eviction/CMakeLists.txt index 45dd821f..0a9a695a 100644 --- a/libCacheSim/cache/eviction/CMakeLists.txt +++ b/libCacheSim/cache/eviction/CMakeLists.txt @@ -1,6 +1,7 @@ set(sourceC MRU.c Random.c + RandomTwo.c LFU.c LFUDA.c ARC.c @@ -43,7 +44,6 @@ set(sourceC if (EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/priv") set(sourceC ${sourceC} - priv/MyClock.c priv/QDLPv0.c diff --git a/libCacheSim/cache/eviction/CR_LFU.c b/libCacheSim/cache/eviction/CR_LFU.c index 33ae33df..d3759a1e 100644 --- a/libCacheSim/cache/eviction/CR_LFU.c +++ b/libCacheSim/cache/eviction/CR_LFU.c @@ -46,7 +46,8 @@ static void free_list_node(void *list_node) { */ cache_t *CR_LFU_init(const common_cache_params_t ccache_params, const char *cache_specific_params) { - cache_t *cache = cache_struct_init("CR_LFU", ccache_params, cache_specific_params); + cache_t *cache = + cache_struct_init("CR_LFU", ccache_params, cache_specific_params); cache->cache_init = CR_LFU_init; cache->cache_free = CR_LFU_free; cache->get = CR_LFU_get; diff --git a/libCacheSim/cache/eviction/Cacheus.c b/libCacheSim/cache/eviction/Cacheus.c index a590393a..c8bbf8a7 100644 --- a/libCacheSim/cache/eviction/Cacheus.c +++ b/libCacheSim/cache/eviction/Cacheus.c @@ -273,7 +273,6 @@ static void Cacheus_evict(cache_t *cache, const request_t *req) { cache_t *lfu = params->LFU; cache_t *lru_g = params->LRU_g; cache_t *lfu_g = params->LFU_g; - SR_LRU_params_t *params_LRU = (SR_LRU_params_t *)(lru->eviction_params); // If two voters decide the same: cache_obj_t *lru_to_evict = lru->to_evict(lru, req); @@ -390,8 +389,8 @@ static void update_lr(cache_t *cache, const request_t *req) { // self.learning_rate = 0.9 // elif self.learning_rate <= 0.001: // self.learning_rate = 0.005 - if (params->lr + sign * abs(params->lr * delta_lr) > 0.001) - params->lr = params->lr + sign * abs(params->lr * delta_lr); + if (params->lr + sign * fabs(params->lr * delta_lr) > 0.001) + params->lr = params->lr + sign * fabs(params->lr * delta_lr); else params->lr = 0.0001; params->unlearn_count = 0; diff --git a/libCacheSim/cache/eviction/Clock.c b/libCacheSim/cache/eviction/Clock.c index 5da554b1..e60f0d31 100644 --- a/libCacheSim/cache/eviction/Clock.c +++ b/libCacheSim/cache/eviction/Clock.c @@ -53,7 +53,8 @@ static bool Clock_remove(cache_t *cache, const obj_id_t obj_id); */ cache_t *Clock_init(const common_cache_params_t ccache_params, const char *cache_specific_params) { - cache_t *cache = cache_struct_init("Clock", ccache_params, cache_specific_params); + cache_t *cache = + cache_struct_init("Clock", ccache_params, cache_specific_params); cache->cache_init = Clock_init; cache->cache_free = Clock_free; cache->get = Clock_get; @@ -98,7 +99,9 @@ cache_t *Clock_init(const common_cache_params_t ccache_params, * @param cache */ static void Clock_free(cache_t *cache) { - free(cache->eviction_params); + Clock_params_t *params = (Clock_params_t *)cache->eviction_params; + free_list(¶ms->q_head, ¶ms->q_tail); + free(params); cache_struct_free(cache); } @@ -157,6 +160,24 @@ static cache_obj_t *Clock_find(cache_t *cache, const request_t *req, return obj; } +void link_at_tail(Clock_params_t *params, cache_obj_t *obj) { + obj->queue.next = NULL; + cache_obj_t *tail = params->q_tail; + + // this is the thread that first makes Tail points to the obj + // other threads must follow this, o.w. oldHead will be nullptr + while (!CAS(¶ms->q_tail, &tail, obj)) {} + + if (tail != NULL) { + tail->queue.next = obj; + } else { + params->q_head = obj; + } +#ifdef USE_BELADY + obj->next_access_vtime = req->next_access_vtime; +#endif +} + /** * @brief insert an object into the cache, * update the hash table and cache metadata @@ -169,15 +190,10 @@ static cache_obj_t *Clock_find(cache_t *cache, const request_t *req, */ static cache_obj_t *Clock_insert(cache_t *cache, const request_t *req) { Clock_params_t *params = (Clock_params_t *)cache->eviction_params; - cache_obj_t *obj = cache_insert_base(cache, req); - prepend_obj_to_head(¶ms->q_head, ¶ms->q_tail, obj); - obj->clock.freq = 0; -#ifdef USE_BELADY - obj->next_access_vtime = req->next_access_vtime; -#endif - + link_at_tail(params, obj); + cache_obj_set_in_cache(obj, true); return obj; } @@ -193,21 +209,24 @@ static cache_obj_t *Clock_insert(cache_t *cache, const request_t *req) { */ static cache_obj_t *Clock_to_evict(cache_t *cache, const request_t *req) { Clock_params_t *params = (Clock_params_t *)cache->eviction_params; - - int n_round = 0; - cache_obj_t *obj_to_evict = params->q_tail; -#ifdef USE_BELADY - while (obj_to_evict->next_access_vtime != INT64_MAX) { -#else - while (obj_to_evict->clock.freq - n_round >= 1) { -#endif - obj_to_evict = obj_to_evict->queue.prev; - if (obj_to_evict == NULL) { - obj_to_evict = params->q_tail; - n_round += 1; + cache_obj_t *obj_to_evict; + while (true){ + obj_to_evict = params->q_head; + while(!CAS(¶ms->q_head, &obj_to_evict, obj_to_evict->queue.next)) {} + if(obj_to_evict->queue.next != NULL) { + obj_to_evict->queue.next->queue.prev = NULL; + } else { + params->q_tail = NULL; + } + if (obj_to_evict->clock.freq >= 1) { + fetch_sub(&obj_to_evict->clock.freq, 1); + link_at_tail(params, obj_to_evict); + } + else{ + break; } } - + cache_evict_base(cache, obj_to_evict, true); return obj_to_evict; } @@ -221,19 +240,7 @@ static cache_obj_t *Clock_to_evict(cache_t *cache, const request_t *req) { * @param evicted_obj if not NULL, return the evicted object to caller */ static void Clock_evict(cache_t *cache, const request_t *req) { - Clock_params_t *params = (Clock_params_t *)cache->eviction_params; - - cache_obj_t *obj_to_evict = params->q_tail; - while (obj_to_evict->clock.freq >= 1) { - obj_to_evict->clock.freq -= 1; - params->n_obj_rewritten += 1; - params->n_byte_rewritten += obj_to_evict->obj_size; - move_obj_to_head(¶ms->q_head, ¶ms->q_tail, obj_to_evict); - obj_to_evict = params->q_tail; - } - - remove_obj_from_list(¶ms->q_head, ¶ms->q_tail, obj_to_evict); - cache_evict_base(cache, obj_to_evict, true); + Clock_to_evict(cache, req); } /** @@ -252,10 +259,10 @@ static void Clock_evict(cache_t *cache, const request_t *req) { * @param obj */ static void Clock_remove_obj(cache_t *cache, cache_obj_t *obj) { - Clock_params_t *params = (Clock_params_t *)cache->eviction_params; + // Clock_params_t *params = (Clock_params_t *)cache->eviction_params; - DEBUG_ASSERT(obj != NULL); - remove_obj_from_list(¶ms->q_head, ¶ms->q_tail, obj); + // DEBUG_ASSERT(obj != NULL); + // remove_obj_from_list(¶ms->q_head, ¶ms->q_tail, obj); cache_remove_obj_base(cache, obj, true); } @@ -273,14 +280,13 @@ static void Clock_remove_obj(cache_t *cache, cache_obj_t *obj) { * cache */ static bool Clock_remove(cache_t *cache, const obj_id_t obj_id) { - cache_obj_t *obj = hashtable_find_obj_id(cache->hashtable, obj_id); - if (obj == NULL) { - return false; + cache_obj_t *obj = hashtable_delete_obj_id(cache->hashtable, obj_id); + if(obj != NULL){ + fetch_sub(&cache->n_obj, 1); + return true; } - - Clock_remove_obj(cache, obj); - - return true; + else + return false; } // *********************************************************************** @@ -291,8 +297,7 @@ static bool Clock_remove(cache_t *cache, const obj_id_t obj_id) { static const char *Clock_current_params(cache_t *cache, Clock_params_t *params) { static __thread char params_str[128]; - int n = - snprintf(params_str, 128, "n-bit-counter=%d\n", params->n_bit_counter); + snprintf(params_str, 128, "n-bit-counter=%d\n", params->n_bit_counter); return params_str; } diff --git a/libCacheSim/cache/eviction/FIFO.c b/libCacheSim/cache/eviction/FIFO.c index 2c5d1f5c..6e0465c7 100644 --- a/libCacheSim/cache/eviction/FIFO.c +++ b/libCacheSim/cache/eviction/FIFO.c @@ -76,7 +76,9 @@ cache_t *FIFO_init(const common_cache_params_t ccache_params, * @param cache */ static void FIFO_free(cache_t *cache) { - free(cache->eviction_params); + FIFO_params_t *params = (FIFO_params_t *)cache->eviction_params; + free_list(¶ms->q_head, ¶ms->q_tail); + free(params); cache_struct_free(cache); } @@ -122,7 +124,7 @@ static bool FIFO_get(cache_t *cache, const request_t *req) { static cache_obj_t *FIFO_find(cache_t *cache, const request_t *req, const bool update_cache) { return cache_find_base(cache, req, update_cache); -} +} /** * @brief insert an object into the cache, @@ -137,8 +139,17 @@ static cache_obj_t *FIFO_find(cache_t *cache, const request_t *req, static cache_obj_t *FIFO_insert(cache_t *cache, const request_t *req) { FIFO_params_t *params = (FIFO_params_t *)cache->eviction_params; cache_obj_t *obj = cache_insert_base(cache, req); - prepend_obj_to_head(¶ms->q_head, ¶ms->q_tail, obj); - + // prepend_obj_to_head(¶ms->q_head, ¶ms->q_tail, obj); + // Use CAS to update the q_tail + obj->queue.next = NULL; + cache_obj_t *tail = params->q_tail; + while (!CAS(¶ms->q_tail, &tail, obj)) {} + if (tail != NULL) { + tail->queue.next = obj; + } else { + params->q_head = obj; + } + cache_obj_set_in_cache(obj, true); return obj; } @@ -154,7 +165,21 @@ static cache_obj_t *FIFO_insert(cache_t *cache, const request_t *req) { */ static cache_obj_t *FIFO_to_evict(cache_t *cache, const request_t *req) { FIFO_params_t *params = (FIFO_params_t *)cache->eviction_params; - return params->q_tail; + cache_obj_t *obj_to_evict = params->q_head; + DEBUG_ASSERT(params->q_head != NULL); + + // we can simply call remove_obj_from_list here, but for the best performance, + // we chose to do it manually + // remove_obj_from_list(¶ms->q_head, ¶ms->q_tail, obj); + + while(!CAS(¶ms->q_head, &obj_to_evict, obj_to_evict->queue.next)) {} + if (obj_to_evict->queue.next != NULL) { + obj_to_evict->queue.next->queue.prev = NULL; + } else { + params->q_tail = NULL; + } + cache_evict_base(cache, obj_to_evict, true); + return obj_to_evict; } /** @@ -167,24 +192,7 @@ static cache_obj_t *FIFO_to_evict(cache_t *cache, const request_t *req) { * @param evicted_obj if not NULL, return the evicted object to caller */ static void FIFO_evict(cache_t *cache, const request_t *req) { - FIFO_params_t *params = (FIFO_params_t *)cache->eviction_params; - cache_obj_t *obj_to_evict = params->q_tail; - DEBUG_ASSERT(params->q_tail != NULL); - - // we can simply call remove_obj_from_list here, but for the best performance, - // we chose to do it manually - // remove_obj_from_list(¶ms->q_head, ¶ms->q_tail, obj); - - params->q_tail = params->q_tail->queue.prev; - if (likely(params->q_tail != NULL)) { - params->q_tail->queue.next = NULL; - } else { - /* cache->n_obj has not been updated */ - DEBUG_ASSERT(cache->n_obj == 1); - params->q_head = NULL; - } - - cache_evict_base(cache, obj_to_evict, true); + FIFO_to_evict(cache, req); } /** @@ -201,17 +209,13 @@ static void FIFO_evict(cache_t *cache, const request_t *req) { * cache */ static bool FIFO_remove(cache_t *cache, const obj_id_t obj_id) { - cache_obj_t *obj = hashtable_find_obj_id(cache->hashtable, obj_id); - if (obj == NULL) { - return false; + cache_obj_t *obj = hashtable_delete_obj_id(cache->hashtable, obj_id); + if(obj != NULL){ + fetch_sub(&cache->n_obj, 1); + return true; } - - FIFO_params_t *params = (FIFO_params_t *)cache->eviction_params; - - remove_obj_from_list(¶ms->q_head, ¶ms->q_tail, obj); - cache_remove_obj_base(cache, obj, true); - - return true; + else + return false; } #ifdef __cplusplus diff --git a/libCacheSim/cache/eviction/FIFO_Merge.c b/libCacheSim/cache/eviction/FIFO_Merge.c index 338f998e..83aba917 100644 --- a/libCacheSim/cache/eviction/FIFO_Merge.c +++ b/libCacheSim/cache/eviction/FIFO_Merge.c @@ -261,7 +261,7 @@ static void FIFO_Merge_evict(cache_t *cache, const request_t *req) { if (cache->n_obj <= params->n_exam_obj) { // just evict one object - this is fifo - cache_obj_t *cache_obj = params->q_tail; + cache_obj = params->q_tail; params->next_to_exam = NULL; remove_obj_from_list(¶ms->q_head, ¶ms->q_tail, cache_obj); cache_evict_base(cache, cache_obj, true); @@ -451,6 +451,8 @@ static double retain_metric(cache_t *cache, cache_obj_t *cache_obj) { default: break; } + abort(); + return -1; } #ifdef __cplusplus diff --git a/libCacheSim/cache/eviction/FIFO_Reinsertion.c b/libCacheSim/cache/eviction/FIFO_Reinsertion.c index 566c3d1c..b2fdd0d6 100644 --- a/libCacheSim/cache/eviction/FIFO_Reinsertion.c +++ b/libCacheSim/cache/eviction/FIFO_Reinsertion.c @@ -270,7 +270,7 @@ static void FIFO_Reinsertion_evict(cache_t *cache, const request_t *req) { if (cache->n_obj <= params->n_exam_obj) { // just evict one object - cache_obj_t *cache_obj = params->next_to_merge->queue.prev; + cache_obj = params->next_to_merge->queue.prev; FIFO_Reinsertion_remove_obj(cache, params->next_to_merge); params->next_to_merge = cache_obj; @@ -473,6 +473,7 @@ static double retain_metric(cache_t *cache, cache_obj_t *cache_obj) { default: break; } + abort(); } #ifdef __cplusplus diff --git a/libCacheSim/cache/eviction/LFU.c b/libCacheSim/cache/eviction/LFU.c index e93e81b8..feee54b8 100644 --- a/libCacheSim/cache/eviction/LFU.c +++ b/libCacheSim/cache/eviction/LFU.c @@ -37,6 +37,7 @@ typedef struct LFU_params { GHashTable *freq_map; uint64_t min_freq; uint64_t max_freq; + pthread_mutex_t mutex_; } LFU_params_t; // *********************************************************************** @@ -108,7 +109,7 @@ cache_t *LFU_init(const common_cache_params_t ccache_params, params->min_freq = 1; params->max_freq = 1; params->freq_one_node = freq_node; - + pthread_mutex_init(¶ms->mutex_, NULL); return cache; } @@ -166,9 +167,11 @@ static bool LFU_get(cache_t *cache, const request_t *req) { static cache_obj_t *LFU_find(cache_t *cache, const request_t *req, const bool update_cache) { LFU_params_t *params = (LFU_params_t *)(cache->eviction_params); + cache_obj_t *cache_obj = cache_find_base(cache, req, update_cache); + pthread_mutex_lock(¶ms->mutex_); - if (cache_obj && likely(update_cache)) { + if (cache_obj && likely(update_cache) && cache_obj_in_cache(cache_obj)) { /* freq incr and move to next freq node */ cache_obj->lfu.freq += 1; if (params->max_freq < cache_obj->lfu.freq) { @@ -177,6 +180,7 @@ static cache_obj_t *LFU_find(cache_t *cache, const request_t *req, // find the freq_node this object belongs to and update its info gpointer old_key = GSIZE_TO_POINTER(cache_obj->lfu.freq - 1); + freq_node_t *old_node = g_hash_table_lookup(params->freq_map, old_key); DEBUG_ASSERT(old_node != NULL); DEBUG_ASSERT(old_node->freq == cache_obj->lfu.freq - 1); @@ -215,6 +219,7 @@ static cache_obj_t *LFU_find(cache_t *cache, const request_t *req, } } } + pthread_mutex_unlock(¶ms->mutex_); return cache_obj; } @@ -230,15 +235,20 @@ static cache_obj_t *LFU_find(cache_t *cache, const request_t *req, */ static cache_obj_t *LFU_insert(cache_t *cache, const request_t *req) { LFU_params_t *params = (LFU_params_t *)(cache->eviction_params); + + cache_obj_t *cache_obj = cache_insert_base(cache, req); + pthread_mutex_lock(¶ms->mutex_); + params->min_freq = 1; freq_node_t *freq_one_node = params->freq_one_node; - cache_obj_t *cache_obj = cache_insert_base(cache, req); cache_obj->lfu.freq = 1; freq_one_node->n_obj += 1; append_obj_to_tail(&freq_one_node->first_obj, &freq_one_node->last_obj, cache_obj); + cache_obj_set_in_cache(cache_obj, true); + pthread_mutex_unlock(¶ms->mutex_); return cache_obj; } @@ -255,25 +265,13 @@ static cache_obj_t *LFU_insert(cache_t *cache, const request_t *req) { */ static cache_obj_t *LFU_to_evict(cache_t *cache, const request_t *req) { LFU_params_t *params = (LFU_params_t *)(cache->eviction_params); - freq_node_t *min_freq_node = get_min_freq_node(params); - return min_freq_node->first_obj; -} - -/** - * @brief evict an object from the cache - * it needs to call cache_evict_base before returning - * which updates some metadata such as n_obj, occupied size, and hash table - * - * @param cache - * @param req not used - */ -static void LFU_evict(cache_t *cache, const request_t *req) { - LFU_params_t *params = (LFU_params_t *)(cache->eviction_params); + pthread_mutex_lock(¶ms->mutex_); freq_node_t *min_freq_node = get_min_freq_node(params); min_freq_node->n_obj--; cache_obj_t *obj_to_evict = min_freq_node->first_obj; + cache_obj_set_in_cache(obj_to_evict, false); remove_obj_from_list(&min_freq_node->first_obj, &min_freq_node->last_obj, obj_to_evict); @@ -287,28 +285,27 @@ static void LFU_evict(cache_t *cache, const request_t *req) { // /* update min freq */ // update_min_freq(params); } - + pthread_mutex_unlock(¶ms->mutex_); + cache_evict_base(cache, obj_to_evict, true); + return obj_to_evict; +} + +/** + * @brief evict an object from the cache + * it needs to call cache_evict_base before returning + * which updates some metadata such as n_obj, occupied size, and hash table + * + * @param cache + * @param req not used + */ +static void LFU_evict(cache_t *cache, const request_t *req) { + LFU_to_evict(cache, req); } void LFU_remove_obj(cache_t *cache, cache_obj_t *obj) { assert(obj != NULL); - LFU_params_t *params = (LFU_params_t *)(cache->eviction_params); - - gpointer key = GSIZE_TO_POINTER(obj->lfu.freq); - freq_node_t *freq_node = g_hash_table_lookup(params->freq_map, key); - DEBUG_ASSERT(freq_node->freq == obj->lfu.freq); - DEBUG_ASSERT(freq_node->n_obj > 0); - - freq_node->n_obj--; - remove_obj_from_list(&freq_node->first_obj, &freq_node->last_obj, obj); - cache_remove_obj_base(cache, obj, true); - - if (freq_node->freq == params->min_freq && freq_node->n_obj == 0) { - /* update min freq */ - update_min_freq(params); - } } /** @@ -325,15 +322,13 @@ void LFU_remove_obj(cache_t *cache, cache_obj_t *obj) { * cache */ bool LFU_remove(cache_t *cache, obj_id_t obj_id) { - LFU_params_t *params = (LFU_params_t *)(cache->eviction_params); - cache_obj_t *obj = hashtable_find_obj_id(cache->hashtable, obj_id); - if (obj == NULL) { - return false; + cache_obj_t *obj = hashtable_delete_obj_id(cache->hashtable, obj_id); + if(obj != NULL){ + fetch_sub(&cache->n_obj, 1); + return true; } - - LFU_remove_obj(cache, obj); - - return true; + else + return false; } // *********************************************************************** diff --git a/libCacheSim/cache/eviction/LFUDA.c b/libCacheSim/cache/eviction/LFUDA.c index 8e7ad8f4..747a026d 100644 --- a/libCacheSim/cache/eviction/LFUDA.c +++ b/libCacheSim/cache/eviction/LFUDA.c @@ -311,7 +311,6 @@ static void LFUDA_remove_obj(cache_t *cache, cache_obj_t *obj) { * cache */ static bool LFUDA_remove(cache_t *cache, obj_id_t obj_id) { - LFUDA_params_t *params = (LFUDA_params_t *)(cache->eviction_params); cache_obj_t *obj = hashtable_find_obj_id(cache->hashtable, obj_id); if (obj == NULL) { return false; diff --git a/libCacheSim/cache/eviction/LHD/CMakeLists.txt b/libCacheSim/cache/eviction/LHD/CMakeLists.txt index 19974a1d..b049c97b 100644 --- a/libCacheSim/cache/eviction/LHD/CMakeLists.txt +++ b/libCacheSim/cache/eviction/LHD/CMakeLists.txt @@ -1,11 +1,4 @@ file(GLOB src *.cpp) add_library (LHD ${src}) -set_target_properties(LHD - PROPERTIES - CXX_STANDARD 11 - CXX_STANDARD_REQUIRED YES - CXX_EXTENSIONS NO - ) - - +set_property(LHD CXX_STANDARD 17) diff --git a/libCacheSim/cache/eviction/LHD/candidate.hpp b/libCacheSim/cache/eviction/LHD/candidate.hpp index d4b120df..17a062fa 100644 --- a/libCacheSim/cache/eviction/LHD/candidate.hpp +++ b/libCacheSim/cache/eviction/LHD/candidate.hpp @@ -18,7 +18,10 @@ struct candidate_t { #endif static candidate_t make(const request_t* req) { - return candidate_t{DEFAULT_APP_ID, static_cast(req->obj_id)}; + candidate_t c; + c.appId = DEFAULT_APP_ID; + c.id = static_cast(req->obj_id); + return c; } inline bool operator==(const candidate_t& that) const { diff --git a/libCacheSim/cache/eviction/LIRS.c b/libCacheSim/cache/eviction/LIRS.c index 3b4cf169..c34ba0f4 100644 --- a/libCacheSim/cache/eviction/LIRS.c +++ b/libCacheSim/cache/eviction/LIRS.c @@ -1,6 +1,6 @@ // // LIRS cache eviction policy implemented by multiple LRUs -// +// // LIRS.c // libcachesim // @@ -15,7 +15,6 @@ extern "C" { // #define DEBUG_MODE 1 - typedef struct LIRS_params { cache_t *LRU_s; cache_t *LRU_q; @@ -73,45 +72,47 @@ static void LIRS_print_cache_compared_to_cacheus(cache_t *cache); */ cache_t *LIRS_init(const common_cache_params_t ccache_params, const char *cache_specific_params) { - cache_t *cache = cache_struct_init("LIRS", ccache_params, cache_specific_params); - cache->cache_init = LIRS_init; - cache->cache_free = LIRS_free; - cache->get = LIRS_get; - cache->find = LIRS_find; - cache->can_insert = LIRS_can_insert; - cache->insert = LIRS_insert; - cache->evict = LIRS_evict; - cache->remove = LIRS_remove; - cache->to_evict = LIRS_to_evict; - - if (ccache_params.consider_obj_metadata) { - cache->obj_md_size = 8 * 2; - } else { - cache->obj_md_size = 0; - } + cache_t *cache = + cache_struct_init("LIRS", ccache_params, cache_specific_params); + cache->cache_init = LIRS_init; + cache->cache_free = LIRS_free; + cache->get = LIRS_get; + cache->find = LIRS_find; + cache->can_insert = LIRS_can_insert; + cache->insert = LIRS_insert; + cache->evict = LIRS_evict; + cache->remove = LIRS_remove; + cache->to_evict = LIRS_to_evict; + + if (ccache_params.consider_obj_metadata) { + cache->obj_md_size = 8 * 2; + } else { + cache->obj_md_size = 0; + } + + cache->eviction_params = (LIRS_params_t *)malloc(sizeof(LIRS_params_t)); + LIRS_params_t *params = (LIRS_params_t *)(cache->eviction_params); - cache->eviction_params = (LIRS_params_t *)malloc(sizeof(LIRS_params_t)); - LIRS_params_t * params = (LIRS_params_t *)(cache->eviction_params); - - params->hirs_ratio = 0.01; - params->hirs_limit = MAX(1, (uint64_t)(params->hirs_ratio * cache->cache_size)); - params->lirs_limit = cache->cache_size - params->hirs_limit; - params->hirs_count = 0; - params->lirs_count = 0; - params->nonresident = 0; - - // initialize LRU for stack S and stack Q - common_cache_params_t ccache_params_s = ccache_params; - ccache_params_s.cache_size = params->lirs_limit; - common_cache_params_t ccache_params_q = ccache_params; - ccache_params_q.cache_size = params->hirs_limit; - common_cache_params_t ccache_params_nh = ccache_params; - - params->LRU_s = LRU_init(ccache_params_s, NULL); - params->LRU_q = LRU_init(ccache_params_q, NULL); - params->LRU_nh = LRU_init(ccache_params_nh, NULL); - - return cache; + params->hirs_ratio = 0.01; + params->hirs_limit = + MAX(1, (uint64_t)(params->hirs_ratio * cache->cache_size)); + params->lirs_limit = cache->cache_size - params->hirs_limit; + params->hirs_count = 0; + params->lirs_count = 0; + params->nonresident = 0; + + // initialize LRU for stack S and stack Q + common_cache_params_t ccache_params_s = ccache_params; + ccache_params_s.cache_size = params->lirs_limit; + common_cache_params_t ccache_params_q = ccache_params; + ccache_params_q.cache_size = params->hirs_limit; + common_cache_params_t ccache_params_nh = ccache_params; + + params->LRU_s = LRU_init(ccache_params_s, NULL); + params->LRU_q = LRU_init(ccache_params_q, NULL); + params->LRU_nh = LRU_init(ccache_params_nh, NULL); + + return cache; } /** @@ -120,12 +121,12 @@ cache_t *LIRS_init(const common_cache_params_t ccache_params, * @param cache */ static void LIRS_free(cache_t *cache) { - LIRS_params_t * params = (LIRS_params_t *)(cache->eviction_params); - params->LRU_s->cache_free(params->LRU_s); - params->LRU_q->cache_free(params->LRU_q); - params->LRU_nh->cache_free(params->LRU_nh); - my_free(sizeof(LIRS_params_t), params); - cache_struct_free(cache); + LIRS_params_t *params = (LIRS_params_t *)(cache->eviction_params); + params->LRU_s->cache_free(params->LRU_s); + params->LRU_q->cache_free(params->LRU_q); + params->LRU_nh->cache_free(params->LRU_nh); + my_free(sizeof(LIRS_params_t), params); + cache_struct_free(cache); } /** @@ -153,8 +154,8 @@ static bool LIRS_get(cache_t *cache, const request_t *req) { limitStack(cache); - #ifdef DEBUG_MODE - LIRS_params_t * params = (LIRS_params_t *)(cache->eviction_params); +#ifdef DEBUG_MODE + LIRS_params_t *params = (LIRS_params_t *)(cache->eviction_params); cache_t *LRU_s = params->LRU_s; if (cache->n_req >= 2) { // printf("obj:%lu size:%ld ", req->obj_id, req->obj_size); @@ -168,13 +169,16 @@ static bool LIRS_get(cache_t *cache, const request_t *req) { printf("number of requests:%ld \n", cache->n_req); printf("number of objects in S:%ld \n", LRU_s->n_obj); - printf("S(%ld):%ld %ld\n", params->lirs_limit, LRU_s->occupied_byte, params->lirs_count); - printf("Q(%ld): %ld %ld \n", params->hirs_limit, params->LRU_q->occupied_byte, params->hirs_count); - printf("NH: %ld %ld \n", params->LRU_nh->occupied_byte, params->nonresident); + printf("S(%ld):%ld %ld\n", params->lirs_limit, LRU_s->occupied_byte, + params->lirs_count); + printf("Q(%ld): %ld %ld \n", params->hirs_limit, + params->LRU_q->occupied_byte, params->hirs_count); + printf("NH: %ld %ld \n", params->LRU_nh->occupied_byte, + params->nonresident); printf("\n\n"); } - #endif - +#endif + return res; } @@ -196,17 +200,17 @@ static bool LIRS_get(cache_t *cache, const request_t *req) { */ static cache_obj_t *LIRS_find(cache_t *cache, const request_t *req, const bool update_cache) { - LIRS_params_t * params = (LIRS_params_t *)(cache->eviction_params); + LIRS_params_t *params = (LIRS_params_t *)(cache->eviction_params); cache_obj_t *obj_s = NULL; cache_obj_t *obj_q = NULL; - // find the object in S and Q, + // find the object in S and Q, // and they will be promoted to the top of the stacks obj_s = params->LRU_s->find(params->LRU_s, req, update_cache); obj_q = params->LRU_q->find(params->LRU_q, req, update_cache); - - cache_obj_t *res = NULL; + + // cache_obj_t *res = NULL; if (obj_s != NULL) { if (obj_s->LIRS.is_LIR) { // accessing an LIR block (hit) @@ -215,22 +219,22 @@ static cache_obj_t *LIRS_find(cache_t *cache, const request_t *req, } else { // accessing an HIR block in S (resident and non-resident) if (obj_s->LIRS.in_cache) { - return hit_RD_HIRinS(cache, obj_s, obj_q); //hit + return hit_RD_HIRinS(cache, obj_s, obj_q); // hit } else { return NULL; } - } + } } else if (obj_q != NULL) { // accessing an HIR blocks in Q (resident) // LRU_q->find already did the movement if (obj_q->LIRS.in_cache) { - return hit_RD_HIRinQ(cache, obj_q); //hit + return hit_RD_HIRinQ(cache, obj_q); // hit } else { assert(false); return NULL; } } else { - return NULL; //miss + return NULL; // miss } } @@ -251,10 +255,9 @@ static cache_obj_t *LIRS_insert(cache_t *cache, const request_t *req) { cache_obj_t *obj_s = params->LRU_s->find(params->LRU_s, req, false); cache_obj_t *obj_q = params->LRU_q->find(params->LRU_q, req, false); - //Upon accessing an HIR non-resident in S - if (obj_s != NULL && obj_s->LIRS.is_LIR == false - && obj_s->LIRS.in_cache == false) { - + // Upon accessing an HIR non-resident in S + if (obj_s != NULL && obj_s->LIRS.is_LIR == false && + obj_s->LIRS.in_cache == false) { // change status of the block to be LIR (obj_s is already promoted to top) obj_s->LIRS.is_LIR = true; obj_s->LIRS.in_cache = true; @@ -265,7 +268,6 @@ static cache_obj_t *LIRS_insert(cache_t *cache, const request_t *req) { // Upon accessing an HIR non-resident in Q if (obj_s == NULL && obj_q != NULL && obj_q->LIRS.in_cache == false) { - // insert the req into S and place it on the top of S cache_obj_t *inserted_obj_s = params->LRU_s->insert(params->LRU_s, req); inserted_obj_s->LIRS.is_LIR = false; @@ -278,12 +280,11 @@ static cache_obj_t *LIRS_insert(cache_t *cache, const request_t *req) { params->hirs_count += inserted_obj_s->obj_size; cache->occupied_byte += inserted_obj_s->obj_size + cache->obj_md_size; cache->n_obj += 1; - } // Upon accessing blocks neither in S nor Q if (obj_s == NULL && obj_q == NULL) { - // when LIR block set is not full, + // when LIR block set is not full, // all reference blocks are given an LIR status if (params->lirs_count + req->obj_size <= params->lirs_limit) { cache_obj_t *inserted_obj_s = params->LRU_s->insert(params->LRU_s, req); @@ -292,7 +293,7 @@ static cache_obj_t *LIRS_insert(cache_t *cache, const request_t *req) { params->lirs_count += inserted_obj_s->obj_size; cache->occupied_byte += inserted_obj_s->obj_size + cache->obj_md_size; } else if (params->hirs_count + req->obj_size <= params->hirs_limit) { - // when LIR block set is full, + // when LIR block set is full, // all reference blocks are given an HIR status cache_obj_t *inserted_obj_s = params->LRU_s->insert(params->LRU_s, req); inserted_obj_s->LIRS.is_LIR = false; @@ -352,11 +353,11 @@ static void LIRS_evict(cache_t *cache, const request_t *req) { cache_obj_t *obj_s = params->LRU_s->find(params->LRU_s, req, false); cache_obj_t *obj_q = params->LRU_q->find(params->LRU_q, req, false); - //Upon accessing an HIR non-resident in S - if (obj_s != NULL && obj_s->LIRS.is_LIR == false - && obj_s->LIRS.in_cache == false) { - //remove the HIR resident at the front of Q - while (params->hirs_count >= params->hirs_limit){ + // Upon accessing an HIR non-resident in S + if (obj_s != NULL && obj_s->LIRS.is_LIR == false && + obj_s->LIRS.in_cache == false) { + // remove the HIR resident at the front of Q + while (params->hirs_count >= params->hirs_limit) { evictHIR(cache); } @@ -364,22 +365,21 @@ static void LIRS_evict(cache_t *cache, const request_t *req) { evictLIR(cache); } - // Upon accessing an HIR non-resident in Q if (obj_s == NULL && obj_q != NULL && obj_q->LIRS.in_cache == false) { - //remove the HIR resident at the front of Q - while (params->hirs_count >= params->hirs_limit){ + // remove the HIR resident at the front of Q + while (params->hirs_count >= params->hirs_limit) { evictHIR(cache); } } // Upon accessing blocks neither in S nor Q if (obj_s == NULL && obj_q == NULL) { - if (params->lirs_count + req->obj_size > params->lirs_limit - && params->hirs_count + req->obj_size > params->hirs_limit) { + if (params->lirs_count + req->obj_size > params->lirs_limit && + params->hirs_count + req->obj_size > params->hirs_limit) { // when both LIR and HIR block sets are full, // the curcumstance is same as accessing an HIR non-resident not in S - //remove the HIR resident at the front of Q + // remove the HIR resident at the front of Q evictHIR(cache); } } @@ -408,11 +408,11 @@ static bool LIRS_remove(cache_t *cache, obj_id_t obj_id) { // object in S stack (or in Q stack) if (obj_s != NULL) { if (obj_s->LIRS.is_LIR) { - params->LRU_s->remove(params->LRU_s, obj_id); - params->lirs_count -= obj_s->obj_size; - cache->occupied_byte -= obj_s->obj_size; - cache->n_obj--; - LIRS_prune(cache); + params->LRU_s->remove(params->LRU_s, obj_id); + params->lirs_count -= obj_s->obj_size; + cache->occupied_byte -= obj_s->obj_size; + cache->n_obj--; + LIRS_prune(cache); } else { if (obj_s->LIRS.in_cache) { params->LRU_s->remove(params->LRU_s, obj_id); @@ -424,7 +424,7 @@ static bool LIRS_remove(cache_t *cache, obj_id_t obj_id) { params->nonresident -= obj_s->obj_size; } if (obj_q != NULL) { - params->LRU_q->remove(params->LRU_q, obj_id); + params->LRU_q->remove(params->LRU_q, obj_id); } } } else if (obj_q != NULL) { @@ -453,11 +453,10 @@ static bool LIRS_remove(cache_t *cache, obj_id_t obj_id) { // *********************************************************************** /* LIRS cannot insert an object larger than Q stack size. -* This function also promise there will be enough space for the object -* to be inserted into the cache. + * This function also promise there will be enough space for the object + * to be inserted into the cache. */ bool LIRS_can_insert(cache_t *cache, const request_t *req) { - bool can_insert = cache_can_insert_default(cache, req); if (can_insert == false) { return false; @@ -467,14 +466,14 @@ bool LIRS_can_insert(cache_t *cache, const request_t *req) { cache_obj_t *obj_q = params->LRU_q->find(params->LRU_q, req, false); // accessing an HIR non-resident in S - if (obj_s != NULL && obj_s->LIRS.is_LIR == false - && obj_s->LIRS.in_cache == false) { + if (obj_s != NULL && obj_s->LIRS.is_LIR == false && + obj_s->LIRS.in_cache == false) { while (params->lirs_count + obj_s->obj_size > params->lirs_limit) { evictLIR(cache); } bool res = params->LRU_nh->remove(params->LRU_nh, obj_s->obj_id); - if (res){ + if (res) { params->nonresident -= obj_s->obj_size; } @@ -483,13 +482,14 @@ bool LIRS_can_insert(cache_t *cache, const request_t *req) { // accessing blocks neither in S nor Q if (obj_s == NULL && obj_q == NULL) { - if (req->obj_size > params->lirs_limit || req->obj_size > params->hirs_limit) { + if (req->obj_size > params->lirs_limit || + req->obj_size > params->hirs_limit) { WARN_ONCE("object size too large\n"); // printf("request num: %ld\n", cache->n_req); return false; } - if (params->lirs_count + req->obj_size > params->lirs_limit - && params->hirs_count + req->obj_size > params->hirs_limit) { + if (params->lirs_count + req->obj_size > params->lirs_limit && + params->hirs_count + req->obj_size > params->hirs_limit) { // when both LIR and HIR block sets are full, // the curcumstance is same as accessing an HIR non-resident not in S while (params->hirs_count + req->obj_size > params->hirs_limit) { @@ -500,19 +500,18 @@ bool LIRS_can_insert(cache_t *cache, const request_t *req) { return true; } - INFO("LIRS_can_insert: should not reach here. n_req = %ld\n", cache->n_req); - assert(false); + INFO("LIRS_can_insert: should not reach here. n_req = %ld\n", + (long)cache->n_req); + abort(); } - static void LIRS_prune(cache_t *cache) { - LIRS_params_t *params = (LIRS_params_t *)(cache->eviction_params); cache_t *LRU_s = params->LRU_s; LRU_params_t *lru_s_params = (LRU_params_t *)(LRU_s->eviction_params); cache_obj_t *obj_to_remove = lru_s_params->q_tail; - + while (obj_to_remove != lru_s_params->q_head) { if (obj_to_remove->LIRS.is_LIR) { break; @@ -521,7 +520,7 @@ static void LIRS_prune(cache_t *cache) { // remove obj from LRU_nh if (obj_to_remove->LIRS.in_cache == false) { bool res = params->LRU_nh->remove(params->LRU_nh, obj_to_remove->obj_id); - if (res){ + if (res) { params->nonresident -= obj_to_remove->obj_size; } } @@ -556,9 +555,9 @@ static cache_obj_t *hit_RD_HIRinS(cache_t *cache, cache_obj_t *cache_obj_s, static cache_obj_t *hit_NR_HIRinS(cache_t *cache, cache_obj_t *cache_obj_s) { LIRS_params_t *params = (LIRS_params_t *)(cache->eviction_params); - + bool res = params->LRU_nh->remove(params->LRU_nh, cache_obj_s->obj_id); - if (res){ + if (res) { params->nonresident -= cache_obj_s->obj_size; } return NULL; @@ -566,7 +565,7 @@ static cache_obj_t *hit_NR_HIRinS(cache_t *cache, cache_obj_t *cache_obj_s) { static cache_obj_t *hit_RD_HIRinQ(cache_t *cache, cache_obj_t *cache_obj_q) { LIRS_params_t *params = (LIRS_params_t *)(cache->eviction_params); - + static __thread request_t *req_local = NULL; if (req_local == NULL) { req_local = new_request(); @@ -577,14 +576,15 @@ static cache_obj_t *hit_RD_HIRinQ(cache_t *cache, cache_obj_t *cache_obj_q) { evictLIR(cache); } params->LRU_s->insert(params->LRU_s, req_local); - cache_obj_t *obj_to_update = params->LRU_s->find(params->LRU_s, req_local, false); + cache_obj_t *obj_to_update = + params->LRU_s->find(params->LRU_s, req_local, false); obj_to_update->LIRS.is_LIR = false; obj_to_update->LIRS.in_cache = true; return obj_to_update; } -static void evictLIR(cache_t *cache){ +static void evictLIR(cache_t *cache) { LIRS_params_t *params = (LIRS_params_t *)(cache->eviction_params); cache_obj_t *obj_to_evict = params->LRU_s->to_evict(params->LRU_s, NULL); @@ -602,10 +602,11 @@ static void evictLIR(cache_t *cache){ if (req_local->obj_size <= params->hirs_limit) { while (params->hirs_count + req_local->obj_size > params->hirs_limit) { evictHIR(cache); - } + } params->LRU_q->insert(params->LRU_q, req_local); - - cache_obj_t *obj_to_update = params->LRU_q->find(params->LRU_q, req_local, false); + + cache_obj_t *obj_to_update = + params->LRU_q->find(params->LRU_q, req_local, false); obj_to_update->LIRS.is_LIR = false; obj_to_update->LIRS.in_cache = true; @@ -617,12 +618,12 @@ static void evictLIR(cache_t *cache){ LIRS_prune(cache); } -static bool evictHIR(cache_t *cache){ +static bool evictHIR(cache_t *cache) { LIRS_params_t *params = (LIRS_params_t *)(cache->eviction_params); cache_obj_t *obj_to_evict = params->LRU_q->to_evict(params->LRU_q, NULL); - //update the corresponding block in S to be non-resident + // update the corresponding block in S to be non-resident static __thread request_t *req_local = NULL; if (req_local == NULL) { req_local = new_request(); @@ -632,7 +633,8 @@ static bool evictHIR(cache_t *cache){ params->hirs_count -= obj_to_evict->obj_size; params->LRU_q->evict(params->LRU_q, NULL); - cache_obj_t *obj_to_update = params->LRU_s->find(params->LRU_s, req_local, false); + cache_obj_t *obj_to_update = + params->LRU_s->find(params->LRU_s, req_local, false); if (obj_to_update != NULL) { obj_to_update->LIRS.in_cache = false; params->LRU_nh->insert(params->LRU_nh, req_local); @@ -654,7 +656,7 @@ static void limitStack(cache_t *cache) { params->nonresident -= obj_to_evict->obj_size; params->LRU_s->remove(params->LRU_s, obj_to_evict->obj_id); params->LRU_nh->evict(params->LRU_nh, NULL); - } else{ + } else { break; } } @@ -667,80 +669,68 @@ static void limitStack(cache_t *cache) { static void LIRS_print_cache(cache_t *cache) { LIRS_params_t *params = (LIRS_params_t *)cache->eviction_params; - printf("S Stack: %lu:%lu %lu:%lu \n", - params->lirs_limit, params->lirs_count, - params->hirs_limit, params->hirs_count); - cache_obj_t *obj = - ((LRU_params_t *)params->LRU_s->eviction_params)->q_head; + printf("S Stack: %lu:%lu %lu:%lu \n", (unsigned long)params->lirs_limit, + (unsigned long)params->lirs_count, (unsigned long)params->hirs_limit, + (unsigned long)params->hirs_count); + cache_obj_t *obj = ((LRU_params_t *)params->LRU_s->eviction_params)->q_head; while (obj) { - printf("%ld(%u, %s, %s)->", obj->obj_id, obj->obj_size, - obj->LIRS.in_cache ? "R" : "N", - obj->LIRS.is_LIR ? "L" : "H"); + printf("%ld(%u, %s, %s)->", (long)obj->obj_id, obj->obj_size, + obj->LIRS.in_cache ? "R" : "N", obj->LIRS.is_LIR ? "L" : "H"); obj = obj->queue.next; } printf("\n"); printf("Q Stack: \n"); - cache_obj_t *obj_q = - ((LRU_params_t *)params->LRU_q->eviction_params)->q_head; + cache_obj_t *obj_q = ((LRU_params_t *)params->LRU_q->eviction_params)->q_head; while (obj_q) { - printf("%ld(%u, %s, %s)->", obj_q->obj_id, obj_q->obj_size, - obj_q->LIRS.in_cache ? "R" : "N", - obj_q->LIRS.is_LIR ? "L" : "H"); + printf("%ld(%u, %s, %s)->", (long)obj_q->obj_id, obj_q->obj_size, + obj_q->LIRS.in_cache ? "R" : "N", obj_q->LIRS.is_LIR ? "L" : "H"); obj_q = obj_q->queue.next; } printf("\n"); printf("NH Stack: \n"); cache_obj_t *obj_nh = - ((LRU_params_t *)params->LRU_nh->eviction_params)->q_head; + ((LRU_params_t *)params->LRU_nh->eviction_params)->q_head; while (obj_nh) { - printf("%ld(%u, %s, %s)->", obj_nh->obj_id, obj_nh->obj_size, - obj_nh->LIRS.in_cache ? "R" : "N", - obj_nh->LIRS.is_LIR ? "L" : "H"); + printf("%ld(%u, %s, %s)->", (long)obj_nh->obj_id, obj_nh->obj_size, + obj_nh->LIRS.in_cache ? "R" : "N", obj_nh->LIRS.is_LIR ? "L" : "H"); obj_nh = obj_nh->queue.next; } printf("\n\n"); - } static void LIRS_print_cache_compared_to_cacheus(cache_t *cache) { - LIRS_params_t *params = (LIRS_params_t *)cache->eviction_params; + LIRS_params_t *params = (LIRS_params_t *)cache->eviction_params; printf("S:\n"); - cache_obj_t *obj = - ((LRU_params_t *)params->LRU_s->eviction_params)->q_tail; + cache_obj_t *obj = ((LRU_params_t *)params->LRU_s->eviction_params)->q_tail; while (obj) { - printf("(o=%ld, is_LIR=%s, in_cache=%s)\n", - obj->obj_id, - obj->LIRS.is_LIR ? "True" : "False", - obj->LIRS.in_cache ? "True" : "False"); + printf("(o=%ld, is_LIR=%s, in_cache=%s)\n", (long)obj->obj_id, + obj->LIRS.is_LIR ? "True" : "False", + obj->LIRS.in_cache ? "True" : "False"); obj = obj->queue.prev; } printf("Q:\n"); - cache_obj_t *obj_q = - ((LRU_params_t *)params->LRU_q->eviction_params)->q_tail; + cache_obj_t *obj_q = ((LRU_params_t *)params->LRU_q->eviction_params)->q_tail; while (obj_q) { - printf("(o=%ld, is_LIR=%s, in_cache=%s)\n", - obj_q->obj_id, - obj_q->LIRS.is_LIR ? "True" : "False", - obj_q->LIRS.in_cache ? "True" : "False"); + printf("(o=%ld, is_LIR=%s, in_cache=%s)\n", (long)obj_q->obj_id, + obj_q->LIRS.is_LIR ? "True" : "False", + obj_q->LIRS.in_cache ? "True" : "False"); obj_q = obj_q->queue.prev; } printf("NH:\n"); cache_obj_t *obj_nh = - ((LRU_params_t *)params->LRU_nh->eviction_params)->q_tail; + ((LRU_params_t *)params->LRU_nh->eviction_params)->q_tail; while (obj_nh) { - printf("(o=%ld, is_LIR=%s, in_cache=%s)\n", - obj_nh->obj_id, - obj_nh->LIRS.is_LIR ? "True" : "False", - obj_nh->LIRS.in_cache ? "True" : "False"); + printf("(o=%ld, is_LIR=%s, in_cache=%s)\n", (long)obj_nh->obj_id, + obj_nh->LIRS.is_LIR ? "True" : "False", + obj_nh->LIRS.in_cache ? "True" : "False"); obj_nh = obj_nh->queue.prev; } printf("\n"); - } #ifdef __cplusplus diff --git a/libCacheSim/cache/eviction/LRU.c b/libCacheSim/cache/eviction/LRU.c index f7f74c9c..f5e93d88 100644 --- a/libCacheSim/cache/eviction/LRU.c +++ b/libCacheSim/cache/eviction/LRU.c @@ -48,7 +48,8 @@ static void LRU_print_cache(const cache_t *cache); */ cache_t *LRU_init(const common_cache_params_t ccache_params, const char *cache_specific_params) { - cache_t *cache = cache_struct_init("LRU", ccache_params, cache_specific_params); + cache_t *cache = + cache_struct_init("LRU", ccache_params, cache_specific_params); cache->cache_init = LRU_init; cache->cache_free = LRU_free; cache->get = LRU_get; @@ -75,8 +76,8 @@ cache_t *LRU_init(const common_cache_params_t ccache_params, LRU_params_t *params = malloc(sizeof(LRU_params_t)); params->q_head = NULL; params->q_tail = NULL; + pthread_mutex_init(¶ms->mutex_, NULL); cache->eviction_params = params; - return cache; } @@ -85,7 +86,11 @@ cache_t *LRU_init(const common_cache_params_t ccache_params, * * @param cache */ -static void LRU_free(cache_t *cache) { cache_struct_free(cache); } +static void LRU_free(cache_t *cache) { + LRU_params_t *params = (LRU_params_t *)cache->eviction_params; + free_list(¶ms->q_head, ¶ms->q_tail); + cache_struct_free(cache); +} /** * @brief this function is the user facing API @@ -130,14 +135,18 @@ static cache_obj_t *LRU_find(cache_t *cache, const request_t *req, const bool update_cache) { LRU_params_t *params = (LRU_params_t *)cache->eviction_params; cache_obj_t *cache_obj = cache_find_base(cache, req, update_cache); + + pthread_mutex_lock(¶ms->mutex_); + if (cache_obj && likely(update_cache) && cache_obj_in_cache(cache_obj)) { - if (cache_obj && likely(update_cache)) { /* lru_head is the newest, move cur obj to lru_head */ #ifdef USE_BELADY if (req->next_access_vtime != INT64_MAX) #endif move_obj_to_head(¶ms->q_head, ¶ms->q_tail, cache_obj); } + pthread_mutex_unlock(¶ms->mutex_); + return cache_obj; } @@ -155,8 +164,13 @@ static cache_obj_t *LRU_insert(cache_t *cache, const request_t *req) { LRU_params_t *params = (LRU_params_t *)cache->eviction_params; cache_obj_t *obj = cache_insert_base(cache, req); + + pthread_mutex_lock(¶ms->mutex_); prepend_obj_to_head(¶ms->q_head, ¶ms->q_tail, obj); + cache_obj_set_in_cache(obj, true); + pthread_mutex_unlock(¶ms->mutex_); + // Now the object is ready for access. return obj; } @@ -172,46 +186,46 @@ static cache_obj_t *LRU_insert(cache_t *cache, const request_t *req) { */ static cache_obj_t *LRU_to_evict(cache_t *cache, const request_t *req) { LRU_params_t *params = (LRU_params_t *)cache->eviction_params; - - DEBUG_ASSERT(params->q_tail != NULL || cache->occupied_byte == 0); - - cache->to_evict_candidate_gen_vtime = cache->n_req; - return params->q_tail; -} - -/** - * @brief evict an object from the cache - * it needs to call cache_evict_base before returning - * which updates some metadata such as n_obj, occupied size, and hash table - * - * @param cache - * @param req not used - */ -static void LRU_evict(cache_t *cache, const request_t *req) { - LRU_params_t *params = (LRU_params_t *)cache->eviction_params; - cache_obj_t *obj_to_evict = params->q_tail; DEBUG_ASSERT(params->q_tail != NULL); // we can simply call remove_obj_from_list here, but for the best performance, // we chose to do it manually // remove_obj_from_list(¶ms->q_head, ¶ms->q_tail, obj) + pthread_mutex_lock(¶ms->mutex_); + cache_obj_t *obj_to_evict = params->q_tail; + cache_obj_set_in_cache(obj_to_evict, false); + + // Remove the item from the hash table and prefetcher (if existed). params->q_tail = params->q_tail->queue.prev; if (likely(params->q_tail != NULL)) { params->q_tail->queue.next = NULL; } else { /* cache->n_obj has not been updated */ - DEBUG_ASSERT(cache->n_obj == 1); + // DEBUG_ASSERT(cache->n_obj == 1); params->q_head = NULL; } #if defined(TRACK_DEMOTION) if (cache->track_demotion) - printf("%ld demote %ld %ld\n", cache->n_req, obj_to_evict->create_time, - obj_to_evict->misc.next_access_vtime); + printf("%ld demote %ld %ld\n", cache->n_req, obj_to_evict->create_time, + obj_to_evict->misc.next_access_vtime); #endif - + pthread_mutex_unlock(¶ms->mutex_); cache_evict_base(cache, obj_to_evict, true); + return obj_to_evict; +} + +/** + * @brief evict an object from the cache + * it needs to call cache_evict_base before returning + * which updates some metadata such as n_obj, occupied size, and hash table + * + * @param cache + * @param req not used + */ +static void LRU_evict(cache_t *cache, const request_t *req) { + LRU_to_evict(cache, req); } /** @@ -231,10 +245,6 @@ static void LRU_evict(cache_t *cache, const request_t *req) { */ static void LRU_remove_obj(cache_t *cache, cache_obj_t *obj) { assert(obj != NULL); - - LRU_params_t *params = (LRU_params_t *)cache->eviction_params; - - remove_obj_from_list(¶ms->q_head, ¶ms->q_tail, obj); cache_remove_obj_base(cache, obj, true); } @@ -252,31 +262,32 @@ static void LRU_remove_obj(cache_t *cache, cache_obj_t *obj) { * cache */ static bool LRU_remove(cache_t *cache, const obj_id_t obj_id) { - cache_obj_t *obj = hashtable_find_obj_id(cache->hashtable, obj_id); - if (obj == NULL) { - return false; + cache_obj_t *obj = hashtable_delete_obj_id(cache->hashtable, obj_id); + if(obj != NULL){ + fetch_sub(&cache->n_obj, 1); + return true; } - LRU_params_t *params = (LRU_params_t *)cache->eviction_params; - - remove_obj_from_list(¶ms->q_head, ¶ms->q_tail, obj); - cache_remove_obj_base(cache, obj, true); - - return true; + else + return false; } static void LRU_print_cache(const cache_t *cache) { LRU_params_t *params = (LRU_params_t *)cache->eviction_params; + pthread_mutex_lock(¶ms->mutex_); + cache_obj_t *cur = params->q_head; // print from the most recent to the least recent if (cur == NULL) { + pthread_mutex_unlock(¶ms->mutex_); printf("empty\n"); return; } while (cur != NULL) { - printf("%lu->", cur->obj_id); + printf("%lu->", (unsigned long)cur->obj_id); cur = cur->queue.next; } printf("END\n"); + pthread_mutex_unlock(¶ms->mutex_); } #ifdef __cplusplus diff --git a/libCacheSim/cache/eviction/LeCaR.c b/libCacheSim/cache/eviction/LeCaR.c index 5cba6bf5..915dfdc7 100644 --- a/libCacheSim/cache/eviction/LeCaR.c +++ b/libCacheSim/cache/eviction/LeCaR.c @@ -585,8 +585,8 @@ bool LeCaR_remove(cache_t *cache, obj_id_t obj_id) { static const char *LeCaR_current_params(cache_t *cache, LeCaR_params_t *params) { static __thread char params_str[128]; - int n = snprintf(params_str, 128, "update-weight=%d,lru-weight=%.lf", - params->update_weight, params->w_lru); + snprintf(params_str, 128, "update-weight=%d,lru-weight=%.lf", + params->update_weight, params->w_lru); return params_str; } diff --git a/libCacheSim/cache/eviction/LeCaRv0.c b/libCacheSim/cache/eviction/LeCaRv0.c index c38a9641..1f57679c 100644 --- a/libCacheSim/cache/eviction/LeCaRv0.c +++ b/libCacheSim/cache/eviction/LeCaRv0.c @@ -18,8 +18,6 @@ extern "C" { #endif -// #define USE_MYCLOCK - typedef struct LeCaRv0_params { cache_t *LRU; // LRU cache_t *LRU_g; // eviction history of LRU @@ -118,12 +116,6 @@ cache_t *LeCaRv0_init(const common_cache_params_t ccache_params, params->LRU_g = LRU_init(ccache_params_g, NULL); params->LFU_g = LRU_init(ccache_params_g, NULL); -#ifdef USE_MYCLOCK - params->LRU->cache_free(params->LRU); - params->LRU = MyClock_init(ccache_params, NULL); - snprintf(cache->cache_name, CACHE_NAME_ARRAY_LEN, "LeCaRv0-myclock"); -#endif - return cache; } @@ -247,6 +239,7 @@ static cache_obj_t *LeCaRv0_to_evict(cache_t *cache, const request_t *req) { } else { params->LFU->to_evict(params->LFU, req); } + return NULL; } /** diff --git a/libCacheSim/cache/eviction/QDLP.c b/libCacheSim/cache/eviction/QDLP.c index 43e57147..8c52eed9 100644 --- a/libCacheSim/cache/eviction/QDLP.c +++ b/libCacheSim/cache/eviction/QDLP.c @@ -440,7 +440,6 @@ static void QDLP_parse_params(cache_t *cache, char *params_str = strdup(cache_specific_params); char *old_params_str = params_str; - char *end; while (params_str != NULL && params_str[0] != '\0') { /* different parameters are separated by comma, diff --git a/libCacheSim/cache/eviction/Random.c b/libCacheSim/cache/eviction/Random.c index c89e3ab7..94f9d374 100644 --- a/libCacheSim/cache/eviction/Random.c +++ b/libCacheSim/cache/eviction/Random.c @@ -48,7 +48,8 @@ cache_t *Random_init(const common_cache_params_t ccache_params, common_cache_params_t ccache_params_copy = ccache_params; ccache_params_copy.hashpower = MAX(12, ccache_params_copy.hashpower - 8); - cache_t *cache = cache_struct_init("Random", ccache_params, cache_specific_params); + cache_t *cache = + cache_struct_init("Random", ccache_params_copy, cache_specific_params); cache->cache_init = Random_init; cache->cache_free = Random_free; cache->get = Random_get; diff --git a/libCacheSim/cache/eviction/RandomTwo.c b/libCacheSim/cache/eviction/RandomTwo.c new file mode 100644 index 00000000..8392d0c4 --- /dev/null +++ b/libCacheSim/cache/eviction/RandomTwo.c @@ -0,0 +1,202 @@ +// +// RandomTwo.c +// libCacheSim +// +// Picks two objects at random and evicts the one that is the least recently +// used RandomTwo eviction +// +// Created by Juncheng on 8/2/16. +// Copyright © 2016 Juncheng. All rights reserved. +// + +#include "../../dataStructure/hashtable/hashtable.h" +#include "../../include/libCacheSim/evictionAlgo.h" +#include "../../include/libCacheSim/macro.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// *********************************************************************** +// **** **** +// **** function declarations **** +// **** **** +// *********************************************************************** + +static void RandomTwo_free(cache_t *cache); +static bool RandomTwo_get(cache_t *cache, const request_t *req); +static cache_obj_t *RandomTwo_find(cache_t *cache, const request_t *req, + const bool update_cache); +static cache_obj_t *RandomTwo_insert(cache_t *cache, const request_t *req); +static cache_obj_t *RandomTwo_to_evict(cache_t *cache, const request_t *req); +static void RandomTwo_evict(cache_t *cache, const request_t *req); +static bool RandomTwo_remove(cache_t *cache, const obj_id_t obj_id); + +// *********************************************************************** +// **** **** +// **** end user facing functions **** +// **** **** +// **** init, free, get **** +// *********************************************************************** +/** + * @brief initialize a RandomTwo cache + * + * @param ccache_params some common cache parameters + * @param cache_specific_params RandomTwo specific parameters, should be NULL + */ +cache_t *RandomTwo_init(const common_cache_params_t ccache_params, + const char *cache_specific_params) { + common_cache_params_t ccache_params_copy = ccache_params; + ccache_params_copy.hashpower = MAX(12, ccache_params_copy.hashpower - 8); + + cache_t *cache = + cache_struct_init("RandomTwo", ccache_params_copy, cache_specific_params); + cache->cache_init = RandomTwo_init; + cache->cache_free = RandomTwo_free; + cache->get = RandomTwo_get; + cache->find = RandomTwo_find; + cache->insert = RandomTwo_insert; + cache->to_evict = RandomTwo_to_evict; + cache->evict = RandomTwo_evict; + cache->remove = RandomTwo_remove; + + return cache; +} + +/** + * free resources used by this cache + * + * @param cache + */ +static void RandomTwo_free(cache_t *cache) { cache_struct_free(cache); } + +/** + * @brief this function is the user facing API + * it performs the following logic + * + * ``` + * if obj in cache: + * update_metadata + * return true + * else: + * if cache does not have enough space: + * evict until it has space to insert + * insert the object + * return false + * ``` + * + * @param cache + * @param req + * @return true if cache hit, false if cache miss + */ +static bool RandomTwo_get(cache_t *cache, const request_t *req) { + return cache_get_base(cache, req); +} + +// *********************************************************************** +// **** **** +// **** developer facing APIs (used by cache developer) **** +// **** **** +// *********************************************************************** + +/** + * @brief check whether an object is in the cache + * + * @param cache + * @param req + * @param update_cache whether to update the cache, + * if true, the object is promoted + * and if the object is expired, it is removed from the cache + * @return true on hit, false on miss + */ +static cache_obj_t *RandomTwo_find(cache_t *cache, const request_t *req, + const bool update_cache) { + cache_obj_t *obj = cache_find_base(cache, req, update_cache); + if (obj != NULL && update_cache) { + obj->RandomTwo.last_access_vtime = cache->n_req; + } + + return obj; +} + +/** + * @brief insert an object into the cache, + * update the hash table and cache metadata + * this function assumes the cache has enough space + * and eviction is not part of this function + * + * @param cache + * @param req + * @return the inserted object + */ +static cache_obj_t *RandomTwo_insert(cache_t *cache, const request_t *req) { + cache_obj_t *obj = cache_insert_base(cache, req); + obj->RandomTwo.last_access_vtime = cache->n_req; + + return obj; +} + +/** + * @brief find the object to be evicted + * this function does not actually evict the object or update metadata + * not all eviction algorithms support this function + * because the eviction logic cannot be decoupled from finding eviction + * candidate, so use assert(false) if you cannot support this function + * + * @param cache the cache + * @return the object to be evicted + */ +static cache_obj_t *RandomTwo_to_evict(cache_t *cache, const request_t *req) { + cache_obj_t *obj_to_evict1 = hashtable_rand_obj(cache->hashtable); + cache_obj_t *obj_to_evict2 = hashtable_rand_obj(cache->hashtable); + if (obj_to_evict1->RandomTwo.last_access_vtime < + obj_to_evict2->RandomTwo.last_access_vtime) + return obj_to_evict1; + else + return obj_to_evict2; +} + +/** + * @brief evict an object from the cache + * it needs to call cache_evict_base before returning + * which updates some metadata such as n_obj, occupied size, and hash table + * + * @param cache + * @param req not used + */ +static void RandomTwo_evict(cache_t *cache, const request_t *req) { + cache_obj_t *obj_to_evict1 = hashtable_rand_obj(cache->hashtable); + cache_obj_t *obj_to_evict2 = hashtable_rand_obj(cache->hashtable); + if (obj_to_evict1->RandomTwo.last_access_vtime < + obj_to_evict2->RandomTwo.last_access_vtime) + cache_evict_base(cache, obj_to_evict1, true); + else + cache_evict_base(cache, obj_to_evict2, true); +} + +/** + * @brief remove an object from the cache + * this is different from cache_evict because it is used to for user trigger + * remove, and eviction is used by the cache to make space for new objects + * + * it needs to call cache_remove_obj_base before returning + * which updates some metadata such as n_obj, occupied size, and hash table + * + * @param cache + * @param obj_id + * @return true if the object is removed, false if the object is not in the + * cache + */ +static bool RandomTwo_remove(cache_t *cache, const obj_id_t obj_id) { + cache_obj_t *obj = hashtable_find_obj_id(cache->hashtable, obj_id); + if (obj == NULL) { + return false; + } + cache_remove_obj_base(cache, obj, true); + + return true; +} + +#ifdef __cplusplus +} +#endif diff --git a/libCacheSim/cache/eviction/S3FIFO.c b/libCacheSim/cache/eviction/S3FIFO.c index d80f80f6..8fcaf490 100644 --- a/libCacheSim/cache/eviction/S3FIFO.c +++ b/libCacheSim/cache/eviction/S3FIFO.c @@ -297,7 +297,7 @@ static cache_obj_t *S3FIFO_insert(cache_t *cache, const request_t *req) { obj->create_time = cache->n_req; #endif - obj->S3FIFO.freq == 0; + obj->S3FIFO.freq = 0; return obj; } @@ -371,8 +371,8 @@ static void S3FIFO_evict_fifo(cache_t *cache, const request_t *req) { static void S3FIFO_evict_main(cache_t *cache, const request_t *req) { S3FIFO_params_t *params = (S3FIFO_params_t *)cache->eviction_params; - cache_t *fifo = params->fifo; - cache_t *ghost = params->fifo_ghost; + // cache_t *fifo = params->fifo; + // cache_t *ghost = params->fifo_ghost; cache_t *main = params->main_cache; // evict from main cache @@ -408,7 +408,7 @@ static void S3FIFO_evict_main(cache_t *cache, const request_t *req) { // main->evict(main, req); bool removed = main->remove(main, obj_to_evict->obj_id); if (!removed) { - ERROR("cannot remove obj %ld\n", obj_to_evict->obj_id); + ERROR("cannot remove obj %ld\n", (long)obj_to_evict->obj_id); } has_evicted = true; @@ -429,7 +429,7 @@ static void S3FIFO_evict(cache_t *cache, const request_t *req) { S3FIFO_params_t *params = (S3FIFO_params_t *)cache->eviction_params; cache_t *fifo = params->fifo; - cache_t *ghost = params->fifo_ghost; + // cache_t *ghost = params->fifo_ghost; cache_t *main = params->main_cache; if (main->get_occupied_byte(main) > main->cache_size || @@ -499,7 +499,7 @@ static void S3FIFO_parse_params(cache_t *cache, char *params_str = strdup(cache_specific_params); char *old_params_str = params_str; - char *end; + // char *end; while (params_str != NULL && params_str[0] != '\0') { /* different parameters are separated by comma, diff --git a/libCacheSim/cache/eviction/S3FIFOd.c b/libCacheSim/cache/eviction/S3FIFOd.c index a9ec28e3..28859b69 100644 --- a/libCacheSim/cache/eviction/S3FIFOd.c +++ b/libCacheSim/cache/eviction/S3FIFOd.c @@ -358,7 +358,7 @@ static cache_obj_t *S3FIFOd_insert(cache_t *cache, const request_t *req) { /* insert into the ARC */ params->hit_on_ghost = false; params->main_cache->get(params->main_cache, req); - obj = obj = params->main_cache->find(params->main_cache, req, false); + obj = params->main_cache->find(params->main_cache, req, false); } else { /* insert into the fifo */ obj = params->fifo->insert(params->fifo, req); @@ -435,7 +435,7 @@ static void S3FIFOd_evict(cache_t *cache, const request_t *req) { while (main->get_occupied_byte(main) > main->cache_size) { // evict from main cache - cache_obj_t *obj = main->to_evict(main, req); + obj = main->to_evict(main, req); copy_cache_obj_to_request(params->req_local, obj); params->main_cache_eviction->get(params->main_cache_eviction, params->req_local); @@ -463,7 +463,7 @@ static void S3FIFOd_evict(cache_t *cache, const request_t *req) { while (main->get_occupied_byte(main) > main->cache_size) { // evict from main cache - cache_obj_t *obj = main->to_evict(main, req); + obj = main->to_evict(main, req); copy_cache_obj_to_request(params->req_local, obj); params->main_cache_eviction->get(params->main_cache_eviction, params->req_local); @@ -536,7 +536,7 @@ static void S3FIFOd_parse_params(cache_t *cache, char *params_str = strdup(cache_specific_params); char *old_params_str = params_str; - char *end; + // char *end; while (params_str != NULL && params_str[0] != '\0') { /* different parameters are separated by comma, diff --git a/libCacheSim/cache/eviction/SLRU.c b/libCacheSim/cache/eviction/SLRU.c index 5f51b6ef..cee57918 100644 --- a/libCacheSim/cache/eviction/SLRU.c +++ b/libCacheSim/cache/eviction/SLRU.c @@ -230,6 +230,7 @@ static bool SLRU_get(cache_t *cache, const request_t *req) { #else bool ck = cache_get_base(cache, req); #endif + return ck; } /** @@ -546,9 +547,9 @@ static void _SLRU_verify_lru_size(cache_t *cache) { } bool SLRU_get_debug(cache_t *cache, const request_t *req) { + // SLRU_params_t *params = (SLRU_params_t *)(cache->eviction_params); cache->n_req += 1; - SLRU_params_t *params = (SLRU_params_t *)(cache->eviction_params); DEBUG_PRINT_CACHE_STATE(cache, params, req); bool cache_hit = cache->find(cache, req, true) != NULL; diff --git a/libCacheSim/cache/eviction/SLRUv0.c b/libCacheSim/cache/eviction/SLRUv0.c index 7b1be481..0530ba5b 100644 --- a/libCacheSim/cache/eviction/SLRUv0.c +++ b/libCacheSim/cache/eviction/SLRUv0.c @@ -259,6 +259,7 @@ static cache_obj_t *SLRUv0_to_evict(cache_t *cache, const request_t *req) { return lru->to_evict(lru, req); } } + return NULL; } /** @@ -432,7 +433,7 @@ static void SLRUv0_print_cache(cache_t *cache) { cache_obj_t *obj = ((LRU_params_t *)params->LRUs[i]->eviction_params)->q_head; while (obj) { - printf("%ld(%u)->", obj->obj_id, obj->obj_size); + printf("%ld(%u)->", (long) obj->obj_id, obj->obj_size); obj = obj->queue.next; } printf(" | "); diff --git a/libCacheSim/cache/eviction/SR_LRU.c b/libCacheSim/cache/eviction/SR_LRU.c index 47fc6e4c..254885e2 100644 --- a/libCacheSim/cache/eviction/SR_LRU.c +++ b/libCacheSim/cache/eviction/SR_LRU.c @@ -114,7 +114,7 @@ static void SR_LRU_free(cache_t *cache) { * @return true if cache hit, false if cache miss */ static bool SR_LRU_get(cache_t *cache, const request_t *req) { - SR_LRU_params_t *params = (SR_LRU_params_t *)(cache->eviction_params); + // SR_LRU_params_t *params = (SR_LRU_params_t *)(cache->eviction_params); bool cache_hit = SR_LRU_find(cache, req, true) != NULL; if (!cache_hit) { diff --git a/libCacheSim/cache/eviction/Sieve.c b/libCacheSim/cache/eviction/Sieve.c index 6764904c..064e6edf 100644 --- a/libCacheSim/cache/eviction/Sieve.c +++ b/libCacheSim/cache/eviction/Sieve.c @@ -12,6 +12,8 @@ typedef struct { cache_obj_t *q_tail; cache_obj_t *pointer; + pthread_mutex_t mutex_; + } Sieve_params_t; // *********************************************************************** @@ -67,6 +69,7 @@ cache_t *Sieve_init(const common_cache_params_t ccache_params, params->pointer = NULL; params->q_head = NULL; params->q_tail = NULL; + pthread_mutex_init(¶ms->mutex_, NULL); return cache; } @@ -77,7 +80,9 @@ cache_t *Sieve_init(const common_cache_params_t ccache_params, * @param cache */ static void Sieve_free(cache_t *cache) { - free(cache->eviction_params); + Sieve_params_t *params = cache->eviction_params; + free_list(¶ms->q_head, ¶ms->q_tail); + free(params); cache_struct_free(cache); } @@ -102,10 +107,7 @@ static void Sieve_free(cache_t *cache) { */ static bool Sieve_get(cache_t *cache, const request_t *req) { - Sieve_params_t *params = (Sieve_params_t *)cache->eviction_params; - bool ck_hit = cache_get_base(cache, req); - return ck_hit; } @@ -128,7 +130,7 @@ static bool Sieve_get(cache_t *cache, const request_t *req) { static cache_obj_t *Sieve_find(cache_t *cache, const request_t *req, const bool update_cache) { cache_obj_t *cache_obj = cache_find_base(cache, req, update_cache); - if (cache_obj != NULL && update_cache) { + if (cache_obj != NULL && update_cache && cache_obj_in_cache(cache_obj)) { cache_obj->sieve.freq = 1; } @@ -149,65 +151,33 @@ static cache_obj_t *Sieve_find(cache_t *cache, const request_t *req, static cache_obj_t *Sieve_insert(cache_t *cache, const request_t *req) { Sieve_params_t *params = cache->eviction_params; cache_obj_t *obj = cache_insert_base(cache, req); + pthread_mutex_lock(¶ms->mutex_); prepend_obj_to_head(¶ms->q_head, ¶ms->q_tail, obj); obj->sieve.freq = 0; + cache_obj_set_in_cache(obj, true); + pthread_mutex_unlock(¶ms->mutex_); return obj; } -/** - * @brief find the object to be evicted - * this function does not actually evict the object or update metadata - * not all eviction algorithms support this function - * because the eviction logic cannot be decoupled from finding eviction - * candidate, so use assert(false) if you cannot support this function - * - * @param cache the cache - * @return the object to be evicted - */ -static cache_obj_t *Sieve_to_evict_with_freq(cache_t *cache, - const request_t *req, - int to_evict_freq) { +static cache_obj_t *Sieve_to_evict(cache_t *cache, const request_t *req) { Sieve_params_t *params = cache->eviction_params; - cache_obj_t *pointer = params->pointer; - cache_obj_t *old_pointer = pointer; + pthread_mutex_lock(¶ms->mutex_); /* if we have run one full around or first eviction */ - if (pointer == NULL) pointer = params->q_tail; - - /* find the first untouched */ - while (pointer != NULL && pointer->sieve.freq > to_evict_freq) { - pointer = pointer->queue.prev; - } - - /* if we have finished one around, start from the tail */ - if (pointer == NULL) { - pointer = params->q_tail; - while (pointer != NULL && pointer->sieve.freq > to_evict_freq) { - pointer = pointer->queue.prev; - } - } - - if (pointer == NULL) return NULL; - - return pointer; -} - -static cache_obj_t *Sieve_to_evict(cache_t *cache, const request_t *req) { - // because we do not change the frequency of the object, - // if all objects have frequency 1, we may return NULL - int to_evict_freq = 0; - - cache_obj_t *obj_to_evict = - Sieve_to_evict_with_freq(cache, req, to_evict_freq); - - while (obj_to_evict == NULL) { - to_evict_freq += 1; + cache_obj_t *obj = params->pointer == NULL ? params->q_tail : params->pointer; - obj_to_evict = Sieve_to_evict_with_freq(cache, req, to_evict_freq); + while (obj->sieve.freq > 0) { + obj->sieve.freq -= 1; + obj = obj->queue.prev == NULL ? params->q_tail : obj->queue.prev; } + cache_obj_set_in_cache(obj, false); + params->pointer = obj->queue.prev; + remove_obj_from_list(¶ms->q_head, ¶ms->q_tail, obj); + pthread_mutex_unlock(¶ms->mutex_); - return obj_to_evict; + cache_evict_base(cache, obj, true); + return obj; } /** @@ -220,19 +190,7 @@ static cache_obj_t *Sieve_to_evict(cache_t *cache, const request_t *req) { * @param evicted_obj if not NULL, return the evicted object to caller */ static void Sieve_evict(cache_t *cache, const request_t *req) { - Sieve_params_t *params = cache->eviction_params; - - /* if we have run one full around or first eviction */ - cache_obj_t *obj = params->pointer == NULL ? params->q_tail : params->pointer; - - while (obj->sieve.freq > 0) { - obj->sieve.freq -= 1; - obj = obj->queue.prev == NULL ? params->q_tail : obj->queue.prev; - } - - params->pointer = obj->queue.prev; - remove_obj_from_list(¶ms->q_head, ¶ms->q_tail, obj); - cache_evict_base(cache, obj, true); + Sieve_to_evict(cache, req); } static void Sieve_remove_obj(cache_t *cache, cache_obj_t *obj_to_remove) { @@ -241,7 +199,7 @@ static void Sieve_remove_obj(cache_t *cache, cache_obj_t *obj_to_remove) { if (obj_to_remove == params->pointer) { params->pointer = obj_to_remove->queue.prev; } - remove_obj_from_list(¶ms->q_head, ¶ms->q_tail, obj_to_remove); + // remove_obj_from_list(¶ms->q_head, ¶ms->q_tail, obj_to_remove); cache_remove_obj_base(cache, obj_to_remove, true); } @@ -259,14 +217,13 @@ static void Sieve_remove_obj(cache_t *cache, cache_obj_t *obj_to_remove) { * cache */ static bool Sieve_remove(cache_t *cache, const obj_id_t obj_id) { - cache_obj_t *obj = hashtable_find_obj_id(cache->hashtable, obj_id); - if (obj == NULL) { - return false; + cache_obj_t *obj = hashtable_delete_obj_id(cache->hashtable, obj_id); + if(obj != NULL){ + fetch_sub(&cache->n_obj, 1); + return true; } - - Sieve_remove_obj(cache, obj); - - return true; + else + return false; } static void Sieve_verify(cache_t *cache) { diff --git a/libCacheSim/cache/eviction/TwoQ.c b/libCacheSim/cache/eviction/TwoQ.c index 4032eb34..f9933880 100644 --- a/libCacheSim/cache/eviction/TwoQ.c +++ b/libCacheSim/cache/eviction/TwoQ.c @@ -19,8 +19,6 @@ extern "C" { #endif -// #define USE_MYCLOCK - typedef struct { cache_t *Ain; cache_t *Aout; @@ -110,11 +108,6 @@ cache_t *TwoQ_init(const common_cache_params_t ccache_params, ccache_params_local.cache_size = params->Am_cache_size; params->Am = LRU_init(ccache_params_local, NULL); -#ifdef USE_MYCLOCK - params->Am->cache_free(params->Am); - params->Am = MyClock_init(ccache_params_local, NULL); - snprintf(cache->cache_name, CACHE_NAME_ARRAY_LEN, "TwoQ-myclock"); -#endif return cache; } @@ -342,7 +335,6 @@ static void TwoQ_parse_params(cache_t *cache, char *params_str = strdup(cache_specific_params); char *old_params_str = params_str; - char *end; while (params_str != NULL && params_str[0] != '\0') { /* different parameters are separated by comma, diff --git a/libCacheSim/cache/eviction/WTinyLFU.c b/libCacheSim/cache/eviction/WTinyLFU.c index 68f83104..42bf9255 100644 --- a/libCacheSim/cache/eviction/WTinyLFU.c +++ b/libCacheSim/cache/eviction/WTinyLFU.c @@ -138,6 +138,8 @@ cache_t *WTinyLFU_init(const common_cache_params_t ccache_params, params->main_cache = Hyperbolic_init(ccache_params_local, NULL); } else if (strcasecmp(params->main_cache_type, "LHD") == 0) { params->main_cache = LHD_init(ccache_params_local, NULL); + } else if (strcasecmp(params->main_cache_type, "SIEVE") == 0) { + params->main_cache = Sieve_init(ccache_params_local, NULL); } else { ERROR("WTinyLFU does not support %s \n", params->main_cache_type); } @@ -251,6 +253,7 @@ cache_obj_t *WTinyLFU_insert(cache_t *cache, const request_t *req) { static cache_obj_t *WTinyLFU_to_evict(cache_t *cache, const request_t *req) { // Warning: don't use this function DEBUG_ASSERT(false); + return NULL; } static void WTinyLFU_evict(cache_t *cache, const request_t *req) { @@ -347,8 +350,6 @@ static void WTinyLFU_parse_params(cache_t *cache, // params->max_request_num = 32 * cache->cache_size; // 32 * cache_size char *params_str = strdup(cache_specific_params); - char *old_params_str = params_str; - char *end; while (params_str != NULL && params_str[0] != '\0') { /* different parameters are separated by comma, * key and value are separated by = */ diff --git a/libCacheSim/cache/eviction/belady/FIFO_Belady.c b/libCacheSim/cache/eviction/belady/FIFO_Belady.c index 3b679443..9841116b 100644 --- a/libCacheSim/cache/eviction/belady/FIFO_Belady.c +++ b/libCacheSim/cache/eviction/belady/FIFO_Belady.c @@ -159,7 +159,7 @@ static bool should_insert(cache_t *cache, int64_t next_access_vtime) { */ static cache_obj_t *FIFO_Belady_find(cache_t *cache, const request_t *req, const bool update_cache) { - FIFO_Belady_params_t *params = (FIFO_Belady_params_t *)cache->eviction_params; + cache_obj_t *cache_obj = cache_find_base(cache, req, update_cache); return cache_obj; @@ -280,7 +280,7 @@ static void FIFO_Belady_print_cache(const cache_t *cache) { return; } while (cur != NULL) { - printf("%lu->", cur->obj_id); + printf("%ld->", (long)cur->obj_id); cur = cur->queue.next; } printf("END\n"); diff --git a/libCacheSim/cache/eviction/belady/LRU_Belady.c b/libCacheSim/cache/eviction/belady/LRU_Belady.c index 495fa80f..5aa449fe 100644 --- a/libCacheSim/cache/eviction/belady/LRU_Belady.c +++ b/libCacheSim/cache/eviction/belady/LRU_Belady.c @@ -295,7 +295,7 @@ static void LRU_Belady_print_cache(const cache_t *cache) { return; } while (cur != NULL) { - printf("%lu->", cur->obj_id); + printf("%ld->", (long)cur->obj_id); cur = cur->queue.next; } printf("END\n"); diff --git a/libCacheSim/cache/eviction/belady/Sieve_Belady.c b/libCacheSim/cache/eviction/belady/Sieve_Belady.c index 05301114..fd66bcc5 100644 --- a/libCacheSim/cache/eviction/belady/Sieve_Belady.c +++ b/libCacheSim/cache/eviction/belady/Sieve_Belady.c @@ -173,7 +173,7 @@ static cache_obj_t *Sieve_Belady_find(cache_t *cache, const request_t *req, const bool update_cache) { cache_obj_t *cache_obj = cache_find_base(cache, req, update_cache); if (cache_obj != NULL && update_cache) { - cache_obj->myclock.freq = 1; + cache_obj->sieve.freq = 1; } return cache_obj; @@ -199,14 +199,14 @@ static cache_obj_t *Sieve_Belady_insert(cache_t *cache, const request_t *req) { if (should_insert(cache, req->next_access_vtime)) { obj = cache_insert_base(cache, req); prepend_obj_to_head(¶ms->q_head, ¶ms->q_tail, obj); - obj->myclock.freq = 0; - obj->myclock.new_obj = true; + obj->sieve.freq = 0; + // obj->sieve.new_obj = true; } #else obj = cache_insert_base(cache, req); prepend_obj_to_head(¶ms->q_head, ¶ms->q_tail, obj); - obj->myclock.freq = 0; - obj->myclock.new_obj = true; + obj->sieve.freq = 0; + // obj->sieve.new_obj = true; #endif return obj; @@ -232,8 +232,8 @@ static void Sieve_Belady_evict(cache_t *cache, const request_t *req) { /* find the first untouched */ while (obj != NULL && should_insert(cache, obj->misc.next_access_vtime)) { - // while (obj != NULL && obj->myclock.freq > 0) { - obj->myclock.freq -= 1; + // while (obj != NULL && obj->sieve.freq > 0) { + obj->sieve.freq -= 1; obj = obj->queue.prev; } @@ -241,8 +241,8 @@ static void Sieve_Belady_evict(cache_t *cache, const request_t *req) { if (obj == NULL) { obj = params->q_tail; while (obj != NULL && should_insert(cache, obj->misc.next_access_vtime)) { - // while (obj != NULL && obj->myclock.freq > 0) { - obj->myclock.freq -= 1; + // while (obj != NULL && obj->sieve.freq > 0) { + obj->sieve.freq -= 1; obj = obj->queue.prev; } } diff --git a/libCacheSim/cache/eviction/cpp/abstractRank.hpp b/libCacheSim/cache/eviction/cpp/abstractRank.hpp index 60fb63c1..a309c562 100644 --- a/libCacheSim/cache/eviction/cpp/abstractRank.hpp +++ b/libCacheSim/cache/eviction/cpp/abstractRank.hpp @@ -12,9 +12,9 @@ #include #include +#include "../../../dataStructure/hashtable/hashtable.h" #include "../../../include/libCacheSim/cache.h" #include "../../../include/libCacheSim/cacheObj.h" -#include "../../../dataStructure/hashtable/hashtable.h" using namespace std; @@ -34,8 +34,8 @@ struct pq_node_type { : obj(obj), priority(priority), last_request_vtime(last_request_vtime){}; void print() const { - printf("obj %lu, priority %f, last_request_vtime %ld\n", obj->obj_id, - priority, last_request_vtime); + printf("obj %lu, priority %f, last_request_vtime %ld\n", + (unsigned long)obj->obj_id, priority, (long)last_request_vtime); } bool operator<(const pq_node_type &rhs) const { diff --git a/libCacheSim/cache/eviction/fifo/LP_ARC.c b/libCacheSim/cache/eviction/fifo/LP_ARC.c index 7bdc8b5d..b1f68127 100644 --- a/libCacheSim/cache/eviction/fifo/LP_ARC.c +++ b/libCacheSim/cache/eviction/fifo/LP_ARC.c @@ -159,7 +159,7 @@ static void LP_ARC_free(cache_t *cache) { * @return true if cache hit, false if cache miss */ static bool LP_ARC_get(cache_t *cache, const request_t *req) { - LP_ARC_params_t *params = (LP_ARC_params_t *)(cache->eviction_params); + // LP_ARC_params_t *params = (LP_ARC_params_t *)(cache->eviction_params); return LP_ARC_get_debug(cache, req); // return cache_get_base(cache, req); @@ -485,13 +485,13 @@ static void LP_ARC_parse_params(cache_t *cache, char *params_str = strdup(cache_specific_params); char *old_params_str = params_str; - char *end; + // char *end; while (params_str != NULL && params_str[0] != '\0') { /* different parameters are separated by comma, * key and value are separated by = */ char *key = strsep((char **)¶ms_str, "="); - char *value = strsep((char **)¶ms_str, ","); + // char *value = strsep((char **)¶ms_str, ","); // skip the white space while (params_str != NULL && *params_str == ' ') { @@ -528,7 +528,7 @@ static void print_cache(cache_t *cache) { } static bool LP_ARC_get_debug(cache_t *cache, const request_t *req) { - LP_ARC_params_t *params = (LP_ARC_params_t *)(cache->eviction_params); + // LP_ARC_params_t *params = (LP_ARC_params_t *)(cache->eviction_params); cache->n_req += 1; diff --git a/libCacheSim/cache/eviction/fifo/LP_SFIFO.c b/libCacheSim/cache/eviction/fifo/LP_SFIFO.c index 6f903bec..72a77a18 100644 --- a/libCacheSim/cache/eviction/fifo/LP_SFIFO.c +++ b/libCacheSim/cache/eviction/fifo/LP_SFIFO.c @@ -137,7 +137,7 @@ static void LP_SFIFO_free(cache_t *cache) { * @return true if cache hit, false if cache miss */ static bool LP_SFIFO_get(cache_t *cache, const request_t *req) { - LP_SFIFO_params_t *params = (LP_SFIFO_params_t *)(cache->eviction_params); + // LP_SFIFO_params_t *params = (LP_SFIFO_params_t *)(cache->eviction_params); // printf("%ld %ld %ld %ld %ld\n", cache->n_req, // params->fifos[0]->get_n_obj(params->fifos[0]), // params->fifos[1]->get_n_obj(params->fifos[1]), @@ -243,6 +243,7 @@ static cache_obj_t *LP_SFIFO_to_evict(cache_t *cache, const request_t *req) { return params->fifos[i]->to_evict(params->fifos[i], req); } } + return NULL; } /** diff --git a/libCacheSim/cache/eviction/fifo/LP_TwoQ.c b/libCacheSim/cache/eviction/fifo/LP_TwoQ.c index 66310389..612348e1 100644 --- a/libCacheSim/cache/eviction/fifo/LP_TwoQ.c +++ b/libCacheSim/cache/eviction/fifo/LP_TwoQ.c @@ -348,7 +348,6 @@ static void LP_TwoQ_parse_params(cache_t *cache, char *params_str = strdup(cache_specific_params); char *old_params_str = params_str; - char *end; while (params_str != NULL && params_str[0] != '\0') { /* different parameters are separated by comma, diff --git a/libCacheSim/cache/eviction/fifo/SFIFO.c b/libCacheSim/cache/eviction/fifo/SFIFO.c index 36c13d20..6ec113da 100644 --- a/libCacheSim/cache/eviction/fifo/SFIFO.c +++ b/libCacheSim/cache/eviction/fifo/SFIFO.c @@ -291,6 +291,7 @@ static cache_obj_t *SFIFO_to_evict(cache_t *cache, const request_t *req) { return params->fifo_tails[i]; } } + return NULL; } /** @@ -512,7 +513,7 @@ static void _SFIFO_verify_fifo_size(cache_t *cache) { bool SFIFO_get_debug(cache_t *cache, const request_t *req) { cache->n_req += 1; - SFIFO_params_t *params = (SFIFO_params_t *)(cache->eviction_params); + // SFIFO_params_t *params = (SFIFO_params_t *)(cache->eviction_params); cache->last_request_metadata = "none"; DEBUG_PRINT_CACHE_STATE(cache, params, req); diff --git a/libCacheSim/cache/eviction/fifo/SFIFOv0.c b/libCacheSim/cache/eviction/fifo/SFIFOv0.c index f21a60b3..c08102df 100644 --- a/libCacheSim/cache/eviction/fifo/SFIFOv0.c +++ b/libCacheSim/cache/eviction/fifo/SFIFOv0.c @@ -11,7 +11,7 @@ // // // we notice that adding belady is not helpful -// +// // // libCacheSim // @@ -72,7 +72,8 @@ static inline int64_t SFIFOv0_get_n_obj(const cache_t *cache); */ cache_t *SFIFOv0_init(const common_cache_params_t ccache_params, const char *cache_specific_params) { - cache_t *cache = cache_struct_init("SFIFOv0", ccache_params, cache_specific_params); + cache_t *cache = + cache_struct_init("SFIFOv0", ccache_params, cache_specific_params); cache->cache_init = SFIFOv0_init; cache->cache_free = SFIFOv0_free; cache->get = SFIFOv0_get; @@ -272,6 +273,7 @@ static cache_obj_t *SFIFOv0_to_evict(cache_t *cache, const request_t *req) { } } #endif + return NULL; } /** @@ -303,7 +305,6 @@ static void SFIFOv0_evict(cache_t *cache, const request_t *req) { fifo->evict(fifo, req); #else - int nth_seg_to_evict = 0; for (int i = 0; i < params->n_queues; i++) { cache_t *fifo = params->FIFOs[i]; if (fifo->get_occupied_byte(fifo) > 0) { @@ -459,7 +460,7 @@ static void SFIFOv0_print_cache(cache_t *cache) { cache_obj_t *obj = ((FIFO_params_t *)params->FIFOs[i]->eviction_params)->q_head; while (obj) { - printf("%ld(%u)->", obj->obj_id, obj->obj_size); + printf("%ld(%u)->", (long)obj->obj_id, (unsigned int)obj->obj_size); obj = obj->queue.next; } printf(" | "); diff --git a/libCacheSim/cache/eviction/nop.c b/libCacheSim/cache/eviction/nop.c index bbf5bba7..5cf78d60 100644 --- a/libCacheSim/cache/eviction/nop.c +++ b/libCacheSim/cache/eviction/nop.c @@ -113,9 +113,9 @@ static bool nop_get(cache_t *cache, const request_t *req) { */ static cache_obj_t *nop_find(cache_t *cache, const request_t *req, const bool update_cache) { -// cache_obj_t *cache_obj = cache_find_base(cache, req, update_cache); + // cache_obj_t *cache_obj = cache_find_base(cache, req, update_cache); -// return cache_obj; + // return cache_obj; return NULL; } @@ -130,9 +130,9 @@ static cache_obj_t *nop_find(cache_t *cache, const request_t *req, * @return the inserted object */ static cache_obj_t *nop_insert(cache_t *cache, const request_t *req) { -// cache_obj_t *obj = cache_insert_base(cache, req); + // cache_obj_t *obj = cache_insert_base(cache, req); -// return obj; + // return obj; return NULL; } @@ -147,7 +147,8 @@ static cache_obj_t *nop_insert(cache_t *cache, const request_t *req) { * @return the object to be evicted */ static cache_obj_t *nop_to_evict(cache_t *cache, const request_t *req) { - assert(false); + assert(false); + return NULL; } /** @@ -159,7 +160,7 @@ static cache_obj_t *nop_to_evict(cache_t *cache, const request_t *req) { * @param req not used */ static void nop_evict(cache_t *cache, const request_t *req) { -// cache_evict_base(cache, obj_to_evict, true); + // cache_evict_base(cache, obj_to_evict, true); } /** diff --git a/libCacheSim/cache/eviction/other/S3LRU.c b/libCacheSim/cache/eviction/other/S3LRU.c index 5b9e495d..5cf499fe 100644 --- a/libCacheSim/cache/eviction/other/S3LRU.c +++ b/libCacheSim/cache/eviction/other/S3LRU.c @@ -288,8 +288,8 @@ static cache_obj_t *S3LRU_insert(cache_t *cache, const request_t *req) { } else { /* insert into the LRU */ if (req->obj_size > params->LRU->cache_size) { - WARN("object size %ld larger than small cache size %ld\n", req->obj_size, - params->LRU->cache_size); + WARN("object size %ld larger than small cache size %ld\n", + (long)req->obj_size, (long)params->LRU->cache_size); return NULL; } obj = params->LRU->insert(params->LRU, req); @@ -303,7 +303,7 @@ static cache_obj_t *S3LRU_insert(cache_t *cache, const request_t *req) { obj->create_time = cache->n_req; #endif - obj->S3FIFO.freq == 0; + obj->S3FIFO.freq = 0; return obj; } @@ -362,8 +362,8 @@ static void S3LRU_evict_LRU(cache_t *cache, const request_t *req) { static void S3LRU_evict_main(cache_t *cache, const request_t *req) { S3LRU_params_t *params = (S3LRU_params_t *)cache->eviction_params; - cache_t *LRU = params->LRU; - cache_t *ghost = params->LRU_ghost; + // cache_t *LRU = params->LRU; + // cache_t *ghost = params->LRU_ghost; cache_t *main = params->main_cache; // evict from main cache @@ -373,7 +373,7 @@ static void S3LRU_evict_main(cache_t *cache, const request_t *req) { DEBUG_ASSERT(obj_to_evict != NULL); bool removed = main->remove(main, obj_to_evict->obj_id); if (!removed) { - ERROR("cannot remove obj %ld\n", obj_to_evict->obj_id); + ERROR("cannot remove obj %ld\n", (long)obj_to_evict->obj_id); } has_evicted = true; @@ -393,7 +393,7 @@ static void S3LRU_evict(cache_t *cache, const request_t *req) { S3LRU_params_t *params = (S3LRU_params_t *)cache->eviction_params; cache_t *LRU = params->LRU; - cache_t *ghost = params->LRU_ghost; + // cache_t *ghost = params->LRU_ghost; cache_t *main = params->main_cache; if (main->get_occupied_byte(main) > main->cache_size || @@ -465,7 +465,7 @@ static void S3LRU_parse_params(cache_t *cache, char *params_str = strdup(cache_specific_params); char *old_params_str = params_str; - char *end; + // char *end; while (params_str != NULL && params_str[0] != '\0') { /* different parameters are separated by comma, diff --git a/libCacheSim/cache/eviction/other/flashProb.c b/libCacheSim/cache/eviction/other/flashProb.c index d625c439..99e61912 100644 --- a/libCacheSim/cache/eviction/other/flashProb.c +++ b/libCacheSim/cache/eviction/other/flashProb.c @@ -179,7 +179,6 @@ static void flashProb_free(cache_t *cache) { * @return true if cache hit, false if cache miss */ static bool flashProb_get(cache_t *cache, const request_t *req) { - flashProb_params_t *params = (flashProb_params_t *)cache->eviction_params; bool cache_hit = cache_get_base(cache, req); @@ -368,7 +367,6 @@ static void flashProb_parse_params(cache_t *cache, char *params_str = strdup(cache_specific_params); char *old_params_str = params_str; - char *end; while (params_str != NULL && params_str[0] != '\0') { /* different parameters are separated by comma, diff --git a/libCacheSim/cache/eviction/priv/MClock.c b/libCacheSim/cache/eviction/priv/MClock.c index e8dea550..f5ae1e38 100644 --- a/libCacheSim/cache/eviction/priv/MClock.c +++ b/libCacheSim/cache/eviction/priv/MClock.c @@ -224,7 +224,7 @@ static void print_curr_hand_pos(cache_t *cache) { int64_t obj_idx = 0; while (obj != NULL) { if (obj == params->hands[hand_idx]) { - printf("hand %d: %ld %p %p %p\n", hand_idx, obj_idx, obj, obj->queue.prev, + printf("hand %d: %ld %p %p %p\n", hand_idx, (long) obj_idx, obj, obj->queue.prev, obj->queue.next); hand_idx++; } @@ -392,13 +392,13 @@ static bool MClock_remove(cache_t *cache, const obj_id_t obj_id) { } static int64_t MClock_get_occupied_byte(const cache_t *cache) { - MClock_params_t *params = (MClock_params_t *)cache->eviction_params; + // MClock_params_t *params = (MClock_params_t *)cache->eviction_params; int64_t occupied_byte = cache->occupied_byte; return occupied_byte; } static int64_t MClock_get_n_obj(const cache_t *cache) { - MClock_params_t *params = (MClock_params_t *)cache->eviction_params; + // MClock_params_t *params = (MClock_params_t *)cache->eviction_params; int64_t n_obj = cache->n_obj; return n_obj; } diff --git a/libCacheSim/cache/eviction/priv/MyClock.c b/libCacheSim/cache/eviction/priv/MyClock.c deleted file mode 100644 index 1ae65936..00000000 --- a/libCacheSim/cache/eviction/priv/MyClock.c +++ /dev/null @@ -1,327 +0,0 @@ - - -#include "../../../dataStructure/hashtable/hashtable.h" -#include "../../../include/libCacheSim/cache.h" - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct { - cache_obj_t *q_head; - cache_obj_t *q_tail; - - cache_obj_t *pointer; - - int64_t n_miss; - int64_t n_new_obj; // used to track the number of objects since last pointer - // went back to the tail (the oldest) -} MyClock_params_t; - -// *********************************************************************** -// **** **** -// **** function declarations **** -// **** **** -// *********************************************************************** - -static void MyClock_free(cache_t *cache); -static bool MyClock_get(cache_t *cache, const request_t *req); -static cache_obj_t *MyClock_find(cache_t *cache, const request_t *req, - const bool update_cache); -static cache_obj_t *MyClock_insert(cache_t *cache, const request_t *req); -static cache_obj_t *MyClock_to_evict(cache_t *cache, const request_t *req); -static void MyClock_evict(cache_t *cache, const request_t *req); -static bool MyClock_remove(cache_t *cache, const obj_id_t obj_id); - -static void MyClock_verify(cache_t *cache); - -// *********************************************************************** -// **** **** -// **** end user facing functions **** -// **** **** -// **** init, free, get **** -// *********************************************************************** - -/** - * @brief initialize cache - * - * @param ccache_params some common cache parameters - * @param cache_specific_params cache specific parameters, see parse_params - * function or use -e "print" with the cachesim binary - */ -cache_t *MyClock_init(const common_cache_params_t ccache_params, - const char *cache_specific_params) { - cache_t *cache = - cache_struct_init("MyClock", ccache_params, cache_specific_params); - cache->cache_init = MyClock_init; - cache->cache_free = MyClock_free; - cache->get = MyClock_get; - cache->find = MyClock_find; - cache->insert = MyClock_insert; - cache->evict = MyClock_evict; - cache->remove = MyClock_remove; - cache->to_evict = MyClock_to_evict; - - if (ccache_params.consider_obj_metadata) { - cache->obj_md_size = 1; - } else { - cache->obj_md_size = 0; - } - - cache->eviction_params = my_malloc(MyClock_params_t); - memset(cache->eviction_params, 0, sizeof(MyClock_params_t)); - MyClock_params_t *params = (MyClock_params_t *)cache->eviction_params; - params->pointer = NULL; - params->q_head = NULL; - params->q_tail = NULL; - params->n_miss = 0; - - return cache; -} - -/** - * free resources used by this cache - * - * @param cache - */ -static void MyClock_free(cache_t *cache) { - free(cache->eviction_params); - cache_struct_free(cache); -} - -/** - * @brief this function is the user facing API - * it performs the following logic - * - * ``` - * if obj in cache: - * update_metadata - * return true - * else: - * if cache does not have enough space: - * evict until it has space to insert - * insert the object - * return false - * ``` - * - * @param cache - * @param req - * @return true if cache hit, false if cache miss - */ - -static bool MyClock_get(cache_t *cache, const request_t *req) { - MyClock_params_t *params = (MyClock_params_t *)cache->eviction_params; - - bool ck_hit = cache_get_base(cache, req); - if (!ck_hit) { - params->n_miss++; - } - - // if (cache->n_req % 1000 == 0) { - // printf("sieve %ld %ld %lf\n", cache->cache_size, cache->n_req, - // (double)params->n_new_obj / (double)cache->cache_size); - // } - - return ck_hit; -} - -// *********************************************************************** -// **** **** -// **** developer facing APIs (used by cache developer) **** -// **** **** -// *********************************************************************** - -/** - * @brief find an object in the cache - * - * @param cache - * @param req - * @param update_cache whether to update the cache, - * if true, the object is promoted - * and if the object is expired, it is removed from the cache - * @return the object or NULL if not found - */ -static cache_obj_t *MyClock_find(cache_t *cache, const request_t *req, - const bool update_cache) { - cache_obj_t *cache_obj = cache_find_base(cache, req, update_cache); - if (cache_obj != NULL && update_cache) { - cache_obj->myclock.freq = 1; - // cache_obj->myclock.freq += 1; - } - - return cache_obj; -} - -/** - * @brief insert an object into the cache, - * update the hash table and cache metadata - * this function assumes the cache has enough space - * eviction should be - * performed before calling this function - * - * @param cache - * @param req - * @return the inserted object - */ -static cache_obj_t *MyClock_insert(cache_t *cache, const request_t *req) { - MyClock_params_t *params = cache->eviction_params; - cache_obj_t *obj = cache_insert_base(cache, req); - prepend_obj_to_head(¶ms->q_head, ¶ms->q_tail, obj); - obj->myclock.freq = 0; - obj->myclock.new_obj = true; - params->n_new_obj++; - - return obj; -} - -/** - * @brief find the object to be evicted - * this function does not actually evict the object or update metadata - * not all eviction algorithms support this function - * because the eviction logic cannot be decoupled from finding eviction - * candidate, so use assert(false) if you cannot support this function - * - * @param cache the cache - * @return the object to be evicted - */ -static cache_obj_t *MyClock_to_evict_with_freq(cache_t *cache, - const request_t *req, - int to_evict_freq) { - MyClock_params_t *params = cache->eviction_params; - cache_obj_t *pointer = params->pointer; - cache_obj_t *old_pointer = pointer; - - /* if we have run one full around or first eviction */ - if (pointer == NULL) pointer = params->q_tail; - - /* find the first untouched */ - while (pointer != NULL && pointer->myclock.freq > to_evict_freq) { - pointer = pointer->queue.prev; - } - - /* if we have finished one around, start from the tail */ - if (pointer == NULL) { - pointer = params->q_tail; - while (pointer != NULL && pointer->myclock.freq > to_evict_freq) { - pointer = pointer->queue.prev; - } - } - - if (pointer == NULL) return NULL; - - return pointer; -} - -static cache_obj_t *MyClock_to_evict(cache_t *cache, const request_t *req) { - // because we do not change the frequency of the object, - // if all objects have frequency 1, we may return NULL - int to_evict_freq = 0; - - cache_obj_t *obj_to_evict = - MyClock_to_evict_with_freq(cache, req, to_evict_freq); - - while (obj_to_evict == NULL) { - to_evict_freq += 1; - - obj_to_evict = MyClock_to_evict_with_freq(cache, req, to_evict_freq); - } - - return obj_to_evict; -} - -/** - * @brief evict an object from the cache - * it needs to call cache_evict_base before returning - * which updates some metadata such as n_obj, occupied size, and hash table - * - * @param cache - * @param req not used - * @param evicted_obj if not NULL, return the evicted object to caller - */ -static void MyClock_evict(cache_t *cache, const request_t *req) { - MyClock_params_t *params = cache->eviction_params; - cache_obj_t *obj = params->pointer; - - /* if we have run one full around or first eviction */ - if (obj == NULL) { - obj = params->q_tail; - } - - /* find the first untouched */ - while ((obj != NULL && obj->myclock.freq > 0)) { - obj->myclock.freq -= 1; - if (obj->myclock.new_obj) params->n_new_obj -= 1; - - obj->myclock.new_obj = false; - obj = obj->queue.prev; - } - - /* if we have finished one around, start from the tail */ - if (obj == NULL) { - obj = params->q_tail; - while (obj != NULL && obj->myclock.freq > 0) { - obj->myclock.freq -= 1; - obj->myclock.new_obj = false; - obj = obj->queue.prev; - } - } - - if (obj->myclock.new_obj) params->n_new_obj -= 1; - params->pointer = obj->queue.prev; - remove_obj_from_list(¶ms->q_head, ¶ms->q_tail, obj); - cache_evict_base(cache, obj, true); -} - -static void MyClock_remove_obj(cache_t *cache, cache_obj_t *obj_to_remove) { - DEBUG_ASSERT(obj_to_remove != NULL); - MyClock_params_t *params = cache->eviction_params; - if (obj_to_remove == params->pointer) { - params->pointer = obj_to_remove->queue.prev; - } - remove_obj_from_list(¶ms->q_head, ¶ms->q_tail, obj_to_remove); - cache_remove_obj_base(cache, obj_to_remove, true); -} - -/** - * @brief remove an object from the cache - * this is different from cache_evict because it is used to for user trigger - * remove, and eviction is used by the cache to make space for new objects - * - * it needs to call cache_remove_obj_base before returning - * which updates some metadata such as n_obj, occupied size, and hash table - * - * @param cache - * @param obj_id - * @return true if the object is removed, false if the object is not in the - * cache - */ -static bool MyClock_remove(cache_t *cache, const obj_id_t obj_id) { - cache_obj_t *obj = hashtable_find_obj_id(cache->hashtable, obj_id); - if (obj == NULL) { - return false; - } - - MyClock_remove_obj(cache, obj); - - return true; -} - -static void MyClock_verify(cache_t *cache) { - MyClock_params_t *params = cache->eviction_params; - int64_t n_obj = 0, n_byte = 0; - cache_obj_t *obj = params->q_head; - - while (obj != NULL) { - assert(hashtable_find_obj_id(cache->hashtable, obj->obj_id) != NULL); - n_obj++; - n_byte += obj->obj_size; - obj = obj->queue.next; - } - - assert(n_obj == cache->get_n_obj(cache)); - assert(n_byte == cache->get_occupied_byte(cache)); -} - -#ifdef __cplusplus -} -#endif diff --git a/libCacheSim/cache/eviction/priv/QDLPv0.c b/libCacheSim/cache/eviction/priv/QDLPv0.c index 7b5b102b..0e0f8bf9 100644 --- a/libCacheSim/cache/eviction/priv/QDLPv0.c +++ b/libCacheSim/cache/eviction/priv/QDLPv0.c @@ -2,7 +2,7 @@ * @file QDLP.c * @brief Implementation of QDLPv0 eviction algorithm * QDLPv0: lazy promotion and quick demotion - * it uses a probationary FIFO queue with a myclock cache + * it uses a probationary FIFO queue with a sieve cache * objects are first inserted into the FIFO queue, and moved to the clock cache * when evicting from the FIFO queue if it has been accessed * if the cache is full, evict from the FIFO cache @@ -320,6 +320,7 @@ static cache_obj_t *QDLPv0_to_evict(cache_t *cache, const request_t *req) { // not implemented, we need to evict from the clock cache assert(0); } + return NULL; } /** diff --git a/libCacheSim/cache/eviction/priv/S3FIFOdv2.c b/libCacheSim/cache/eviction/priv/S3FIFOdv2.c index f9a0509a..6598a07e 100644 --- a/libCacheSim/cache/eviction/priv/S3FIFOdv2.c +++ b/libCacheSim/cache/eviction/priv/S3FIFOdv2.c @@ -81,18 +81,18 @@ static const char *DEFAULT_CACHE_PARAMS = // **** **** // *********************************************************************** cache_t *S3FIFOdv2_init(const common_cache_params_t ccache_params, - const char *cache_specific_params); + const char *cache_specific_params); static void S3FIFOdv2_free(cache_t *cache); static bool S3FIFOdv2_get(cache_t *cache, const request_t *req); -static bool S3FIFOdv2_rebalance(cache_t *cache); +static void S3FIFOdv2_rebalance(cache_t *cache); static void S3FIFOdv2_choose_threshold(cache_t *cache); static void S3FIFOdv2_update_main_reinsert_threshold(cache_t *cache); static void S3FIFOdv2_update_main_reinsert_threshold_incorrect(cache_t *cache); static void S3FIFOdv2_update_small_reinsert_threshold(cache_t *cache); static cache_obj_t *S3FIFOdv2_find(cache_t *cache, const request_t *req, - const bool update_cache); + const bool update_cache); static cache_obj_t *S3FIFOdv2_insert(cache_t *cache, const request_t *req); static cache_obj_t *S3FIFOdv2_to_evict(cache_t *cache, const request_t *req); static void S3FIFOdv2_evict(cache_t *cache, const request_t *req); @@ -101,7 +101,7 @@ static inline int64_t S3FIFOdv2_get_occupied_byte(const cache_t *cache); static inline int64_t S3FIFOdv2_get_n_obj(const cache_t *cache); static inline bool S3FIFOdv2_can_insert(cache_t *cache, const request_t *req); static void S3FIFOdv2_parse_params(cache_t *cache, - const char *cache_specific_params); + const char *cache_specific_params); static void S3FIFOdv2_evict_fifo(cache_t *cache, const request_t *req); static void S3FIFOdv2_evict_main(cache_t *cache, const request_t *req); @@ -113,7 +113,7 @@ static void S3FIFOdv2_evict_main(cache_t *cache, const request_t *req); // *********************************************************************** cache_t *S3FIFOdv2_init(const common_cache_params_t ccache_params, - const char *cache_specific_params) { + const char *cache_specific_params) { cache_t *cache = cache_struct_init("S3FIFOdv2", ccache_params, cache_specific_params); cache->cache_init = S3FIFOdv2_init; @@ -294,7 +294,7 @@ static void S3FIFOdv2_update_main_reinsert_threshold_incorrect(cache_t *cache) { n_obj += params->main_freq_cnt[i]; } if (n_obj != n_main_obj) - ERROR("n_obj %ld != n_main_obj %d\n", n_obj, n_main_obj); + ERROR("n_obj %ld != n_main_obj %d\n", (long)n_obj, n_main_obj); // if (params->move_to_main_threshold > 1) { // printf("set threshold to %d, ", params->move_to_main_threshold); @@ -335,7 +335,7 @@ static void S3FIFOdv2_update_main_reinsert_threshold(cache_t *cache) { n_obj += params->main_freq_cnt[i]; } if (n_obj != n_main_obj) - ERROR("n_obj %ld != n_main_obj %d\n", n_obj, n_main_obj); + ERROR("n_obj %ld != n_main_obj %d\n", (long)n_obj, n_main_obj); // if (params->main_reinsert_threshold > 1) { // printf("set threshold to %d, ", params->main_reinsert_threshold); @@ -376,7 +376,7 @@ static void S3FIFOdv2_update_small_reinsert_threshold(cache_t *cache) { n_obj += params->small_freq_cnt[i]; } if (n_obj != n_small_obj) - ERROR("n_obj %ld != n_small_obj %d\n", n_obj, n_small_obj); + ERROR("n_obj %ld != n_small_obj %d\n", (long)n_obj, n_small_obj); // if (params->move_to_main_threshold > 1) { // printf("set threshold to %d, ", params->move_to_main_threshold); @@ -414,8 +414,8 @@ static void S3FIFOdv2_choose_threshold(cache_t *cache) { if (params->move_to_main_threshold != 1) { for (int i = 0; i < MAX_FREQ_THRESHOLD; i++) { - printf("%8ld/%-4d ", params->eviction_freq_cnt[i], - params->threshold_cnt[i]); + printf("%8ld/%-4d ", (long)params->eviction_freq_cnt[i], + (int)params->threshold_cnt[i]); } printf("move_to_main_threshold: %d\n", params->move_to_main_threshold); @@ -470,18 +470,18 @@ static bool S3FIFOdv2_rebalance0(cache_t *cache) { } memset(params->first_access_cnt_over_age_fifo, 0, - sizeof(params->first_access_cnt_over_age_fifo)); + sizeof(*params->first_access_cnt_over_age_fifo)); memset(params->first_access_cnt_over_age_main, 0, - sizeof(params->first_access_cnt_over_age_main)); + sizeof(*params->first_access_cnt_over_age_main)); return true; } -static bool S3FIFOdv2_rebalance(cache_t *cache) { +static void S3FIFOdv2_rebalance(cache_t *cache) { S3FIFOdv2_params_t *params = (S3FIFOdv2_params_t *)cache->eviction_params; - double s = params->n_obj_admit_to_fifo + params->n_obj_admit_to_main + - params->n_obj_move_to_main; + // double s = params->n_obj_admit_to_fifo + params->n_obj_admit_to_main + + // params->n_obj_move_to_main; size_t n_slots = MIN(params->fifo->cache_size, params->main_cache->cache_size) / 100 + 1; @@ -526,7 +526,7 @@ static bool S3FIFOdv2_rebalance(cache_t *cache) { * @return the object or NULL if not found */ static cache_obj_t *S3FIFOdv2_find(cache_t *cache, const request_t *req, - const bool update_cache) { + const bool update_cache) { S3FIFOdv2_params_t *params = (S3FIFOdv2_params_t *)cache->eviction_params; // if update cache is false, we only check the fifo and main caches @@ -635,7 +635,7 @@ static cache_obj_t *S3FIFOdv2_insert(cache_t *cache, const request_t *req) { obj->create_time = cache->n_req; #endif - obj->S3FIFO.freq == 0; + obj->S3FIFO.freq = 0; return obj; } @@ -716,8 +716,8 @@ static void S3FIFOdv2_evict_fifo(cache_t *cache, const request_t *req) { static void S3FIFOdv2_evict_main(cache_t *cache, const request_t *req) { S3FIFOdv2_params_t *params = (S3FIFOdv2_params_t *)cache->eviction_params; - cache_t *fifo = params->fifo; - cache_t *ghost = params->fifo_ghost; + // cache_t *fifo = params->fifo; + // cache_t *ghost = params->fifo_ghost; cache_t *main = params->main_cache; // evict from main cache @@ -755,7 +755,7 @@ static void S3FIFOdv2_evict_main(cache_t *cache, const request_t *req) { // main->evict(main, req); bool removed = main->remove(main, obj_to_evict->obj_id); if (!removed) { - ERROR("cannot remove obj %ld\n", obj_to_evict->obj_id); + ERROR("cannot remove obj %ld\n", (long)obj_to_evict->obj_id); } has_evicted = true; @@ -776,7 +776,7 @@ static void S3FIFOdv2_evict(cache_t *cache, const request_t *req) { S3FIFOdv2_params_t *params = (S3FIFOdv2_params_t *)cache->eviction_params; cache_t *fifo = params->fifo; - cache_t *ghost = params->fifo_ghost; + // cache_t *ghost = params->fifo_ghost; cache_t *main = params->main_cache; if (main->get_occupied_byte(main) > main->cache_size || @@ -841,12 +841,12 @@ static const char *S3FIFOdv2_current_params(S3FIFOdv2_params_t *params) { } static void S3FIFOdv2_parse_params(cache_t *cache, - const char *cache_specific_params) { + const char *cache_specific_params) { S3FIFOdv2_params_t *params = (S3FIFOdv2_params_t *)(cache->eviction_params); char *params_str = strdup(cache_specific_params); char *old_params_str = params_str; - char *end; + // char *end; while (params_str != NULL && params_str[0] != '\0') { /* different parameters are separated by comma, diff --git a/libCacheSim/cache/eviction/priv/myMQv1.c b/libCacheSim/cache/eviction/priv/myMQv1.c index 55655564..0dde2cc7 100644 --- a/libCacheSim/cache/eviction/priv/myMQv1.c +++ b/libCacheSim/cache/eviction/priv/myMQv1.c @@ -2,8 +2,8 @@ // multi queue // each queue has a specified size and a specified ghost size // promotion upon hit -// -// in progress, this is not finished +// +// in progress, this is not finished // // // myMQv1.c @@ -60,7 +60,6 @@ static int64_t myMQv1_get_n_obj(const cache_t *cache); // **** **** // *********************************************************************** - /** * @brief initialize the cache * @@ -70,7 +69,8 @@ static int64_t myMQv1_get_n_obj(const cache_t *cache); */ cache_t *myMQv1_init(const common_cache_params_t ccache_params, const char *cache_specific_params) { - cache_t *cache = cache_struct_init("myMQv1", ccache_params, cache_specific_params); + cache_t *cache = + cache_struct_init("myMQv1", ccache_params, cache_specific_params); cache->cache_init = myMQv1_init; cache->cache_free = myMQv1_free; cache->get = myMQv1_get; @@ -169,7 +169,6 @@ static cache_obj_t *myMQv1_find(cache_t *cache, const request_t *req, cache_obj_t *cache_obj = cache_find_base(cache, req, update_cache); ERROR("todo"); - return cache_obj; } @@ -185,12 +184,11 @@ static cache_obj_t *myMQv1_find(cache_t *cache, const request_t *req, * @return the inserted object */ static cache_obj_t *myMQv1_insert(cache_t *cache, const request_t *req) { - myMQv1_params_t *params = (myMQv1_params_t *)cache->eviction_params; + // myMQv1_params_t *params = (myMQv1_params_t *)cache->eviction_params; cache_obj_t *obj = cache_insert_base(cache, req); ERROR("todo"); - return obj; } @@ -219,11 +217,9 @@ static cache_obj_t *myMQv1_to_evict(cache_t *cache, const request_t *req) { * @param evicted_obj if not NULL, return the evicted object to caller */ static void myMQv1_evict(cache_t *cache, const request_t *req) { - myMQv1_params_t *params = (myMQv1_params_t *)cache->eviction_params; + // myMQv1_params_t *params = (myMQv1_params_t *)cache->eviction_params; ERROR("todo"); - - } /** @@ -278,7 +274,7 @@ static const char *myMQv1_current_params(cache_t *cache, myMQv1_params_t *params) { static __thread char params_str[128]; int n = snprintf(params_str, 128, "n-caches=%ld;cache-size-ratio=%d", - params->n_caches, + (long)params->n_caches, (int)(params->cache_sizes[0] * 100 / cache->cache_size)); for (int i = 1; i < params->n_caches; i++) { @@ -342,7 +338,6 @@ static void myMQv1_parse_params(cache_t *cache, } } else if (strcasecmp(key, "ghost-size-ratio") == 0) { int n_ghost_caches = 0; - int64_t cache_size_sum = 0; int64_t ghost_size_array[myMQv1_MAX_N_cache]; char *v = strsep((char **)&value, ":"); while (v != NULL) { diff --git a/libCacheSim/cache/prefetch/CMakeLists.txt b/libCacheSim/cache/prefetch/CMakeLists.txt index 025b6409..e41aa166 100644 --- a/libCacheSim/cache/prefetch/CMakeLists.txt +++ b/libCacheSim/cache/prefetch/CMakeLists.txt @@ -1,5 +1,5 @@ -add_library(prefetchC Mithril.c) +add_library(prefetchC Mithril.c OBL.c PG.c) add_library(prefetch INTERFACE) target_link_libraries(prefetch INTERFACE prefetchC) diff --git a/libCacheSim/cache/prefetch/Mithril.c b/libCacheSim/cache/prefetch/Mithril.c index 9678340a..6f237854 100644 --- a/libCacheSim/cache/prefetch/Mithril.c +++ b/libCacheSim/cache/prefetch/Mithril.c @@ -8,8 +8,6 @@ // Created by Zhelong on 23/8/15. // Copyright © 2023 Zhelong. All rights reserved. // -#include "../../include/libCacheSim/prefetchAlgo/Mithril.h" - #include #include #include @@ -19,6 +17,7 @@ #include #include "../../include/libCacheSim/prefetchAlgo.h" +#include "../../include/libCacheSim/prefetchAlgo/Mithril.h" #include "glibconfig.h" #define TRACK_BLOCK 192618l @@ -341,7 +340,7 @@ void Mithril_prefetch(cache_t *cache, const request_t *req) { (Mithril_params->pf_list_size + 1); request_t *new_req = my_malloc(request_t); - memcpy(new_req, req, sizeof(request_t)); + copy_request(new_req, req); if (prefetch_table_index) { int i; @@ -498,7 +497,7 @@ static inline bool _Mithril_check_sequential(cache_t *cache, if (Mithril_params->sequential_K == 0) return FALSE; request_t *new_req = my_malloc(request_t); - memcpy(new_req, req, sizeof(request_t)); + copy_request(new_req, req); bool is_sequential = TRUE; gint sequential_K = Mithril_params->sequential_K; if (sequential_K == -1) { /* when use AMP, this is -1 */ @@ -529,7 +528,7 @@ static inline void _Mithril_rec_min_support_one(cache_t *cache, printf("\n"); else printf(", block at old_pos %ld\n", - *(gint64 *)GET_ROW_IN_MTABLE(Mithril_params, old_pos - 1)); + (long)*(gint64 *)GET_ROW_IN_MTABLE(Mithril_params, old_pos - 1)); } else { gint64 b = TRACK_BLOCK; @@ -537,8 +536,9 @@ static inline void _Mithril_rec_min_support_one(cache_t *cache, g_hash_table_lookup(rmtable->hashtable, GINT_TO_POINTER(b))); if (old_pos != 0) { ERROR("ts %lu, checking %ld, %ld is found at pos %d\n", - Mithril_params->ts, TRACK_BLOCK, - *(gint64 *)GET_ROW_IN_MTABLE(Mithril_params, old_pos - 1), old_pos); + (unsigned long)Mithril_params->ts, (long)TRACK_BLOCK, + (long)*(gint64 *)GET_ROW_IN_MTABLE(Mithril_params, old_pos - 1), + old_pos); abort(); } } @@ -571,7 +571,7 @@ static inline void _Mithril_rec_min_support_one(cache_t *cache, GET_ROW_IN_MTABLE(Mithril_params, rmtable->mining_table->len - 1); if (req->obj_id != row_in_mtable[0]) { ERROR("after inserting, hashtable mining not consistent %ld %ld\n", - req->obj_id, row_in_mtable[0]); + (long)req->obj_id, (long)row_in_mtable[0]); abort(); } #endif @@ -582,7 +582,8 @@ static inline void _Mithril_rec_min_support_one(cache_t *cache, #ifdef SANITY_CHECK if (req->obj_id != row_in_mtable[0]) { ERROR("ts %lu, hashtable mining found position not correct %ld %ld\n", - Mithril_params->ts, req->obj_id, row_in_mtable[0]); + (unsigned long)Mithril_params->ts, (long)req->obj_id, + (long)row_in_mtable[0]); abort(); } #endif @@ -683,7 +684,7 @@ static inline void _Mithril_record_entry(cache_t *cache, const request_t *req) { "remove old entry from recording table, " "but it is not in recording hashtable, " "block %ld, recording table pos %ld, ts %ld ", - row_in_rtable[0], (long)rmtable->rtable_cur_row, + (long)row_in_rtable[0], (long)rmtable->rtable_cur_row, (long)Mithril_params->ts); long temp = rmtable->rtable_cur_row - 1; @@ -719,7 +720,7 @@ static inline void _Mithril_record_entry(cache_t *cache, const request_t *req) { "inconsistent entry in mtable " "and mining hashtable current request %ld, " "mining table %ld\n", - req->obj_id, row_in_mtable[0]); + (long)req->obj_id, (long)row_in_mtable[0]); abort(); } #endif @@ -767,7 +768,7 @@ static inline void _Mithril_record_entry(cache_t *cache, const request_t *req) { #ifdef SANITY_CHECK if (req->obj_id != row_in_rtable[0]) { ERROR("Hashtable recording found position not correct %ld %ld\n", - req->obj_id, row_in_rtable[0]); + (long)req->obj_id, (long) row_in_rtable[0]); abort(); } #endif @@ -819,9 +820,9 @@ static inline void _Mithril_record_entry(cache_t *cache, const request_t *req) { #ifdef SANITY_CHECK if (row_in_rtable == cur_row_in_rtable) ERROR("FOUND SRC DEST same, ts %ld %p %p %ld %ld %d %ld\n", - Mithril_params->ts, row_in_rtable, cur_row_in_rtable, - *row_in_rtable, *cur_row_in_rtable, index, - rmtable->rtable_cur_row - 1); + (long)Mithril_params->ts, row_in_rtable, cur_row_in_rtable, + (long)*row_in_rtable, (long)*cur_row_in_rtable, index, + (long)rmtable->rtable_cur_row - 1); #endif memcpy(row_in_rtable, cur_row_in_rtable, sizeof(TS_REPRESENTATION) * rmtable->rtable_row_len); @@ -843,7 +844,7 @@ static inline void _Mithril_record_entry(cache_t *cache, const request_t *req) { #ifdef SANITY_CHECK if (inserted_row_in_mtable[0] != (gint64)req->obj_id) { ERROR("current block %ld, moving mining row block %ld\n", - (gint64)req->obj_id, inserted_row_in_mtable[0]); + (long)req->obj_id, (long)inserted_row_in_mtable[0]); abort(); } #endif @@ -906,7 +907,7 @@ void print_one_line(gpointer key, gpointer value, gpointer user_data) { (Mithril_params->pf_list_size + 1); printf("src %d, prefetch ", src_key); for (int i = 1; i < Mithril_params->pf_list_size + 1; i++) { - printf("%ld ", Mithril_params->ptable_array[dim1][dim2 + i]); + printf("%ld ", (long)Mithril_params->ptable_array[dim1][dim2 + i]); } printf("\n"); } @@ -1011,7 +1012,8 @@ static void _Mithril_mining(cache_t *cache) { #ifdef PROFILING printf("ts: %lu, clearing training data takes %lf seconds\n", - Mithril_params->ts, g_timer_elapsed(timer, µsecond)); + (unsigned long)Mithril_params->ts, + g_timer_elapsed(timer, µsecond)); g_timer_stop(timer); g_timer_destroy(timer); #endif @@ -1056,8 +1058,8 @@ static void _Mithril_add_to_prefetch_table(cache_t *cache, gpointer gp1, #ifdef SANITY_CHECK if (Mithril_params->ptable_array[dim1][dim2] != GPOINTER_TO_INT(gp1)) { fprintf(stderr, "ERROR prefetch table pos wrong %d %ld, dim %d %d\n", - GPOINTER_TO_INT(gp1), Mithril_params->ptable_array[dim1][dim2], - dim1, dim2); + GPOINTER_TO_INT(gp1), + (long)Mithril_params->ptable_array[dim1][dim2], dim1, dim2); exit(1); } #endif diff --git a/libCacheSim/cache/prefetch/OBL.c b/libCacheSim/cache/prefetch/OBL.c new file mode 100644 index 00000000..a802b0c1 --- /dev/null +++ b/libCacheSim/cache/prefetch/OBL.c @@ -0,0 +1,178 @@ +// +// an OBL module that supports sequential prefetching for block storage. Each +// object (logical block address) should be uniform in size. +// +// +// OBL.c +// libCacheSim +// +// Created by Zhelong on 24/1/29. +// Copyright © 2024 Zhelong. All rights reserved. +// +#include "../../include/libCacheSim/prefetchAlgo/OBL.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "../../include/libCacheSim/prefetchAlgo.h" +// #define DEBUG + +#ifdef __cplusplus +extern "C" { +#endif + +// *********************************************************************** +// **** **** +// **** helper function declarations **** +// **** **** +// *********************************************************************** + +const char *OBL_default_params(void) { return "block-size=512, sequential-confidence-k=4"; } + +static void set_OBL_default_init_params(OBL_init_params_t *init_params) { + init_params->block_size = 512; + init_params->sequential_confidence_k = 4; +} + +static void OBL_parse_init_params(const char *cache_specific_params, OBL_init_params_t *init_params) { + char *params_str = strdup(cache_specific_params); + + while (params_str != NULL && params_str[0] != '\0') { + char *key = strsep((char **)¶ms_str, "="); + char *value = strsep((char **)¶ms_str, ","); + while (params_str != NULL && *params_str == ' ') { + params_str++; + } + if (strcasecmp(key, "block-size") == 0) { + init_params->block_size = atoi(value); + } else if (strcasecmp(key, "sequential-confidence-k") == 0) { + init_params->sequential_confidence_k = atoi(value); + } else { + ERROR("OBL does not have parameter %s\n", key); + printf("default params: %s\n", OBL_default_params()); + exit(1); + } + } +} + +static void set_OBL_params(OBL_params_t *OBL_params, OBL_init_params_t *init_params, uint64_t cache_size) { + OBL_params->block_size = init_params->block_size; + OBL_params->sequential_confidence_k = init_params->sequential_confidence_k; + OBL_params->do_prefetch = false; + if (OBL_params->sequential_confidence_k <= 0) { + printf("sequential_confidence_k should be positive\n"); + exit(1); + } + OBL_params->prev_access_block = (obj_id_t *)malloc(OBL_params->sequential_confidence_k * sizeof(obj_id_t)); + for (int i = 0; i < OBL_params->sequential_confidence_k; i++) { + OBL_params->prev_access_block[i] = UINT64_MAX; + } + OBL_params->curr_idx = 0; +} + +/************************************************************************** + ** prefetcher interfaces + ** + ** create, free, clone, handle_find, handle_insert, handle_evict, prefetch + **************************************************************************/ +prefetcher_t *create_OBL_prefetcher(const char *init_params, uint64_t cache_size); +/** + check if the previous access is sequential. If true, set do_prefetch to true. + +@param cache the cache struct +@param req the request containing the request +@return +*/ +static void OBL_handle_find(cache_t *cache, const request_t *req, bool hit) { + OBL_params_t *OBL_params = (OBL_params_t *)(cache->prefetcher->params); + int32_t sequential_confidence_k = OBL_params->sequential_confidence_k; + + // assert(req->obj_size == OBL_params->block_size); + bool flag = true; + for (int i = 0; i < sequential_confidence_k; i++) { + if (OBL_params->prev_access_block[(OBL_params->curr_idx + 1 + i) % sequential_confidence_k] != + req->obj_id - sequential_confidence_k + i) { + flag = false; + break; + } + } + OBL_params->do_prefetch = flag; + OBL_params->curr_idx = (OBL_params->curr_idx + 1) % sequential_confidence_k; + OBL_params->prev_access_block[OBL_params->curr_idx] = req->obj_id; +} + +/** + prefetch next block if the previous access is sequential + + @param cache the cache struct + @param req the request containing the request + @return + */ +void OBL_prefetch(cache_t *cache, const request_t *req) { + OBL_params_t *OBL_params = (OBL_params_t *)(cache->prefetcher->params); + + if (OBL_params->do_prefetch) { + OBL_params->do_prefetch = false; + request_t *new_req = new_request(); + new_req->obj_size = OBL_params->block_size; + new_req->obj_id = req->obj_id + 1; + if (cache->find(cache, new_req, false)) { + free_request(new_req); + return; + } + while (cache->get_occupied_byte(cache) + OBL_params->block_size > cache->cache_size) { + cache->evict(cache, req); + } + cache->insert(cache, new_req); + free_request(new_req); + } +} + +void free_OBL_prefetcher(prefetcher_t *prefetcher) { + OBL_params_t *OBL_params = (OBL_params_t *)prefetcher->params; + free(OBL_params->prev_access_block); + + my_free(sizeof(OBL_params_t), OBL_params); + if (prefetcher->init_params) { + free(prefetcher->init_params); + } + my_free(sizeof(prefetcher_t), prefetcher); +} + +prefetcher_t *clone_OBL_prefetcher(prefetcher_t *prefetcher, uint64_t cache_size) { + return create_OBL_prefetcher(prefetcher->init_params, cache_size); +} + +prefetcher_t *create_OBL_prefetcher(const char *init_params, uint64_t cache_size) { + OBL_init_params_t *OBL_init_params = my_malloc(OBL_init_params_t); + memset(OBL_init_params, 0, sizeof(OBL_init_params_t)); + + set_OBL_default_init_params(OBL_init_params); + if (init_params != NULL) { + OBL_parse_init_params(init_params, OBL_init_params); + } + + OBL_params_t *OBL_params = my_malloc(OBL_params_t); + set_OBL_params(OBL_params, OBL_init_params, cache_size); + + prefetcher_t *prefetcher = (prefetcher_t *)my_malloc(prefetcher_t); + memset(prefetcher, 0, sizeof(prefetcher_t)); + prefetcher->params = OBL_params; + prefetcher->prefetch = OBL_prefetch; + prefetcher->handle_find = OBL_handle_find; + prefetcher->handle_insert = NULL; + prefetcher->handle_evict = NULL; + prefetcher->free = free_OBL_prefetcher; + prefetcher->clone = clone_OBL_prefetcher; + if (init_params) { + prefetcher->init_params = strdup(init_params); + } + + my_free(sizeof(OBL_init_params_t), OBL_init_params); + return prefetcher; +} \ No newline at end of file diff --git a/libCacheSim/cache/prefetch/PG.c b/libCacheSim/cache/prefetch/PG.c new file mode 100644 index 00000000..db16c0e2 --- /dev/null +++ b/libCacheSim/cache/prefetch/PG.c @@ -0,0 +1,380 @@ +// +// a PG module that supports different obj size +// +// +// PG.c +// libCacheSim +// +// Created by Juncheng on 11/20/16. +// Copyright © 2016 Juncheng. All rights reserved. +// +// Modified by Zhelong on 2/21/24. + +#include +#include +#include +#include +#include +#include + +#include "../../include/libCacheSim/prefetchAlgo.h" + +#define TRACK_BLOCK 192618l +#define SANITY_CHECK 1 +#define PROFILING +// #define DEBUG + +#include "../../include/libCacheSim/prefetchAlgo/PG.h" +#include "glibconfig.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// *********************************************************************** +// **** **** +// **** helper function declarations **** +// **** **** +// *********************************************************************** +static inline void _graphNode_destroy(gpointer data); +static inline void _PG_add_to_graph(cache_t *cache, const request_t *req); +static inline GList *_PG_get_prefetch_list(cache_t *cache, const request_t *req); + +const char *PG_default_params(void) { + return "lookahead-range=20, " + "block-size=1, max-metadata-size=0.1, " + "prefetch-threshold=0.05"; +} + +static void set_PG_default_init_params(PG_init_params_t *init_params) { + init_params->lookahead_range = 20; + init_params->block_size = 1; // for general use + init_params->max_metadata_size = 0.1; + init_params->prefetch_threshold = 0.05; +} + +static void PG_parse_init_params(const char *cache_specific_params, PG_init_params_t *init_params) { + char *params_str = strdup(cache_specific_params); + + while (params_str != NULL && params_str[0] != '\0') { + char *key = strsep((char **)¶ms_str, "="); + char *value = strsep((char **)¶ms_str, ","); + while (params_str != NULL && *params_str == ' ') { + params_str++; + } + if (strcasecmp(key, "lookahead-range") == 0) { + init_params->lookahead_range = atoi(value); + } else if (strcasecmp(key, "block-size") == 0) { + init_params->block_size = (unsigned long)atoi(value); + } else if (strcasecmp(key, "max-metadata-size") == 0) { + init_params->max_metadata_size = atof(value); + } else if (strcasecmp(key, "prefetch-threshold") == 0) { + init_params->prefetch_threshold = atof(value); + } else if (strcasecmp(key, "print") == 0 || strcasecmp(key, "default") == 0) { + printf("default params: %s\n", PG_default_params()); + exit(0); + } else { + ERROR("pg does not have parameter %s\n", key); + printf("default params: %s\n", PG_default_params()); + exit(1); + } + } +} + +static void set_PG_params(PG_params_t *PG_params, PG_init_params_t *init_params, uint64_t cache_size) { + PG_params->lookahead_range = init_params->lookahead_range; + PG_params->block_size = init_params->block_size; + PG_params->cur_metadata_size = 0; + PG_params->max_metadata_size = (uint64_t)(init_params->block_size * cache_size * init_params->max_metadata_size); + PG_params->prefetch_threshold = init_params->prefetch_threshold; + + PG_params->stop_recording = FALSE; + + PG_params->graph = g_hash_table_new_full(g_direct_hash, g_direct_equal, NULL, _graphNode_destroy); + PG_params->prefetched = g_hash_table_new_full(g_direct_hash, g_direct_equal, NULL, NULL); + PG_params->past_requests = g_new0(guint64, PG_params->lookahead_range); + + PG_params->past_request_pointer = 0; + PG_params->num_of_hit = 0; + PG_params->num_of_prefetch = 0; + + PG_params->cache_size_map = g_hash_table_new_full(g_direct_hash, g_direct_equal, NULL, NULL); +} + +// *********************************************************************** +// **** **** +// **** prefetcher interfaces **** +// **** **** +// **** create, free, clone, handle_find, handle_evict, prefetch **** +// *********************************************************************** +prefetcher_t *create_PG_prefetcher(const char *init_params, uint64_t cache_size); +/** + 1. record the request in cache_size_map for being aware of prefetching object's + size in the future. + 2. call `_PG_add_to_graph` to update graph. + + @param cache the cache struct + @param req the request containing the request + @return +*/ +static void PG_handle_find(cache_t *cache, const request_t *req, bool hit) { + PG_params_t *PG_params = (PG_params_t *)(cache->prefetcher->params); + + /*use cache_size_map to record the current requested obj's size*/ + g_hash_table_insert(PG_params->cache_size_map, GINT_TO_POINTER(req->obj_id), GINT_TO_POINTER(req->obj_size)); + + _PG_add_to_graph(cache, req); + + if (g_hash_table_contains(PG_params->prefetched, GINT_TO_POINTER(req->obj_id))) { + PG_params->num_of_hit++; + g_hash_table_remove(PG_params->prefetched, GINT_TO_POINTER(req->obj_id)); + if (g_hash_table_contains(PG_params->prefetched, GINT_TO_POINTER(req->obj_id))) { + fprintf(stderr, "ERROR found prefetch\n"); + } + } +} + +/** + remove this obj from `prefetched` if it was previously prefetched into cache. + + @param cache the cache struct + @param req the request containing the request + @return +*/ +void PG_handle_evict(cache_t *cache, const request_t *check_req) { + PG_params_t *PG_params = (PG_params_t *)(cache->prefetcher->params); + + g_hash_table_remove(PG_params->prefetched, GINT_TO_POINTER(check_req->obj_id)); +} + +/** + prefetch some objects which are from `_PG_get_prefetch_list` + + @param cache the cache struct + @param req the request containing the request + @return + */ +void PG_prefetch(cache_t *cache, const request_t *req) { + PG_params_t *PG_params = (PG_params_t *)(cache->prefetcher->params); + + // begin prefetching + GList *prefetch_list = _PG_get_prefetch_list(cache, req); + if (prefetch_list) { + GList *node = prefetch_list; + request_t *new_req = my_malloc(request_t); + copy_request(new_req, req); + while (node) { + new_req->obj_id = GPOINTER_TO_INT(node->data); + new_req->obj_size = + GPOINTER_TO_INT(g_hash_table_lookup(PG_params->cache_size_map, GINT_TO_POINTER(new_req->obj_id))); + if (!cache->find(cache, new_req, false)) { + while ((long)cache->get_occupied_byte(cache) + new_req->obj_size + cache->obj_md_size > + (long)cache->cache_size) { + cache->evict(cache, new_req); + } + cache->insert(cache, new_req); + + PG_params->num_of_prefetch += 1; + + g_hash_table_insert(PG_params->prefetched, GINT_TO_POINTER(new_req->obj_id), GINT_TO_POINTER(1)); + } + node = node->next; + } + + my_free(sizeof(request_t), new_req); + g_list_free(prefetch_list); + } +} + +void free_PG_prefetcher(prefetcher_t *prefetcher) { + PG_params_t *PG_params = (PG_params_t *)prefetcher->params; + + g_hash_table_destroy(PG_params->cache_size_map); + g_hash_table_destroy(PG_params->graph); + g_hash_table_destroy(PG_params->prefetched); + + g_free(PG_params->past_requests); + + my_free(sizeof(PG_params_t), PG_params); + if (prefetcher->init_params) { + free(prefetcher->init_params); + } + my_free(sizeof(prefetcher_t), prefetcher); +} + +prefetcher_t *clone_PG_prefetcher(prefetcher_t *prefetcher, uint64_t cache_size) { + return create_PG_prefetcher(prefetcher->init_params, cache_size); +} + +prefetcher_t *create_PG_prefetcher(const char *init_params, uint64_t cache_size) { + PG_init_params_t *PG_init_params = my_malloc(PG_init_params_t); + memset(PG_init_params, 0, sizeof(PG_init_params_t)); + + set_PG_default_init_params(PG_init_params); + if (init_params != NULL) { + PG_parse_init_params(init_params, PG_init_params); + check_params((PG_init_params)); + } + + PG_params_t *PG_params = my_malloc(PG_params_t); + + set_PG_params(PG_params, PG_init_params, cache_size); + + prefetcher_t *prefetcher = (prefetcher_t *)my_malloc(prefetcher_t); + memset(prefetcher, 0, sizeof(prefetcher_t)); + prefetcher->params = PG_params; + prefetcher->prefetch = PG_prefetch; + prefetcher->handle_find = PG_handle_find; + prefetcher->handle_insert = NULL; + prefetcher->handle_evict = PG_handle_evict; + prefetcher->free = free_PG_prefetcher; + prefetcher->clone = clone_PG_prefetcher; + if (init_params) { + prefetcher->init_params = strdup(init_params); + } + + my_free(sizeof(PG_init_params_t), PG_init_params); + return prefetcher; +} + +/******************** PG help function ********************/ +static inline void _graphNode_destroy(gpointer data) { + graphNode_t *graphNode = (graphNode_t *)data; + g_hash_table_destroy(graphNode->graph); + pqueue_free(graphNode->pq); + g_free(graphNode); +} + +/** + 1. insert the `req->obj_id` to the past_request_pointer. + 2. update the graph using `past_requests[past_request_pointer]` as the + node and `node->past_requests[i]` as the directed arc. + + @param cache the cache struct + @param req the request containing the request + @return + */ +static inline void _PG_add_to_graph(cache_t *cache, const request_t *req) { + PG_params_t *PG_params = (PG_params_t *)(cache->prefetcher->params); + guint64 block, current_block = 0; + char req_lbl[MAX_OBJ_ID_LEN], current_req_lbl[MAX_OBJ_ID_LEN]; + graphNode_t *graphNode = NULL; + + current_block = get_Nth_past_request_l(PG_params, PG_params->past_request_pointer); + if (current_block) { + graphNode = (graphNode_t *)g_hash_table_lookup(PG_params->graph, GINT_TO_POINTER(current_block)); + } + + // now update past requests + set_Nth_past_request_l(PG_params, PG_params->past_request_pointer++, (guint64)(req->obj_id)); + + PG_params->past_request_pointer = PG_params->past_request_pointer % PG_params->lookahead_range; + + if (!(current_req_lbl[0] || current_block)) { + // this is the first request + return; + } + + if (graphNode == NULL) { + if (!PG_params->stop_recording) { + // current block is not in graph, insert + gpointer key = GINT_TO_POINTER(current_block); + graphNode = g_new0(graphNode_t, 1); + graphNode->graph = g_hash_table_new_full(g_direct_hash, g_direct_equal, NULL, g_free); + graphNode->pq = pqueue_init(2); + graphNode->total_count = 0; + g_hash_table_insert(PG_params->graph, key, graphNode); + PG_params->cur_metadata_size += (8 + 8 * 3); + } else { + // no space for meta data + return; + } + } + + for (int i = 0; i < PG_params->lookahead_range; i++) { + graphNode->total_count++; + + block = get_Nth_past_request_l(PG_params, i); + if (block == 0) break; + + pq_node_t *pq_node = (pq_node_t *)g_hash_table_lookup(graphNode->graph, GINT_TO_POINTER(block)); + if (pq_node) { + // relation already exists + pq_node->pri.pri++; + pqueue_change_priority(graphNode->pq, pq_node->pri, pq_node); + +#ifdef SANITY_CHECK + if (pq_node->obj_id != block) { + ERROR("pq node content not equal block\n"); + } +#endif + + } else { + // there is no probability between current_block->block + if (!PG_params->stop_recording) { + pq_node_t *pq_node = g_new0(pq_node_t, 1); + pq_node->obj_id = block; + pq_node->pri.pri = 1; + pqueue_insert(graphNode->pq, pq_node); + g_hash_table_insert(graphNode->graph, GINT_TO_POINTER(pq_node->obj_id), pq_node); + PG_params->cur_metadata_size += (8 + 8 * 3); + } else { + // no space for meta data + return; + } + } + } + + if (PG_params->max_metadata_size <= PG_params->cur_metadata_size) { + PG_params->stop_recording = TRUE; + } +} + +/** + get some objs which are associated with req->obj_id and their probability + is higher than `prefetch_threshold`. + + @param cache the cache struct + @param req the request containing the request + @return list containing all objs that should be prefetched + */ +static inline GList *_PG_get_prefetch_list(cache_t *cache, const request_t *req) { + PG_params_t *PG_params = (PG_params_t *)(cache->prefetcher->params); + GList *list = NULL; + graphNode_t *graphNode = g_hash_table_lookup(PG_params->graph, GINT_TO_POINTER(req->obj_id)); + + if (graphNode == NULL) { + return list; + } + + GList *pq_node_list = NULL; + while (1) { + pq_node_t *pqNode = pqueue_pop(graphNode->pq); + if (pqNode == NULL) { + break; + } + if ((double)(pqNode->pri.pri) / (graphNode->total_count) > PG_params->prefetch_threshold) { + list = g_list_prepend(list, GINT_TO_POINTER(pqNode->obj_id)); + pq_node_list = g_list_prepend(pq_node_list, pqNode); + } else { + // printf("threshold %lf\n", + // (double)(pqNode->pri)/(graphNode->total_count)); + break; + } + } + + if (pq_node_list) { + GList *node = pq_node_list; + while (node) { + pqueue_insert(graphNode->pq, node->data); + node = node->next; + } + } + g_list_free(pq_node_list); + + return list; +} + +#ifdef __cplusplus +} +#endif diff --git a/libCacheSim/dataStructure/CMakeLists.txt b/libCacheSim/dataStructure/CMakeLists.txt index f1193e1b..0fc28e42 100644 --- a/libCacheSim/dataStructure/CMakeLists.txt +++ b/libCacheSim/dataStructure/CMakeLists.txt @@ -1,4 +1,6 @@ add_subdirectory(hash) +add_subdirectory(test) + set(source pqueue.c splay.c @@ -7,6 +9,7 @@ set(source hash/murmur3.c hashtable/chainedHashtable.c hashtable/chainedHashTableV2.c + hashtable/cChainedHashTable.c ) add_library (dataStructure ${source}) diff --git a/libCacheSim/dataStructure/bloom.c b/libCacheSim/dataStructure/bloom.c index ae27b58f..14878857 100644 --- a/libCacheSim/dataStructure/bloom.c +++ b/libCacheSim/dataStructure/bloom.c @@ -27,19 +27,16 @@ #include "bloom.h" #include "hash/hash.h" #define XXH_INLINE_ALL -#include "hash/xxhash.h" #include "hash/xxh3.h" - +#include "hash/xxhash.h" #define MAKESTRING(n) STRING(n) #define STRING(n) #n - -inline static int test_bit_set_bit(unsigned char * buf, - unsigned int x, int set_bit) -{ +inline static int test_bit_set_bit(unsigned char *buf, unsigned int x, + int set_bit) { unsigned int byte = x >> 3; - unsigned char c = buf[byte]; // expensive memory access + unsigned char c = buf[byte]; // expensive memory access unsigned int mask = 1 << (x % 8); if (c & mask) { @@ -52,25 +49,23 @@ inline static int test_bit_set_bit(unsigned char * buf, } } - -static int bloom_check_add(struct bloom * bloom, - const void * buffer, int len, int add) -{ +static int bloom_check_add(struct bloom *bloom, const void *buffer, int len, + int add) { if (bloom->ready == 0) { printf("bloom at %p not initialized!\n", (void *)bloom); return -1; } int hits = 0; -// register unsigned int a = murmurhash2(buffer, len, 0x9747b28c); -// register unsigned int b = murmurhash2(buffer, len, a); + // register unsigned int a = murmurhash2(buffer, len, 0x9747b28c); + // register unsigned int b = murmurhash2(buffer, len, a); register unsigned int a = XXH64(buffer, len, HASH_SEED0); register unsigned int b = XXH64(buffer, len, HASH_SEED1); register unsigned int x; register unsigned int i; for (i = 0; i < bloom->hashes; i++) { - x = (a + i*b) % bloom->bits; + x = (a + i * b) % bloom->bits; if (test_bit_set_bit(bloom->bf, x, add)) { hits++; } else if (!add) { @@ -80,22 +75,18 @@ static int bloom_check_add(struct bloom * bloom, } if (hits == bloom->hashes) { - return 1; // 1 == element already in (or collision) + return 1; // 1 == element already in (or collision) } return 0; } - -int bloom_init_size(struct bloom * bloom, int entries, double error, - unsigned int cache_size) -{ +int bloom_init_size(struct bloom *bloom, int entries, double error, + unsigned int cache_size) { return bloom_init(bloom, entries, error); } - -int bloom_init(struct bloom * bloom, int entries, double error) -{ +int bloom_init(struct bloom *bloom, int entries, double error) { bloom->ready = 0; if (entries < 1000 || error == 0) { @@ -106,7 +97,7 @@ int bloom_init(struct bloom * bloom, int entries, double error) bloom->error = error; double num = log(bloom->error); - double denom = 0.480453013918201; // ln(2)^2 + double denom = 0.480453013918201; // ln(2)^2 bloom->bpe = -(num / denom); double dentries = (double)entries; @@ -121,29 +112,23 @@ int bloom_init(struct bloom * bloom, int entries, double error) bloom->hashes = (int)ceil(0.693147180559945 * bloom->bpe); // ln(2) bloom->bf = (unsigned char *)calloc(bloom->bytes, sizeof(unsigned char)); - if (bloom->bf == NULL) { // LCOV_EXCL_START + if (bloom->bf == NULL) { // LCOV_EXCL_START return 1; - } // LCOV_EXCL_STOP + } // LCOV_EXCL_STOP bloom->ready = 1; return 0; } - -int bloom_check(struct bloom * bloom, const void * buffer, int len) -{ +int bloom_check(struct bloom *bloom, const void *buffer, int len) { return bloom_check_add(bloom, buffer, len, 0); } - -int bloom_add(struct bloom * bloom, const void * buffer, int len) -{ +int bloom_add(struct bloom *bloom, const void *buffer, int len) { return bloom_check_add(bloom, buffer, len, 1); } - -void bloom_print(struct bloom * bloom) -{ +void bloom_print(struct bloom *bloom) { printf("bloom at %p\n", (void *)bloom); printf(" ->entries = %d\n", bloom->entries); printf(" ->error = %f\n", bloom->error); @@ -153,25 +138,17 @@ void bloom_print(struct bloom * bloom) printf(" ->hash functions = %d\n", bloom->hashes); } - -void bloom_free(struct bloom * bloom) -{ +void bloom_free(struct bloom *bloom) { if (bloom->ready) { free(bloom->bf); } bloom->ready = 0; } - -int bloom_reset(struct bloom * bloom) -{ +int bloom_reset(struct bloom *bloom) { if (!bloom->ready) return 1; memset(bloom->bf, 0, bloom->bytes); return 0; } - -const char * bloom_version() -{ - return MAKESTRING(BLOOM_VERSION); -} \ No newline at end of file +const char *bloom_version(void) { return MAKESTRING(BLOOM_VERSION); } \ No newline at end of file diff --git a/libCacheSim/dataStructure/bloom.h b/libCacheSim/dataStructure/bloom.h index 3156f087..ddd3d61c 100644 --- a/libCacheSim/dataStructure/bloom.h +++ b/libCacheSim/dataStructure/bloom.h @@ -164,7 +164,7 @@ int bloom_reset(struct bloom * bloom); * Return: version string * */ -const char * bloom_version(); +const char * bloom_version(void); #ifdef __cplusplus } diff --git a/libCacheSim/dataStructure/hash/CMakeLists.txt b/libCacheSim/dataStructure/hash/CMakeLists.txt index 579c4bf8..e32a49bd 100644 --- a/libCacheSim/dataStructure/hash/CMakeLists.txt +++ b/libCacheSim/dataStructure/hash/CMakeLists.txt @@ -1,5 +1,5 @@ -aux_source_directory(. DIR_LIB_SRCS) -add_library (ds_hash ${DIR_LIB_SRCS}) - +# aux_source_directory(. DIR_LIB_SRCS) +# add_library (ds_hash ${DIR_LIB_SRCS}) +add_library (ds_hash murmur3.c) diff --git a/libCacheSim/dataStructure/hashtable/cChainedHashTable.c b/libCacheSim/dataStructure/hashtable/cChainedHashTable.c new file mode 100644 index 00000000..6a4f1383 --- /dev/null +++ b/libCacheSim/dataStructure/hashtable/cChainedHashTable.c @@ -0,0 +1,246 @@ +// +// TODO +// High level view of the concurrent chained hash table. +// - Rwlock pool: a pool of rwlocks. The size of the pool is 2^(hashpower-7). +// - Hashtable: +// - Each bucket is a linked list of cache_obj_t. The head of the linked list is stored in the ptr_table. The size of the ptr_table is 2^hashpower. +// - Each bucket maps to a rw_lock in the rwlock pool. Mulpitle buckets likely maps to the same rwlock. +// Mapping: rwlock_id = bucket_id & (rw_count-1) +// +// Rwlock pool (count=4) A hashtable +// |-----------------| | ---------------| +// | rw_lock 0 | | bucket 0 | ----> cache_obj_t* ----> cache_obj_t* ----> NULL +// |-----------------| | ---------------| +// | rw_lock 1 | | bucket 1 | ----> cache_obj_t* +// |-----------------| | ---------------| +// | rw_lock 2 | | bucket 2 | ----> NULL +// |-----------------| | ---------------| +// | rw_lock 3 | | bucket 3 | ----> cache_obj_t* ----> cache_obj_t* ----> nULL +// |-----------------| | ---------------| +// | bucket 4 | ----> NULL +// | ---------------| +// | bucket 5 | ----> NULL +// | ---------------| + +#ifdef __cplusplus +extern "C" { +#endif + +#include "cChainedHashTable.h" + +#include +#include +#include +#include +#include +#include + +#include "../../include/libCacheSim/logging.h" +#include "../../include/libCacheSim/macro.h" +#include "../../utils/include/mymath.h" +#include "../hash/hash.h" + +/************************ help func ************************/ + +/** + * This function finds an object in the hashtable bucket. + * @method find_pointer_locked + * @author Chaos + * @date 2024-4-19 + * @param hashtable [Handle of the hashtable.] + * @param bucket_id [Id of the hashtable bucket.] + * @param obj_id [Id of the object to find.] + * @return [Double pointer to the object. (The pointer to 'obj->hash_next')] + */ + +static inline cache_obj_t** find_pointer_locked(const hashtable_t *hashtable, uint64_t bucket_id, obj_id_t obj_id){ + cache_obj_t **ptr = &hashtable->ptr_table[bucket_id]; + while(*ptr != NULL && obj_id != (*ptr)->obj_id){ + ptr = &(*ptr)->hash_next; + } + return ptr; +} + + +/************************ hashtable func ************************/ +/** + * [This function is not thread-safe. Other threads muse wait for the return of this function.] + * @method create_concurrent_chained_hashtable + * @author Chaos + * @date 2023-11-21 + * @param hashpower [The power of 2 of the size of the hashtable. Default value is 20.] + * @return [Handle of the created hashtable.] + */ +hashtable_t *create_concurrent_chained_hashtable(const uint16_t hashpower) { + hashtable_t *hashtable = my_malloc(hashtable_t); + memset(hashtable, 0, sizeof(hashtable_t)); + size_t size = sizeof(cache_obj_t *) * hashsize(hashtable->hashpower); + hashtable->ptr_table = my_malloc_n(cache_obj_t *, hashsize(hashpower)); + if (hashtable->ptr_table == NULL) { + ERROR("allcoate hash table %zu entry * %lu B = %ld MiB failed\n", + sizeof(cache_obj_t *), (unsigned long)(hashsize(hashpower)), + (long)(sizeof(cache_obj_t *) * hashsize(hashpower) / 1024 / 1024)); + exit(1); + } + memset(hashtable->ptr_table, 0, size); + +#ifdef USE_HUGEPAGE + madvise(hashtable->table, size, MADV_HUGEPAGE); +#endif + hashtable->external_obj = false; + hashtable->hashpower = hashpower; + hashtable->n_obj = 0; + hashtable->rwlocks_ = init_RWLocks((hashpower > 10) ? (hashpower-10) : 0); + return hashtable; +} + +/** + * This function finds an object in the hashtable. + * If the object is in the hashtable: + * - return the pointer. + * Else: + * - return NULL. + * @Author Chaos + * @Date 2024-04-18 + */ + +cache_obj_t *concurrent_chained_hashtable_find_obj_id(const hashtable_t *hashtable, + const obj_id_t obj_id) { + uint64_t hv = get_hash_value_int_64(&obj_id) & hashmask(hashtable->hashpower); + /** Add read lock for query */ + pthread_rwlock_t* rwlock_ = getRWLock(hashtable->rwlocks_, hv); + pthread_rwlock_rdlock(rwlock_); + /************within lock******************/ + cache_obj_t* entry = *find_pointer_locked(hashtable, hv, obj_id); + /************out lock******************/ + pthread_rwlock_unlock(rwlock_); + return entry; +} + +cache_obj_t *concurrent_chained_hashtable_find(const hashtable_t *hashtable, + const request_t *req) { + return concurrent_chained_hashtable_find_obj_id(hashtable, req->obj_id); +} + +cache_obj_t *concurrent_chained_hashtable_find_obj(const hashtable_t *hashtable, + const cache_obj_t *obj_to_find) { + return concurrent_chained_hashtable_find_obj_id(hashtable, obj_to_find->obj_id); +} + +/** + * This function inserts an object to the hashtable. + * If the object is in the hashtable: + * - delete the old one. + * + * Then, increase the number of objects in the hashtable and + * return the inserted object. + * @Author Chaos + * @Date 2024-04-18 + */ +cache_obj_t *concurrent_chained_hashtable_insert_obj(hashtable_t *hashtable, + cache_obj_t *cache_obj) { + // DEBUG_ASSERT(hashtable->external_obj); + uint64_t hv = get_hash_value_int_64(&cache_obj->obj_id) & hashmask(hashtable->hashpower); + /** Add write lock for insertion */ + pthread_rwlock_t* rwlock_ = getRWLock(hashtable->rwlocks_, hv); + pthread_rwlock_wrlock(rwlock_); + + /************within lock******************/ + cache_obj_t** ptr = find_pointer_locked(hashtable, hv, cache_obj->obj_id); + cache_obj_t* old = *ptr; + cache_obj->hash_next = (old == NULL ? NULL : old->hash_next); + // Initialize the in_cache flag. Set to true after the object is inserted into the cache eviction. + cache_obj_set_in_cache(cache_obj, false); + *ptr = cache_obj; + // If overwritten + if (old != NULL) { + cache_obj_set_in_cache(old, false); + } + else{ + // If successfully inserted, increase the number of objects in the hashtable. + hashtable->n_obj += 1; + } + /************out lock******************/ + + pthread_rwlock_unlock(rwlock_); + return old; +} + +cache_obj_t *concurrent_chained_hashtable_insert(hashtable_t *hashtable, + const request_t *req) { + cache_obj_t *new_cache_obj = create_cache_obj_from_request(req); + return concurrent_chained_hashtable_insert_obj(hashtable, new_cache_obj); +} + + +/** + * This function deletes an object in the hashtable. + * If the object is in the hashtable: + * - decrease the number of objects in the hashtable + * - return true. + * Else: + * - return false. + * @Author Chaos + * @Date 2023-11-22 + */ +cache_obj_t *concurrent_chained_hashtable_delete_obj_id(hashtable_t *hashtable, + const obj_id_t obj_id) { + uint64_t hv = get_hash_value_int_64(&obj_id) & hashmask(hashtable->hashpower); + /** Add write lock for removal */ + pthread_rwlock_t* rwlock_ = getRWLock(hashtable->rwlocks_, hv); + pthread_rwlock_wrlock(rwlock_); + /************within lock******************/ + cache_obj_t** ptr = find_pointer_locked(hashtable, hv, obj_id); + cache_obj_t* result = *ptr; + if (result != NULL) { + cache_obj_set_in_cache(result, false); + *ptr = result->hash_next; + hashtable->n_obj -= 1; + } + /************out lock******************/ + pthread_rwlock_unlock(rwlock_); + return result; +} + + +void concurrent_chained_hashtable_delete(hashtable_t *hashtable, + cache_obj_t *cache_obj) { + concurrent_chained_hashtable_delete_obj_id(hashtable, cache_obj->obj_id); +} + +bool concurrent_chained_hashtable_try_delete(hashtable_t *hashtable, + cache_obj_t *cache_obj) { + cache_obj_t* result = concurrent_chained_hashtable_delete_obj_id(hashtable, cache_obj->obj_id); + return result != NULL; +} + +cache_obj_t *concurrent_chained_hashtable_rand_obj(const hashtable_t *hashtable) { + uint64_t pos = next_rand() & hashmask(hashtable->hashpower); + /** Add read lock for random query */ + pthread_rwlock_t* rwlock_ = getRWLock(hashtable->rwlocks_, pos); + pthread_rwlock_rdlock(rwlock_); + + while (hashtable->ptr_table[pos] == NULL){ + pthread_rwlock_unlock(rwlock_); + pos = next_rand() & hashmask(hashtable->hashpower); + /** Add read lock for random query */ + rwlock_ = getRWLock(hashtable->rwlocks_, pos); + pthread_rwlock_rdlock(rwlock_); + } + pthread_rwlock_unlock(rwlock_); + return hashtable->ptr_table[pos]; +} + +// The hashtable is not responsible for freeing the objects. The release of objects is managed by the cache eviciton policies. +void free_concurrent_chained_hashtable(hashtable_t *hashtable) { + // if (!hashtable->external_obj) + // concurrent_chained_hashtable_foreach(hashtable, foreach_free_obj_locked, NULL); + my_free(sizeof(cache_obj_t *) * hashsize(hashtable->hashpower), + hashtable->ptr_table); + destory_RWLocks(hashtable->rwlocks_); +} + + +#ifdef __cplusplus +} +#endif diff --git a/libCacheSim/dataStructure/hashtable/cChainedHashTable.h b/libCacheSim/dataStructure/hashtable/cChainedHashTable.h new file mode 100644 index 00000000..50ee4350 --- /dev/null +++ b/libCacheSim/dataStructure/hashtable/cChainedHashTable.h @@ -0,0 +1,63 @@ +// This file defines a concurrent chained hashtable data structure. It is based on chainedHashTableV2. +// Some Features: +// - Static hashtable size +// - Thread safe +// - No global lock +// +// Created by Chaos on 11/23/2023. +// + +#ifndef libCacheSim_CCHAINEDHASHTABLE_H +#define libCacheSim_CCHAINEDHASHTABLE_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include + +#include "../../include/libCacheSim/cacheObj.h" +#include "../../include/libCacheSim/request.h" +#include "hashtableStruct.h" + +hashtable_t *create_concurrent_chained_hashtable(const uint16_t hashpower_init); + +void free_concurrent_chained_hashtable(hashtable_t *hashtable); + +cache_obj_t *concurrent_chained_hashtable_find_obj_id(const hashtable_t *hashtable, + const obj_id_t obj_id); + +cache_obj_t *concurrent_chained_hashtable_find(const hashtable_t *hashtable, + const request_t *req); + +cache_obj_t *concurrent_chained_hashtable_find_obj(const hashtable_t *hashtable, + const cache_obj_t *obj_to_evict); + +/* return an empty cache_obj_t */ +cache_obj_t *concurrent_chained_hashtable_insert(hashtable_t *hashtable, + const request_t *req); + +cache_obj_t *concurrent_chained_hashtable_insert_obj(hashtable_t *hashtable, + cache_obj_t *cache_obj); + +bool concurrent_chained_hashtable_try_delete(hashtable_t *hashtable, + cache_obj_t *cache_obj); + +void concurrent_chained_hashtable_delete(hashtable_t *hashtable, + cache_obj_t *cache_obj); + +cache_obj_t *concurrent_chained_hashtable_delete_obj_id(hashtable_t *hashtable, + const obj_id_t obj_id); + +cache_obj_t *concurrent_chained_hashtable_rand_obj(const hashtable_t *hashtable); + +void concurrent_chained_hashtable_foreach(hashtable_t *hashtable, + hashtable_iter iter_func, void *user_data); + + +#ifdef __cplusplus +} +#endif + +#endif // libCacheSim_CCHAINEDHASHTABLE_H diff --git a/libCacheSim/dataStructure/hashtable/chainedHashTable.h b/libCacheSim/dataStructure/hashtable/chainedHashTable.h index bb4a2965..4c7e9b04 100644 --- a/libCacheSim/dataStructure/hashtable/chainedHashTable.h +++ b/libCacheSim/dataStructure/hashtable/chainedHashTable.h @@ -67,4 +67,4 @@ void _chained_hashtable_expand(hashtable_t *hashtable); } #endif -#endif // libCacheSim_CHAINEDHASHTABLE_H +#endif // libCacheSim_CHAINEDHASHTABLE_H \ No newline at end of file diff --git a/libCacheSim/dataStructure/hashtable/chainedHashTableV2.c b/libCacheSim/dataStructure/hashtable/chainedHashTableV2.c index 8a5eb960..1d4e6a2e 100644 --- a/libCacheSim/dataStructure/hashtable/chainedHashTableV2.c +++ b/libCacheSim/dataStructure/hashtable/chainedHashTableV2.c @@ -23,8 +23,6 @@ extern "C" { #endif -#include "chainedHashTableV2.h" - #include #include #include @@ -36,6 +34,7 @@ extern "C" { #include "../../include/libCacheSim/macro.h" #include "../../utils/include/mymath.h" #include "../hash/hash.h" +#include "chainedHashTableV2.h" #define OBJ_EMPTY(cache_obj) ((cache_obj)->obj_size == 0) #define NEXT_OBJ(cur_obj) (((cache_obj_t *)(cur_obj))->hash_next) @@ -248,26 +247,37 @@ bool chained_hashtable_try_delete_v2(hashtable_t *hashtable, return false; } -void chained_hashtable_delete_obj_id_v2(hashtable_t *hashtable, +bool chained_hashtable_delete_obj_id_v2(hashtable_t *hashtable, const obj_id_t obj_id) { - hashtable->n_obj -= 1; - uint64_t hv = get_hash_value_int_64(obj_id) & hashmask(hashtable->hashpower); - cache_obj_t *cache_obj = hashtable->ptr_table[hv]; - if (cache_obj != NULL && cache_obj->obj_id == obj_id) { - hashtable->ptr_table[hv] = cache_obj->hash_next; - if (!hashtable->external_obj) free_cache_obj(cache_obj); - return; - } - cache_obj = cache_obj->hash_next; - while (cache_obj != NULL && cache_obj->obj_id != obj_id) { - cache_obj = cache_obj->hash_next; + uint64_t hv = get_hash_value_int_64(&obj_id) & hashmask(hashtable->hashpower); + cache_obj_t *cur_obj = hashtable->ptr_table[hv]; + // the hash bucket is empty + if(cur_obj == NULL) return false; + + // the object to remove is the first object in the hash bucket + if (cur_obj->obj_id == obj_id) { + hashtable->ptr_table[hv] = cur_obj->hash_next; + if (!hashtable->external_obj) free_cache_obj(cur_obj); + hashtable->n_obj -= 1; + return true; } - if (cache_obj != NULL) { - cache_obj->hash_next = cache_obj->hash_next; - if (!hashtable->external_obj) free_cache_obj(cache_obj); + cache_obj_t *prev_obj; + + do { + prev_obj = cur_obj; + cur_obj = cur_obj->hash_next; + } while(cur_obj != NULL && cur_obj->obj_id != obj_id); + + if (cur_obj != NULL) { + prev_obj->hash_next = cur_obj->hash_next; + if (!hashtable->external_obj) free_cache_obj(cur_obj); + hashtable->n_obj -= 1; + return true; } + // the object to remove is not in the hash table + return false; } cache_obj_t *chained_hashtable_rand_obj_v2(const hashtable_t *hashtable) { @@ -352,7 +362,8 @@ static int count_n_obj_in_bucket(cache_obj_t *curr_obj) { obj_id_arr[chain_len] = curr_obj->obj_id; for (int i = 0; i < chain_len; i++) { if (obj_id_arr[i] == curr_obj->obj_id) { - ERROR("obj_id %lu is duplicated in hashtable\n", curr_obj->obj_id); + ERROR("obj_id %lu is duplicated in hashtable\n", + (unsigned long)curr_obj->obj_id); abort(); } } @@ -384,4 +395,4 @@ static void print_hashbucket_item_distribution(const hashtable_t *hashtable) { #ifdef __cplusplus } -#endif +#endif \ No newline at end of file diff --git a/libCacheSim/dataStructure/hashtable/chainedHashTableV2.h b/libCacheSim/dataStructure/hashtable/chainedHashTableV2.h index ade28eb9..5ec88c7b 100644 --- a/libCacheSim/dataStructure/hashtable/chainedHashTableV2.h +++ b/libCacheSim/dataStructure/hashtable/chainedHashTableV2.h @@ -40,6 +40,9 @@ bool chained_hashtable_try_delete_v2(hashtable_t *hashtable, void chained_hashtable_delete_v2(hashtable_t *hashtable, cache_obj_t *cache_obj); +bool chained_hashtable_delete_obj_id_v2(hashtable_t *hashtable, + const obj_id_t obj_id); + cache_obj_t *chained_hashtable_rand_obj_v2(const hashtable_t *hashtable); void chained_hashtable_foreach_v2(hashtable_t *hashtable, @@ -56,4 +59,4 @@ void check_hashtable_integrity2_v2(const hashtable_t *hashtable, } #endif -#endif // libCacheSim_CHAINEDHASHTABLEV2_H +#endif // libCacheSim_CHAINEDHASHTABLEV2_H \ No newline at end of file diff --git a/libCacheSim/dataStructure/hashtable/chainedHashtable.c b/libCacheSim/dataStructure/hashtable/chainedHashtable.c index 1f8cb6fe..38281fcf 100644 --- a/libCacheSim/dataStructure/hashtable/chainedHashtable.c +++ b/libCacheSim/dataStructure/hashtable/chainedHashtable.c @@ -196,13 +196,13 @@ cache_obj_t *chained_hashtable_insert(hashtable_t *hashtable, request_t *req) { cache_obj->hash_next = new_cache_obj; cache_obj = new_cache_obj; } - hashtable->n_obj += 1; + __sync_fetch_and_add(&hashtable->n_obj, 1); return cache_obj; } /* you need to free the extra_metadata before deleting from hash table */ void chained_hashtable_delete(hashtable_t *hashtable, cache_obj_t *cache_obj) { - hashtable->n_obj -= 1; + __sync_fetch_and_sub(&hashtable->n_obj, 1); uint64_t hv = get_hash_value_int_64(&cache_obj->obj_id) & hashmask(hashtable->hashpower); cache_obj_t *cache_obj_in_bucket = &hashtable->table[hv]; diff --git a/libCacheSim/dataStructure/hashtable/hashtable.h b/libCacheSim/dataStructure/hashtable/hashtable.h index 7185a551..56d2b2a5 100644 --- a/libCacheSim/dataStructure/hashtable/hashtable.h +++ b/libCacheSim/dataStructure/hashtable/hashtable.h @@ -6,6 +6,7 @@ #ifdef __cplusplus extern "C" { + #endif #include "../../include/config.h" @@ -14,52 +15,78 @@ extern "C" { #include "../../utils/include/mymath.h" #include "hashtableStruct.h" -#if HASHTABLE_TYPE == CHAINED_HASHTABLE -#include "chainedHashTable.h" -#define create_hashtable(hashpower) create_chained_hashtable(hashpower) -#define hashtable_find(hashtable, obj_id) \ - chained_hashtable_find(hashtable, obj_id) -#define hashtable_find_req(hashtable, req) \ - chained_hashtable_find(hashtable, req) -#define hashtable_find_obj(hashtable, cache_obj) \ - chained_hashtable_find_obj(hashtable, cache_obj) -#define hashtable_insert(hashtable, req) \ - chained_hashtable_insert(hashtable, req) -#define hashtable_insert_obj(hashtable, cache_obj) assert(0); -#define hashtable_delete(hashtable, cache_obj) \ - chained_hashtable_delete(hashtable, cache_obj) -#define hashtable_rand_obj(hashtable) chained_hashtable_rand_obj(hashtable) -#define hashtable_foreach(hashtable, iter_func, user_data) \ - chained_hashtable_foreach(hashtable, iter_func, user_data) -#define free_hashtable(hashtable) free_chained_hashtable(hashtable) -#define hashtable_add_ptr_to_monitoring(hashtable, ptr) \ - chained_hashtable_add_ptr_to_monitoring(hashtable, ptr) -#define HASHTABLE_VER 1 +// #if HASHTABLE_TYPE == CHAINED_HASHTABLE +// #include "chainedHashTable.h" +// #define create_hashtable(hashpower) create_chained_hashtable(hashpower) +// #define hashtable_find(hashtable, obj_id) \ +// chained_hashtable_find(hashtable, obj_id) +// #define hashtable_find_req(hashtable, req) \ +// chained_hashtable_find(hashtable, req) +// #define hashtable_find_obj(hashtable, cache_obj) \ +// chained_hashtable_find_obj(hashtable, cache_obj) +// #define hashtable_insert(hashtable, req) \ +// chained_hashtable_insert(hashtable, req) +// #define hashtable_insert_obj(hashtable, cache_obj) assert(0); +// #define hashtable_delete(hashtable, cache_obj) \ +// chained_hashtable_delete(hashtable, cache_obj) +// #define hashtable_rand_obj(hashtable) chained_hashtable_rand_obj(hashtable) +// #define hashtable_foreach(hashtable, iter_func, user_data) \ +// chained_hashtable_foreach(hashtable, iter_func, user_data) +// #define free_hashtable(hashtable) free_chained_hashtable(hashtable) +// #define hashtable_add_ptr_to_monitoring(hashtable, ptr) \ +// chained_hashtable_add_ptr_to_monitoring(hashtable, ptr) +// #define HASHTABLE_VER 1 + +// #elif HASHTABLE_TYPE == CHAINED_HASHTABLEV2 +// #include "chainedHashTableV2.h" +// #define create_hashtable(hashpower) create_chained_hashtable_v2(hashpower) +// #define hashtable_find(hashtable, req) chained_hashtable_find_v2(hashtable, req) +// #define hashtable_find_obj_id(hashtable, obj_id) \ +// chained_hashtable_find_obj_id_v2(hashtable, obj_id) +// #define hashtable_find_obj(hashtable, cache_obj) \ +// chained_hashtable_find_obj_v2(hashtable, cache_obj) +// #define hashtable_insert(hashtable, req) \ +// chained_hashtable_insert_v2(hashtable, req) +// #define hashtable_insert_obj(hashtable, cache_obj) \ +// chained_hashtable_insert_obj_v2(hashtable, cache_obj) +// #define hashtable_delete(hashtable, cache_obj) \ +// chained_hashtable_delete_v2(hashtable, cache_obj) +// #define hashtable_try_delete(hashtable, cache_obj) \ +// chained_hashtable_try_delete_v2(hashtable, cache_obj) +// #define hashtable_delete_obj_id(hashtable, obj_id) \ +// chained_hashtable_delete_obj_id_v2(hashtable, obj_id) +// #define hashtable_rand_obj(hashtable) chained_hashtable_rand_obj_v2(hashtable) +// #define hashtable_foreach(hashtable, iter_func, user_data) \ +// chained_hashtable_foreach_v2(hashtable, iter_func, user_data) +// #define free_hashtable(hashtable) free_chained_hashtable_v2(hashtable) +// #define hashtable_add_ptr_to_monitoring(hashtable, ptr) +// #define HASHTABLE_VER 2 -#elif HASHTABLE_TYPE == CHAINED_HASHTABLEV2 -#include "chainedHashTableV2.h" -#define create_hashtable(hashpower) create_chained_hashtable_v2(hashpower) -#define hashtable_find(hashtable, req) chained_hashtable_find_v2(hashtable, req) +// #elif HASHTABLE_TYPE == CONCURRENT_CHAINED_HASHTABLE +#if HASHTABLE_TYPE == CONCURRENT_CHAINED_HASHTABLE +#include "cChainedHashTable.h" +#define create_hashtable(hashpower) create_concurrent_chained_hashtable(hashpower) +#define hashtable_find(hashtable, req) concurrent_chained_hashtable_find(hashtable, req) #define hashtable_find_obj_id(hashtable, obj_id) \ - chained_hashtable_find_obj_id_v2(hashtable, obj_id) + concurrent_chained_hashtable_find_obj_id(hashtable, obj_id) #define hashtable_find_obj(hashtable, cache_obj) \ - chained_hashtable_find_obj_v2(hashtable, cache_obj) + concurrent_chained_hashtable_find_obj(hashtable, cache_obj) #define hashtable_insert(hashtable, req) \ - chained_hashtable_insert_v2(hashtable, req) + concurrent_chained_hashtable_insert(hashtable, req) #define hashtable_insert_obj(hashtable, cache_obj) \ - chained_hashtable_insert_obj_v2(hashtable, cache_obj) + concurrent_chained_hashtable_insert_obj(hashtable, cache_obj) #define hashtable_delete(hashtable, cache_obj) \ - chained_hashtable_delete_v2(hashtable, cache_obj) + concurrent_chained_hashtable_delete(hashtable, cache_obj) #define hashtable_try_delete(hashtable, cache_obj) \ - chained_hashtable_try_delete_v2(hashtable, cache_obj) + concurrent_chained_hashtable_try_delete(hashtable, cache_obj) #define hashtable_delete_obj_id(hashtable, obj_id) \ - chained_hashtable_delete_obj_id_v2(hashtable, obj_id) -#define hashtable_rand_obj(hashtable) chained_hashtable_rand_obj_v2(hashtable) + concurrent_chained_hashtable_delete_obj_id(hashtable, obj_id) +#define hashtable_rand_obj(hashtable) concurrent_chained_hashtable_rand_obj(hashtable) #define hashtable_foreach(hashtable, iter_func, user_data) \ - chained_hashtable_foreach_v2(hashtable, iter_func, user_data) -#define free_hashtable(hashtable) free_chained_hashtable_v2(hashtable) + concurrent_chained_hashtable_foreach(hashtable, iter_func, user_data) +#define free_hashtable(hashtable) free_concurrent_chained_hashtable(hashtable) #define hashtable_add_ptr_to_monitoring(hashtable, ptr) -#define HASHTABLE_VER 2 +#define HASHTABLE_VER 3 #elif HASHTABLE_TYPE == CUCKCOO_HASHTABLE #include "cuckooHashTable.h" diff --git a/libCacheSim/dataStructure/hashtable/hashtableStruct.h b/libCacheSim/dataStructure/hashtable/hashtableStruct.h index 13ad7b1b..1d24c36d 100644 --- a/libCacheSim/dataStructure/hashtable/hashtableStruct.h +++ b/libCacheSim/dataStructure/hashtable/hashtableStruct.h @@ -10,8 +10,10 @@ extern "C" { #endif #include +#include #include "../../include/libCacheSim/cacheObj.h" +#include "../../utils/include/mymutex.h" #define hashsize(n) ((uint64_t)1 << (uint16_t)(n)) #define hashsizeULL(n) ((unsigned long long)1 << (uint16_t)(n)) @@ -40,6 +42,7 @@ typedef struct hashtable { }; void *extra_data; }; + RWLocks_t* rwlocks_; } hashtable_t; #ifdef __cplusplus diff --git a/libCacheSim/dataStructure/minimalIncrementCBF.c b/libCacheSim/dataStructure/minimalIncrementCBF.c index cbeea2ea..7fbd347f 100644 --- a/libCacheSim/dataStructure/minimalIncrementCBF.c +++ b/libCacheSim/dataStructure/minimalIncrementCBF.c @@ -69,6 +69,7 @@ int minimalIncrementCBF_init(struct minimalIncrementCBF * CBF, int entries, doub int minimalIncrementCBF_init_size(struct minimalIncrementCBF * CBF, int entries, double error, unsigned int cache_size) { minimalIncrementCBF_init(CBF, entries, error); + return 0; } static int minimalIncrementCBF_check_add(struct minimalIncrementCBF * CBF, @@ -167,6 +168,6 @@ int minimalIncrementCBF_decay(struct minimalIncrementCBF * CBF) { return 0; } -const char * minimalIncrementCBF_version() { +const char * minimalIncrementCBF_version(void) { return MAKESTRING(VERSION); } \ No newline at end of file diff --git a/libCacheSim/dataStructure/minimalIncrementCBF.h b/libCacheSim/dataStructure/minimalIncrementCBF.h index ada2227b..5022cb9b 100644 --- a/libCacheSim/dataStructure/minimalIncrementCBF.h +++ b/libCacheSim/dataStructure/minimalIncrementCBF.h @@ -142,7 +142,7 @@ int minimalIncrementCBF_decay(struct minimalIncrementCBF * bloom); * Return: version string * */ -const char * minimalIncrementCBF_version(); +const char * minimalIncrementCBF_version(void); #ifdef __cplusplus } diff --git a/libCacheSim/dataStructure/robin_hood.h b/libCacheSim/dataStructure/robin_hood.h index 1a5c294d..7ee5d323 100644 --- a/libCacheSim/dataStructure/robin_hood.h +++ b/libCacheSim/dataStructure/robin_hood.h @@ -207,7 +207,7 @@ static Counts& counts() { // workaround missing "is_trivially_copyable" in g++ < 5.0 // See https://stackoverflow.com/a/31798726/48181 #if defined(__GNUC__) && __GNUC__ < 5 -# define ROBIN_HOOD_IS_TRIVIALLY_COPYABLE(...) __has_trivial_copy(__VA_ARGS__) +# define ROBIN_HOOD_IS_TRIVIALLY_COPYABLE(...) __is_trivially_copyable(__VA_ARGS__) #else # define ROBIN_HOOD_IS_TRIVIALLY_COPYABLE(...) std::is_trivially_copyable<__VA_ARGS__>::value #endif diff --git a/libCacheSim/dataStructure/test/CMakeLists.txt b/libCacheSim/dataStructure/test/CMakeLists.txt new file mode 100644 index 00000000..1bd287e6 --- /dev/null +++ b/libCacheSim/dataStructure/test/CMakeLists.txt @@ -0,0 +1,4 @@ +add_executable(testCHashtable test_concurrent_hashtable.c) +target_link_libraries(testCHashtable dataStructure utils cachelib pthread) + +add_test(NAME testCHashtable COMMAND testCHashtable) diff --git a/libCacheSim/dataStructure/test/test_concurrent_hashtable.c b/libCacheSim/dataStructure/test/test_concurrent_hashtable.c new file mode 100644 index 00000000..3bbab1a4 --- /dev/null +++ b/libCacheSim/dataStructure/test/test_concurrent_hashtable.c @@ -0,0 +1,546 @@ +/* + * Created by ChaosD on 11/22/23. + * This file is used to test the concurrent throughput of hashtable structures. + * Now the test only supports two types of hashtable: chainedHashTableV2, and cChainedHashTable. + * It supports three types of operations: read obj, insert obj, and delete obj. + */ + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include +#include +#include +#include + +#include "../../utils/include/mymath.h" +#include "../../include/libCacheSim/cacheObj.h" +#include "../hashtable/chainedHashTableV2.h" +#include "../hashtable/cChainedHashTable.h" + +/** Power of number of cache objects to insert. */ +size_t g_power = 19; + +/** Number of cache objects to insert. */ +size_t g_numkeys; // 2^18=262144 + +/** Number of threads to spawn for each type of operation. */ +size_t g_thread_num = 1; + +/** Number of seconds to run read test. */ +size_t g_test_len = 20; // 10s + +/** Type of test. */ +size_t g_test_type = 0; // 0: Three tests: Insert, Read, and Delete. Each test is non-overlapping. + // 1: Mixed. Each thread performs insert, read, and delete in turn. + // The three operations are overlapping between different threads. + +/** Type of hashtable. */ +size_t g_ht_type = 0; // 0: cChainedHashTable; 1: chainedHashTableV2; + + +/** Function pointer to find a cache object in the cache instance. */ +typedef cache_obj_t* (*func_hashtable_find_obj_id_t)(const hashtable_t *hashtable, + const obj_id_t obj_id); +/** Function pointer to insert a cache object in the cache instance. */ +typedef cache_obj_t* (*func_hashtable_insert_obj_t)(hashtable_t *hashtable, + cache_obj_t *cache_obj); +/** Function pointer to remove a cache object in the cache instance. */ +typedef cache_obj_t* (*func_hashtable_delete_obj_id_t)(hashtable_t *hashtable, + const obj_id_t obj_id); + +typedef hashtable_t *(*func_create_hashtable_t)(const uint16_t hashpower_init); + +/** Function pointer to free a hashtable instance. */ +typedef void (*func_free_hashtable_t)(hashtable_t *hashtable); + + +struct handle_hashtable_t{ + func_hashtable_find_obj_id_t func_find_obj; + func_hashtable_insert_obj_t func_insert_obj; + func_hashtable_delete_obj_id_t func_delete_obj; + func_create_hashtable_t func_create_hashtable; + func_free_hashtable_t func_free_hashtable; +} ht_handle; + +/** + * Struct of parameters for test threads. + * @author Chaos + * @date 2023-11-22 + * @number table [Handle of a cache instance] + * @number r_counter [Number of cache objects read] + * @number i_counter [Number of cache objects inserted] + * @number d_counter [Number of cache objects deleted] + * @number finished [Signal of the end of read] + * @number range [Range of obj ID to read] + * @number thread_id [thread ID number] + * @number func_find [Function pointer to find a cache object] + * @number func_insert [Function pointer to insert a cache object] + * @number func_delete [Function pointer to delete a cache object] + */ +typedef struct thread_para{ + hashtable_t* table; + uint64_t* r_counter; + uint64_t* i_counter; + uint64_t* d_counter; + bool* finished; + uint64_t range; + uint32_t thread_id; +}thread_para_t; + +/** + * [This function reads a cache object from the cache instance. It is used as a thread function.] + * @method func_read + * @author Chaos + * @date 2023-11-22 + */ +void *func_read(void* arg){ + thread_para_t* para = (thread_para_t*) arg; + hashtable_t* table = para->table; + uint64_t* r_counter = para->r_counter; + bool* finished = para->finished; + uint64_t range = para->range; + uint32_t thread_id = para->thread_id; + +// We keep track of our own local counter for reads, to avoid +// over-burdening the shared atomic counter + size_t reads = 0; + size_t success_reads = 0; + size_t fail_reads = 0; + + struct timeval start, end; + // We use thread id as seed to generate a reproducible pseudo random sequence. + // The items in the sequence are operated obj IDs. + obj_id_t cur_obj_id = thread_id; + // The function keeps runing until the flag `finished` == True. + gettimeofday(&start, NULL); + for (uint64_t it = 0;; it++) { + if(it == range){ + it = 0; + cur_obj_id = thread_id; + } + if (*finished) { + __sync_fetch_and_add(r_counter, reads); + gettimeofday(&end, NULL); + double timeuse = (end.tv_sec - start.tv_sec) + + (end.tv_usec - start.tv_usec) / 1000000.0; + INFO("Thread %d read %zu objects in %.1f seconds, throughput is %.2f MOPS. %zu success, %zu fail\n", \ + thread_id, reads, timeuse, reads / timeuse / 1000000, success_reads, fail_reads); + + return NULL; + } + cur_obj_id = get_next_rand(cur_obj_id); + if(ht_handle.func_find_obj(table, cur_obj_id)){ + success_reads++; + } + else{ + fail_reads++; + } + reads++; + } +} + +/** + * [This function inserts a cache object to the cache instance. It is used as a thread function.] + * @method func_insert + * @author Chaos + * @date 2023-11-22 + */ +void *func_insert(void* arg){ + thread_para_t* para = (thread_para_t*) arg; + hashtable_t* table = para->table; + uint64_t* i_counter = para->i_counter; + uint64_t range = para->range; + uint32_t thread_id = para->thread_id; + +// We keep track of our own local counter for inserts, to avoid +// over-burdening the shared atomic counter + size_t inserts = 0; + // We use thread id as seed to generate a reproducible pseudo random sequence. + // The items in the sequence are operated obj IDs. + obj_id_t cur_obj_id = thread_id; + struct timeval start, end; + gettimeofday(&start, NULL); + for (uint64_t it = 0; it < range; it++) { + cur_obj_id = get_next_rand(cur_obj_id); + cache_obj_t* cur_obj = create_cache_obj_from_obj_id(cur_obj_id); + cache_obj_t* result = ht_handle.func_insert_obj(table, cur_obj); + if(result){ + free_cache_obj(result); + } + inserts++; + } + /** Update the shared atomic counter */ + __sync_fetch_and_add(i_counter, inserts); + gettimeofday(&end, NULL); + double timeuse = (end.tv_sec - start.tv_sec) + + (end.tv_usec - start.tv_usec) / 1000000.0; + INFO("Thread %d inserted %zu objects in %.1f seconds, throughput is %.2f MOPS\n", \ + thread_id, inserts, timeuse, inserts / timeuse / 1000000); +} + +/** + * [This function inserts a cache object to the cache instance. It is used as a thread function.] + * @method func_remove + * @author Chaos + * @date 2023-11-22 + */ +void *func_remove(void* arg){ + thread_para_t* para = (thread_para_t*) arg; + hashtable_t* table = para->table; + uint64_t* d_counter = para->d_counter; + uint64_t range = para->range; + uint32_t thread_id = para->thread_id; + +// We keep track of our own local counter for removals, to avoid +// over-burdening the shared atomic counter + size_t removals = 0; + size_t success_removal = 0; + size_t fail_removal = 0; + + // We use thread id as seed to generate a reproducible pseudo random sequence. + // The items in the sequence are operated obj IDs. + obj_id_t cur_obj_id = thread_id; + struct timeval start, end; + gettimeofday(&start, NULL); + for (uint64_t it = 0; it < range; it++) { + cur_obj_id = get_next_rand(cur_obj_id); + cache_obj_t* result = ht_handle.func_delete_obj(table, cur_obj_id); + if(result){ + success_removal++; + free_cache_obj(result); + } + else{ + fail_removal++; + } + removals++; + } + // Update the shared atomic counter + __sync_fetch_and_add(d_counter, removals); + gettimeofday(&end, NULL); + double timeuse = (end.tv_sec - start.tv_sec) + + (end.tv_usec - start.tv_usec) / 1000000.0; + INFO("Thread %d removed %zu objects in %.1f seconds, throughput is %.2f MOPS. %zu success, %zu fail\n", \ + thread_id, removals, timeuse, removals / timeuse / 1000000, success_removal, fail_removal); +} + + +/** + * This function is for mixed tests. It is used as a thread function. It works as follows: + * 1. Insert `range` objects to the cache instance. + * 2. Read `range` objects from the cache instance for 10 rounds. + * 3. Delete `range` objects from the cache instance. + * 4. Repeat step 1-3 until the flag `finished` == True. + * The `range` objects are generated by a pseudo random sequence. The seed of the sequence is the thread ID. + * @method func_mixed + * @author Chaos + * @date 2023-11-23 + */ +void *func_mixed(void* arg){ + thread_para_t* para = (thread_para_t*) arg; + hashtable_t* table = para->table; + uint64_t* r_counter = para->r_counter; + uint64_t* i_counter = para->i_counter; + uint64_t* d_counter = para->d_counter; + bool* finished = para->finished; + uint64_t range = para->range; + uint32_t thread_id = para->thread_id; + + +// We keep track of our own local counter for reads, to avoid +// over-burdening the shared atomic counter + size_t reads = 0; + size_t inserts = 0; + size_t removals = 0; + + // We use thread id as seed to generate a reproducible pseudo random sequence. + // The items in the sequence are operated obj IDs. + obj_id_t cur_obj_id = thread_id; + // The loop is broken when the flag `finished` == True. + while(true){ + cur_obj_id = thread_id; + inserts = 0; + removals = 0; + for(uint64_t it = 0; it < range; it++){ + cur_obj_id = get_next_rand(cur_obj_id); + cache_obj_t* cur_obj = create_cache_obj_from_obj_id(cur_obj_id); + cache_obj_t* old_item = ht_handle.func_insert_obj(table, cur_obj); + if(old_item) + free_cache_obj(old_item); + inserts++; + } + for(uint64_t rounds = 0; rounds < 10; rounds++){ + cur_obj_id = thread_id; + reads = 0; + for (uint64_t it = 0; it < range; it++){ + cur_obj_id = get_next_rand(cur_obj_id); + ht_handle.func_find_obj(table, cur_obj_id); + reads++; + } + __sync_fetch_and_add(r_counter, reads); + } + for(uint64_t it = 0; it < range; it++){ + cur_obj_id = get_next_rand(cur_obj_id); + cache_obj_t* removed_item = ht_handle.func_delete_obj(table, cur_obj_id); + if(removed_item) + free_cache_obj(removed_item); + removals++; + } + __sync_fetch_and_add(i_counter, inserts); + __sync_fetch_and_add(d_counter, removals); + if(*finished){ + return NULL; + } + } +} + +/** + * [This function parses the command line arguments.] + * @method parse_arg + * @author Chaos + * @date 2023-11-22 + * @param argc [count of input arguments] + * @param argv [input arguments] + * @param description [description of the arguments] + * @param args [optional arguments] + * @param arg_vars [variables to store the argument values] + * @param arg_help [help message for each argument] + * @param arg_num [number of optional arguments] + */ +void parse_arg(int argc, char const **argv, const char *description, + const char *args[], size_t *arg_vars[], const char *arg_help[], + size_t arg_num) { + errno = 0; + for (int i = 0; i < argc; i++) { + for (size_t j = 0; j < arg_num; j++) { + if (strcmp(argv[i], args[j]) == 0) { + if (i == argc - 1) { + ERROR("You must provide a positive integer argument" \ + " after the %s argument\n", args[j]); + } else { + size_t argval = strtoull(argv[i + 1], NULL, 10); + if (errno != 0) { + ERROR("The argument to %s must be a valid size_t\n", args[j]); + } else { + *(arg_vars[j]) = argval; + } + } + } + } + + if (strcmp(argv[i], "--help") == 0) { + printf("%s\n", description); + printf("Arguments:\n"); + for (size_t j = 0; j < arg_num; j++) { + printf("%s\t(default %zu):\t%s\n", args[j], *arg_vars[j], arg_help[j]); + } + exit(0); + } + } +} + +/** + * This function runs a stress test on inserts, finds, and deletes. + * @method StressTest + * @author Chaos + * @date 2023-11-22 + */ +void stress_test() { + // Initialize threads; + pthread_t insert_threads[g_thread_num]; + pthread_t read_threads[g_thread_num]; + pthread_t remove_threads[g_thread_num]; + + int rc; + uint64_t num_inserts = 0; + uint64_t num_removals = 0; + uint64_t num_reads = 0; + bool finished = false; + + // Based on the hashtable type, we assign the function pointers to the function handles. + // If hashtable type is cChainedHashTable. + if(g_ht_type == 0){ + ht_handle.func_find_obj = concurrent_chained_hashtable_find_obj_id; + ht_handle.func_insert_obj = concurrent_chained_hashtable_insert_obj; + ht_handle.func_delete_obj = concurrent_chained_hashtable_delete_obj_id; + ht_handle.func_create_hashtable = create_concurrent_chained_hashtable; + ht_handle.func_free_hashtable = free_concurrent_chained_hashtable; + } + // If hashtable type is chainedHashTableV2. + // else if(g_ht_type == 1){ + // ht_handle.func_find_obj = chained_hashtable_find_obj_id_v2; + // ht_handle.func_insert_obj = chained_hashtable_insert_obj_v2; + // ht_handle.func_delete_obj = chained_hashtable_delete_obj_id_v2; + // ht_handle.func_create_hashtable = create_chained_hashtable_v2; + // ht_handle.func_free_hashtable = free_chained_hashtable_v2; + // } + + // If hashtable type is invalid. + else{ + ERROR("ERROR: Invalid hashtable type\n"); + } + + // Creates a hashtable + hashtable_t *table = ht_handle.func_create_hashtable(g_power); + + // If the user select the first test. + if(g_test_type == 0){ + // Spawns insert threads + for (size_t i = 0; i < g_thread_num; i++) { + thread_para_t para = { + .table = table, + .i_counter = &num_inserts, + .range = g_numkeys / g_thread_num, + .thread_id = i, + }; + rc = pthread_create(&insert_threads[i], NULL, func_insert, (void *)¶); + if(rc){ + ERROR("ERROR; return code from pthread_create() is %d\n", rc); + } + } + + // Joins insert threads + for (size_t i = 0; i < g_thread_num; i++) { + rc = pthread_join(insert_threads[i], NULL); + if(rc){ + ERROR("ERROR; return code from pthread_join() is %d\n", rc); + } + } + + // Spawns read threads + for(size_t i = 0; i < g_thread_num; i++){ + thread_para_t para = { + .table = table, + .r_counter = &num_reads, + .finished = &finished, + .range = g_numkeys / g_thread_num, + .thread_id = i, + }; + rc = pthread_create(&read_threads[i], NULL, func_read, (void *)¶); + if(rc){ + ERROR("ERROR; return code from pthread_create() is %d\n", rc); + } + } + + // Sleeps before ending the threads + sleep(g_test_len); + finished = true; + // Joins read threads + for (size_t i = 0; i < g_thread_num; i++) { + rc = pthread_join(read_threads[i], NULL); + if(rc){ + ERROR("ERROR; return code from pthread_join() is %d\n", rc); + } + } + + // Spawns remove threads + for(size_t i = 0; i < g_thread_num; i++){ + thread_para_t para = { + .table = table, + .d_counter = &num_removals, + .range = g_numkeys / g_thread_num, + .thread_id = i, + }; + rc = pthread_create(&remove_threads[i], NULL, func_remove, (void *)¶); + if(rc){ + ERROR("ERROR; return code from pthread_create() is %d\n", rc); + } + } + + // Joins delete threads + for (size_t i = 0; i < g_thread_num; i++) { + rc = pthread_join(remove_threads[i], NULL); + if(rc){ + ERROR("ERROR; return code from pthread_join() is %d\n", rc); + } + } + } + // If the user select the second test -- mixed test. + else if(g_test_type == 1){ + uint64_t num_inserts_last_sec = 0; + uint64_t num_removals_last_sec = 0; + uint64_t num_reads_last_sec = 0; + + // Spawns mixed threads + for(size_t i = 0; i < g_thread_num; i++){ + thread_para_t para = { + .table = table, + .r_counter = &num_reads, + .i_counter = &num_inserts, + .d_counter = &num_removals, + .finished = &finished, + .range = g_numkeys / g_thread_num, + .thread_id = i, + }; + rc = pthread_create(&read_threads[i], NULL, func_mixed, (void *)¶); + if(rc){ + ERROR("ERROR; return code from pthread_create() is %d\n", rc); + } + } + printf("----------Throughput MOPS----------\n"); + printf("Seconds\tInsert\tRead\tDelete\tTotal\n"); + + // Prints the throughput every second + for(int i = 0; i < g_test_len; i++){ + sleep(1); + uint64_t num_inserts_this_sec = num_inserts - num_inserts_last_sec; + uint64_t num_reads_this_sec = num_reads - num_reads_last_sec; + uint64_t num_removals_this_sec = num_removals - num_removals_last_sec; + printf("%d\t%.2lf\t%.2lf\t%.2lf\t%.2lf\n", i+1, num_inserts_this_sec/1000000.0, num_reads_this_sec/1000000.0, num_removals_this_sec/1000000.0, (num_inserts_this_sec + num_reads_this_sec + num_removals_this_sec)/1000000.0); + num_inserts_last_sec = num_inserts; + num_reads_last_sec = num_reads; + num_removals_last_sec = num_removals; + } + // Ends the threads. The threads will end when the flag `finished` == True. + finished = true; + // Joins threads + for (size_t i = 0; i < g_thread_num; i++) { + rc = pthread_join(read_threads[i], NULL); + if(rc){ + ERROR("ERROR; return code from pthread_join() is %d\n", rc); + } + } + } + // If the user select an invalid test type. + else{ + ERROR("Invalid test type: %lu. It should be 0 or 1.\n", g_test_type); + } + // Frees the hashtable + ht_handle.func_free_hashtable(table); + + printf("----------Results----------\n"); + printf("Number of inserts:\t%lu\n", num_inserts); + printf("Number of reads:\t%lu\n", num_reads); + printf("Number of removals:\t%lu\n", num_removals); + printf("Total throughput:\t%.2lf MOPS\n", (num_inserts + num_reads + num_removals)/g_test_len/1000000.0); +} + + +int main(int argc, char const *argv[]) +{ + const char *args[] = {"--power", "--thread-num", "--time", "--test-type", "--ht-type"}; + size_t *arg_vars[] = {&g_power, &g_thread_num, &g_test_len, &g_test_type, &g_ht_type}; + const char *arg_help[] = { + "The number of keys to size the table with, expressed as a power of 2", + "The number of threads to spawn for each type of operation", + "The number of seconds to run the test for lookup", + "The type of test. \n\ + 0: Three tests: Insert, Read, and Delete. Each test is non-overlapping. \n\ + 1: Mixed. Each thread performs insert, read, and delete in turn. The three operations are overlapping between different threads.", + "The type of hashtable. \n\ + 0: cChainedHashTable; \n\ + 1:chainedHashTableV2"}; + parse_arg(argc, argv, "Runs a stress test on concurrent hashtables for inserts, finds, and deletes.", + args, arg_vars, arg_help, sizeof(args) / sizeof(const char *)); + g_numkeys = 1 << g_power; + stress_test(); + return 0; +} + +#ifdef __cplusplus +} +#endif \ No newline at end of file diff --git a/libCacheSim/include/config.h b/libCacheSim/include/config.h index a9db61f4..44561c8f 100644 --- a/libCacheSim/include/config.h +++ b/libCacheSim/include/config.h @@ -32,11 +32,11 @@ extern "C" { #endif #ifndef HASHTABLE_TYPE -#define HASHTABLE_TYPE CHAINED_HASHTABLEV2 +#define HASHTABLE_TYPE CONCURRENT_CHAINED_HASHTABLE #endif #ifndef HASH_POWER_DEFAULT -#define HASH_POWER_DEFAULT 23 +#define HASH_POWER_DEFAULT 20 #endif #ifndef CHAINED_HASHTABLE_EXPAND_THRESHOLD diff --git a/libCacheSim/include/libCacheSim/cache.h b/libCacheSim/include/libCacheSim/cache.h index bfe43f45..d7e3979e 100644 --- a/libCacheSim/include/libCacheSim/cache.h +++ b/libCacheSim/include/libCacheSim/cache.h @@ -81,6 +81,7 @@ typedef struct { int64_t n_obj; int64_t occupied_byte; int64_t cache_size; + double runtime; /* current trace time, used to determine obj expiration */ int64_t curr_rtime; @@ -307,9 +308,10 @@ static inline void print_cache_stat(const cache_t *cache) { printf( "%s cache size %ld, occupied size %ld, n_req %ld, n_obj %ld, default TTL " "%ld, per_obj_metadata_size %d\n", - cache->cache_name, cache->cache_size, cache->get_occupied_byte(cache), - cache->n_req, cache->get_n_obj(cache), cache->default_ttl, - cache->obj_md_size); + cache->cache_name, (long)cache->cache_size, + (long)cache->get_occupied_byte(cache), (long)cache->n_req, + (long)cache->get_n_obj(cache), (long)cache->default_ttl, + (int)cache->obj_md_size); } /** diff --git a/libCacheSim/include/libCacheSim/cacheObj.h b/libCacheSim/include/libCacheSim/cacheObj.h index 86a378fd..1d174561 100644 --- a/libCacheSim/include/libCacheSim/cacheObj.h +++ b/libCacheSim/include/libCacheSim/cacheObj.h @@ -11,6 +11,7 @@ #include "../config.h" #include "mem.h" +#include "../../utils/include/mymutex.h" #ifdef __cplusplus extern "C" { @@ -98,6 +99,10 @@ typedef struct { int lru_id; } SLRU_obj_metadata_t; +typedef struct { + int64_t last_access_vtime; +} RandomTwo_obj_metadata_t; + typedef struct { int64_t last_access_vtime; int32_t freq; @@ -126,18 +131,13 @@ typedef struct { int32_t freq; } __attribute__((packed)) misc_metadata_t; -typedef struct { - int32_t clock_id; - int32_t freq; - int32_t n_miss; - bool visited; - bool new_obj; -} __attribute__((packed)) myclock_obj_params_t; - // ############################## cache obj ################################### struct cache_obj; + +#define FLAG_IN_CACHE_BIT 0b001 + typedef struct cache_obj { - struct cache_obj *hash_next; + struct cache_obj *hash_next; // The first field to make an aligned pointer obj_id_t obj_id; uint32_t obj_size; struct { @@ -147,6 +147,8 @@ typedef struct cache_obj { #ifdef SUPPORT_TTL uint32_t exp_time; #endif + uint8_t flags; + /* age is defined as the time since the object entered the cache */ #if defined(TRACK_EVICTION_V_AGE) || \ defined(TRACK_DEMOTION) || defined(TRACK_CREATE_TIME) @@ -165,6 +167,7 @@ typedef struct cache_obj { SR_LRU_obj_metadata_t SR_LRU; CR_LFU_obj_metadata_t CR_LFU; Hyperbolic_obj_metadata_t hyperbolic; + RandomTwo_obj_metadata_t RandomTwo; Belady_obj_metadata_t Belady; FIFO_Merge_obj_metadata_t FIFO_Merge; FIFO_Reinsertion_obj_metadata_t FIFO_Reinsertion; @@ -173,7 +176,6 @@ typedef struct cache_obj { QDLP_obj_metadata_t QDLP; LIRS_obj_metadata_t LIRS; S3FIFO_obj_metadata_t S3FIFO; - myclock_obj_params_t myclock; Sieve_obj_params_t sieve; #if defined(ENABLE_GLCACHE) && ENABLE_GLCACHE == 1 @@ -182,7 +184,32 @@ typedef struct cache_obj { }; } __attribute__((packed)) cache_obj_t; +/** + * Set/get flags of the cache_obj atomically + */ + +inline bool cache_obj_in_cache(cache_obj_t *obj) { + return obj->flags & FLAG_IN_CACHE_BIT; +} + +inline void cache_obj_set_in_cache(cache_obj_t *obj, bool in_cache) { + if (in_cache) { + fetch_or(&obj->flags, FLAG_IN_CACHE_BIT); + } else { + fetch_and(&obj->flags, ~FLAG_IN_CACHE_BIT); + } +} + + struct request; + +/** + * verify the fingerprint of current cache_obj + * @param cache_obj + * @param bool [true if the fingerprint is correct] + */ +bool verify_cache_obj_fingerprint(const cache_obj_t *cache_obj); + /** * copy the cache_obj to req_dest * @param req_dest @@ -206,6 +233,13 @@ void copy_request_to_cache_obj(cache_obj_t *cache_obj, */ cache_obj_t *create_cache_obj_from_request(const struct request *req); +/** + * create a empty cache_obj from obj_id + * @param obj_id + * @return + */ +cache_obj_t *create_cache_obj_from_obj_id(const obj_id_t obj_id); + /** * the cache_obj has built-in a doubly list, in the case the list is used as * a singly list (list_prev is not used, next is used) @@ -269,6 +303,12 @@ void prepend_obj_to_head(cache_obj_t **head, cache_obj_t **tail, */ void append_obj_to_tail(cache_obj_t **head, cache_obj_t **tail, cache_obj_t *cache_obj); +/** + * free the the doubly linked list + * @param head + * @param tail + */ +void free_list(cache_obj_t **head, cache_obj_t **tail); /** * free cache_obj, this is only used when the cache_obj is explicitly * malloced diff --git a/libCacheSim/include/libCacheSim/dist.h b/libCacheSim/include/libCacheSim/dist.h index 82c090ab..9b1daf9c 100644 --- a/libCacheSim/include/libCacheSim/dist.h +++ b/libCacheSim/include/libCacheSim/dist.h @@ -19,7 +19,7 @@ typedef enum { FUTURE_STACK_DIST, } dist_type_e; -static char *dist_type_str[] = { +static char *g_dist_type_name[] = { "DIST_SINCE_LAST_ACCESS", "DIST_SINCE_FIRST_ACCESS", "STACK_DIST", diff --git a/libCacheSim/include/libCacheSim/enum.h b/libCacheSim/include/libCacheSim/enum.h index c05c9b0a..fbc9900b 100644 --- a/libCacheSim/include/libCacheSim/enum.h +++ b/libCacheSim/include/libCacheSim/enum.h @@ -65,7 +65,7 @@ typedef enum { UNKNOWN_TRACE, } __attribute__((__packed__)) trace_type_e; -static char *trace_type_str[UNKNOWN_TRACE + 2] = { +static char *g_trace_type_name[UNKNOWN_TRACE + 2] = { "CSV_TRACE", "BIN_TRACE", "PLAIN_TXT_TRACE", diff --git a/libCacheSim/include/libCacheSim/evictionAlgo.h b/libCacheSim/include/libCacheSim/evictionAlgo.h index 2fd35c4e..e87d0e76 100644 --- a/libCacheSim/include/libCacheSim/evictionAlgo.h +++ b/libCacheSim/include/libCacheSim/evictionAlgo.h @@ -1,6 +1,7 @@ #pragma once #include "cache.h" +#include #ifdef __cplusplus extern "C" { @@ -15,6 +16,7 @@ typedef struct { typedef struct { cache_obj_t *q_head; cache_obj_t *q_tail; + pthread_mutex_t mutex_; } LRU_params_t; /* used by LFU related */ @@ -23,6 +25,7 @@ typedef struct freq_node { cache_obj_t *first_obj; cache_obj_t *last_obj; uint32_t n_obj; + pthread_mutex_t mutex_; } freq_node_t; typedef struct { @@ -97,6 +100,9 @@ cache_t *MRU_init(const common_cache_params_t ccache_params, cache_t *Random_init(const common_cache_params_t ccache_params, const char *cache_specific_params); +cache_t *RandomTwo_init(const common_cache_params_t ccache_params, + const char *cache_specific_params); + cache_t *SLRU_init(const common_cache_params_t ccache_params, const char *cache_specific_params); @@ -129,6 +135,7 @@ cache_t *flashProb_init(const common_cache_params_t ccache_params, cache_t *LRU_Prob_init(const common_cache_params_t ccache_params, const char *cache_specific_params); + cache_t *SFIFOv0_init(const common_cache_params_t ccache_params, const char *cache_specific_params); @@ -177,9 +184,6 @@ cache_t *LP_ARC_init(const common_cache_params_t ccache_params, cache_t *LP_TwoQ_init(const common_cache_params_t ccache_params, const char *cache_specific_params); -cache_t *MyClock_init(const common_cache_params_t ccache_params, - const char *cache_specific_params); - cache_t *QDLPv0_init(const common_cache_params_t ccache_params, const char *cache_specific_params); diff --git a/libCacheSim/include/libCacheSim/prefetchAlgo.h b/libCacheSim/include/libCacheSim/prefetchAlgo.h index 5a0c3c78..89b25f9c 100644 --- a/libCacheSim/include/libCacheSim/prefetchAlgo.h +++ b/libCacheSim/include/libCacheSim/prefetchAlgo.h @@ -1,6 +1,8 @@ #ifndef PREFETCHINGALGO_H #define PREFETCHINGALGO_H +#include + #include "cache.h" #include "request.h" @@ -13,6 +15,7 @@ typedef struct prefetcher *(*prefetcher_create_func_ptr)(const char *); typedef void (*prefetcher_prefetch_func_ptr)(cache_t *, const request_t *); typedef void (*prefetcher_handle_find_func_ptr)(cache_t *, const request_t *, bool); +typedef void (*prefetcher_handle_insert_func_ptr)(cache_t *, const request_t *); typedef void (*prefetcher_handle_evict_func_ptr)(cache_t *, const request_t *); typedef void (*prefetcher_free_func_ptr)(struct prefetcher *); typedef struct prefetcher *(*prefetcher_clone_func_ptr)(struct prefetcher *, @@ -23,6 +26,7 @@ typedef struct prefetcher { void *init_params; prefetcher_prefetch_func_ptr prefetch; prefetcher_handle_find_func_ptr handle_find; + prefetcher_handle_insert_func_ptr handle_insert; prefetcher_handle_evict_func_ptr handle_evict; prefetcher_free_func_ptr free; prefetcher_clone_func_ptr clone; @@ -30,6 +34,8 @@ typedef struct prefetcher { prefetcher_t *create_Mithril_prefetcher(const char *init_paramsm, uint64_t cache_size); +prefetcher_t *create_OBL_prefetcher(const char *init_paramsm, uint64_t cache_size); +prefetcher_t *create_PG_prefetcher(const char *init_paramsm, uint64_t cache_size); static inline prefetcher_t *create_prefetcher(const char *prefetching_algo, const char *prefetching_params, @@ -37,6 +43,10 @@ static inline prefetcher_t *create_prefetcher(const char *prefetching_algo, prefetcher_t *prefetcher = NULL; if (strcasecmp(prefetching_algo, "Mithril") == 0) { prefetcher = create_Mithril_prefetcher(prefetching_params, cache_size); + } else if (strcasecmp(prefetching_algo, "OBL") == 0) { + prefetcher = create_OBL_prefetcher(prefetching_params, cache_size); + } else if (strcasecmp(prefetching_algo, "PG") == 0) { + prefetcher = create_PG_prefetcher(prefetching_params, cache_size); } else { ERROR("prefetching algo %s not supported\n", prefetching_algo); } diff --git a/libCacheSim/include/libCacheSim/prefetchAlgo/OBL.h b/libCacheSim/include/libCacheSim/prefetchAlgo/OBL.h new file mode 100644 index 00000000..b926cf71 --- /dev/null +++ b/libCacheSim/include/libCacheSim/prefetchAlgo/OBL.h @@ -0,0 +1,29 @@ +#ifndef OBL_H +#define OBL_H +#include +#include +#include +#include +#include +#include + +#include "../cache.h" + +typedef struct OBL_params { + int32_t block_size; + bool do_prefetch; + + uint32_t curr_idx; // current index in the prev_access_block + int32_t sequential_confidence_k; // number of prev sequential accesses to be + // considered as a sequential access + obj_id_t* prev_access_block; // prev k accessed +} OBL_params_t; + +typedef struct OBL_init_params { + int32_t block_size; + int32_t sequential_confidence_k; +} OBL_init_params_t; + +#define DO_PREFETCH(id) + +#endif \ No newline at end of file diff --git a/libCacheSim/include/libCacheSim/prefetchAlgo/PG.h b/libCacheSim/include/libCacheSim/prefetchAlgo/PG.h new file mode 100644 index 00000000..98adbc2e --- /dev/null +++ b/libCacheSim/include/libCacheSim/prefetchAlgo/PG.h @@ -0,0 +1,81 @@ +// +// PG.h +// mimircache +// +// Created by Juncheng on 6/2/16. +// Copyright © 2016 Juncheng. All rights reserved. +// +// Modified by Zhelong on 2/21/24. + +/** since this is sequence based prefetching, we will use gint64 for block + * number **/ + +#ifndef PG_h +#define PG_h + +#include + +#include "../../../dataStructure/pqueue.h" +#include "../../../traceReader/generalReader/readerInternal.h" +#include "../cache.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* check params vaild */ +#define check_params(params) \ + (assert(params->lookahead_range > 0 && params->lookahead_range <= 100 && params->max_metadata_size > 0 && \ + params->max_metadata_size < 1 && params->prefetch_threshold > 0 && params->prefetch_threshold < 1 && \ + params->block_size > 0)) + +#define PG_getPage(x) ((struct PG_Page*)g_hashtable_lookup()) + +// #define get_Nth_past_request_c(PG_params, n) +// ((char**)((PG_params)->past_requests))[(n)] +#define get_Nth_past_request_c(PG_params, n, des) strcpy((des), ((char**)((PG_params)->past_requests))[(n)]) + +#define get_Nth_past_request_l(PG_params, n) ((guint64*)((PG_params)->past_requests))[(n)] +#define set_Nth_past_request_c(PG_params, n, v) strcpy(((char**)((PG_params)->past_requests))[(n)], (char*)(v)) +#define set_Nth_past_request_l(PG_params, n, v) ((guint64*)((PG_params)->past_requests))[(n)] = (v) + +typedef struct { + uint8_t lookahead_range; + uint block_size; // In the PG algorithm, the existence of block_size, like Mithril, is to correct the maximum + // metadata size while ignoring object size + uint64_t cur_metadata_size; + uint64_t max_metadata_size; // unit byte + double prefetch_threshold; + + gboolean stop_recording; + + GHashTable* graph; // key -> graphNode_t + GHashTable* prefetched; + void* past_requests; // past requests, using array instead of queue to avoid + // frequent memory allocation + + uint32_t past_request_pointer; + gint64 num_of_prefetch; + gint64 num_of_hit; + + GHashTable* cache_size_map; // key -> size +} PG_params_t; + +typedef struct { + GHashTable* graph; // key -> pq_node_t + pqueue_t* pq; + uint64_t total_count; +} graphNode_t; + +typedef struct { + uint8_t lookahead_range; + uint block_size; + double max_metadata_size; + double prefetch_threshold; +} PG_init_params_t; + +#ifdef __cplusplus +} +#endif + +#endif /* PG_H */ diff --git a/libCacheSim/include/libCacheSim/reader.h b/libCacheSim/include/libCacheSim/reader.h index 8a9305ff..644fb667 100644 --- a/libCacheSim/include/libCacheSim/reader.h +++ b/libCacheSim/include/libCacheSim/reader.h @@ -98,7 +98,7 @@ typedef struct reader { /************* used by binary trace *************/ /* mmap the file, this should not change during runtime */ char *mapped_file; - uint64_t mmap_offset; + size_t mmap_offset; struct zstd_reader *zstd_reader_p; bool is_zstd_file; /* the size of one request in binary trace */ @@ -117,8 +117,8 @@ typedef struct reader { /* if true, ignore the obj_size in the trace, and use size one */ bool ignore_obj_size; - /* this is used when - * a) the reader splits a large req into multiple chunked requests + /* this is used when + * a) the reader splits a large req into multiple chunked requests * b) the trace file uses a count field */ int n_req_left; int64_t last_req_clock_time; @@ -180,8 +180,9 @@ reader_t *setup_reader(const char *trace_path, trace_type_e trace_type, const reader_init_param_t *reader_init_param); /* this is the same function as setup_reader */ -static inline reader_t *open_trace(const char *path, const trace_type_e type, - const reader_init_param_t *reader_init_param) { +static inline reader_t *open_trace( + const char *path, const trace_type_e type, + const reader_init_param_t *reader_init_param) { return setup_reader(path, type, reader_init_param); } @@ -271,12 +272,13 @@ static inline void print_reader(reader_t *reader) { "%p, line_buf_size: %zu, csv_delimiter: %c, csv_has_header: %d, " "obj_id_is_num: %d, ignore_size_zero_req: %d, ignore_obj_size: %d, " "n_req_left: %d, last_req_clock_time: %ld\n", - trace_type_str[reader->trace_type], reader->trace_path, - reader->trace_start_offset, reader->mmap_offset, reader->is_zstd_file, - reader->item_size, reader->file, reader->line_buf, reader->line_buf_size, - reader->csv_delimiter, reader->csv_has_header, reader->obj_id_is_num, - reader->ignore_size_zero_req, reader->ignore_obj_size, - reader->n_req_left, reader->last_req_clock_time); + g_trace_type_name[reader->trace_type], reader->trace_path, + reader->trace_start_offset, (long)reader->mmap_offset, + reader->is_zstd_file, reader->item_size, reader->file, reader->line_buf, + reader->line_buf_size, reader->csv_delimiter, reader->csv_has_header, + reader->obj_id_is_num, reader->ignore_size_zero_req, + reader->ignore_obj_size, reader->n_req_left, + (long)reader->last_req_clock_time); } #ifdef __cplusplus diff --git a/libCacheSim/include/libCacheSim/request.h b/libCacheSim/include/libCacheSim/request.h index 33e9d944..23cc7298 100644 --- a/libCacheSim/include/libCacheSim/request.h +++ b/libCacheSim/include/libCacheSim/request.h @@ -20,7 +20,7 @@ extern "C" { /* need to optimize this for CPU cacheline */ typedef struct request { int64_t clock_time; /* use uint64_t because vscsi uses microsec timestamp */ - uint64_t hv; /* hash value, used when offloading hash to reader */ + uint64_t hv; /* hash value, used when offloading hash to reader */ obj_id_t obj_id; int64_t obj_size; int32_t ttl; @@ -53,10 +53,10 @@ typedef struct request { /* used in trace analysis */ int64_t vtime_since_last_access; int64_t rtime_since_last_access; - int64_t prev_size; /* prev size */ + int64_t prev_size; /* prev size */ int32_t create_rtime; - bool compulsory_miss; /* use this field only when it is set */ - bool overwrite; // this request overwrites a previous object + bool compulsory_miss; /* use this field only when it is set */ + bool overwrite; // this request overwrites a previous object bool first_seen_in_window; /* the first time see in the time window */ /* used in trace analysis */ @@ -68,7 +68,7 @@ typedef struct request { * allocate a new request_t struct and fill in necessary field * @return */ -static inline request_t *new_request() { +static inline request_t *new_request(void) { request_t *req = my_malloc(request_t); memset(req, 0, sizeof(request_t)); req->obj_size = 1; @@ -87,7 +87,7 @@ static inline request_t *new_request() { * @param req_dest * @param req_src */ -static inline void copy_request(request_t *req_dest, request_t *req_src) { +static inline void copy_request(request_t *req_dest, const request_t *req_src) { memcpy(req_dest, req_src, sizeof(request_t)); } @@ -96,7 +96,7 @@ static inline void copy_request(request_t *req_dest, request_t *req_src) { * @param req * @return */ -static inline request_t *clone_request(request_t *req) { +static inline request_t *clone_request(const request_t *req) { request_t *req_new = my_malloc(request_t); copy_request(req_new, req); return req_new; diff --git a/libCacheSim/include/libCacheSim/sampling.h b/libCacheSim/include/libCacheSim/sampling.h index 6c679f9f..d54c6b31 100644 --- a/libCacheSim/include/libCacheSim/sampling.h +++ b/libCacheSim/include/libCacheSim/sampling.h @@ -13,7 +13,7 @@ typedef bool (*trace_sampling_func)(struct sampler *sampler, request_t *req); typedef struct sampler *(*clone_sampler_func)(const struct sampler *sampler); -typedef struct sampler *(*free_sampler_func)(struct sampler *sampler); +typedef void (*free_sampler_func)(struct sampler *sampler); enum sampler_type { SPATIAL_SAMPLER, diff --git a/libCacheSim/profiler/dist.c b/libCacheSim/profiler/dist.c index 913ad29f..1309c1dd 100644 --- a/libCacheSim/profiler/dist.c +++ b/libCacheSim/profiler/dist.c @@ -134,7 +134,7 @@ int32_t *get_stack_dist(reader_t *reader, const dist_type_e dist_type, stack_dist = get_stack_dist_add_req(req, &splay_tree, hash_table, curr_ts, &last_access_ts); if (stack_dist > (int64_t)UINT32_MAX) { - ERROR("stack distance %ld is larger than UINT32_MAX\n", stack_dist); + ERROR("stack distance %ld is larger than UINT32_MAX\n", (long)stack_dist); abort(); } if (dist_type == STACK_DIST) { @@ -159,7 +159,8 @@ int32_t *get_stack_dist(reader_t *reader, const dist_type_e dist_type, return stack_dist_array; } -int32_t *get_access_dist(reader_t *reader, const dist_type_e dist_type, int64_t *array_size) { +int32_t *get_access_dist(reader_t *reader, const dist_type_e dist_type, + int64_t *array_size) { int64_t curr_ts = 0; int64_t dist = 0; request_t *req = new_request(); @@ -174,7 +175,7 @@ int32_t *get_access_dist(reader_t *reader, const dist_type_e dist_type, int64_t while (req->valid) { dist = get_access_dist_add_req(req, hash_table, curr_ts, dist_type); if (dist > (int64_t)UINT32_MAX) { - ERROR("access distance %ld is larger than UINT32_MAX\n", dist); + ERROR("access distance %ld is larger than UINT32_MAX\n", (long)dist); abort(); } @@ -195,7 +196,7 @@ void save_dist(reader_t *const reader, const int32_t *dist_array, int64_t array_size, const char *const ofilepath, const dist_type_e dist_type) { char *file_path = (char *)malloc(strlen(ofilepath) + 128); - sprintf(file_path, "%s.%s", ofilepath, dist_type_str[dist_type]); + sprintf(file_path, "%s.%s", ofilepath, g_dist_type_name[dist_type]); FILE *file = fopen(file_path, "wb"); fwrite(dist_array, sizeof(int32_t), get_num_of_req(reader), file); fclose(file); @@ -203,10 +204,10 @@ void save_dist(reader_t *const reader, const int32_t *dist_array, } void save_dist_txt(reader_t *const reader, const int32_t *dist_array, - int64_t array_size, const char *const ofilepath, - const dist_type_e dist_type) { + int64_t array_size, const char *const ofilepath, + const dist_type_e dist_type) { char *file_path = (char *)malloc(strlen(ofilepath) + 128); - sprintf(file_path, "%s.%s.txt", ofilepath, dist_type_str[dist_type]); + sprintf(file_path, "%s.%s.txt", ofilepath, g_dist_type_name[dist_type]); FILE *file = fopen(file_path, "w"); for (int i = 0; i < array_size; i++) { fprintf(file, "%d\n", dist_array[i]); @@ -254,16 +255,16 @@ void _write_dist_cnt(gpointer key, gpointer value, gpointer user_data) { int64_t dist = (int64_t)GPOINTER_TO_SIZE(key); int64_t cnt = (int64_t)GPOINTER_TO_SIZE(value); FILE *file = (FILE *)user_data; - fprintf(file, "%ld:%ld, ", dist, cnt); + fprintf(file, "%ld:%ld, ", (long)dist, (long)cnt); } void save_dist_as_cnt_txt(reader_t *const reader, const int32_t *dist_array, - const int64_t array_size, const char *const ofilepath, - const dist_type_e dist_type) { + const int64_t array_size, const char *const ofilepath, + const dist_type_e dist_type) { assert(get_num_of_req(reader) == array_size); char *file_path = (char *)malloc(strlen(ofilepath) + 128); - sprintf(file_path, "%s.%s.cnt", ofilepath, dist_type_str[dist_type]); + sprintf(file_path, "%s.%s.cnt", ofilepath, g_dist_type_name[dist_type]); FILE *file = fopen(file_path, "w"); GHashTable *hash_table = diff --git a/libCacheSim/profiler/simulator.c b/libCacheSim/profiler/simulator.c index 91471bb0..06de2259 100644 --- a/libCacheSim/profiler/simulator.c +++ b/libCacheSim/profiler/simulator.c @@ -19,7 +19,8 @@ extern "C" { #include "../include/libCacheSim/plugin.h" #include "../utils/include/myprint.h" #include "../utils/include/mystr.h" - +#include "../utils/include/mysys.h" +#include "../utils/include/mymutex.h" typedef struct simulator_multithreading_params { reader_t *reader; ssize_t n_caches; @@ -28,7 +29,7 @@ typedef struct simulator_multithreading_params { reader_t *warmup_reader; int warmup_sec; /* num of seconds of requests used for warming up cache */ cache_stat_t *result; - GMutex mtx; /* prevent simultaneous write to progress */ + // GMutex mtx; /* prevent simultaneous write to progress */ gint *progress; gpointer other_data; bool free_cache_when_finish; @@ -36,8 +37,10 @@ typedef struct simulator_multithreading_params { static void _simulate(gpointer data, gpointer user_data) { sim_mt_params_t *params = (sim_mt_params_t *)user_data; - int idx = GPOINTER_TO_UINT(data) - 1; + int idx = (GPOINTER_TO_UINT(data) - 1) % params->n_caches; + int t_idx = GPOINTER_TO_UINT(data) / params->n_caches; set_rand_seed(0); + // INFO("Start simulation. Thread %d in Cache %s (index=%d)\n", t_idx, params->caches[idx]->cache_name, idx); cache_stat_t *result = params->result; reader_t *cloned_reader = clone_reader(params->reader); @@ -50,10 +53,12 @@ static void _simulate(gpointer data, gpointer user_data) { if (params->warmup_reader) { reader_t *warmup_cloned_reader = clone_reader(params->warmup_reader); read_one_req(warmup_cloned_reader, req); + req->obj_id = req->obj_id % (uint64_t)UINT32_MAX + t_idx * 1000000000ULL; while (req->valid) { - bool ck = local_cache->get(local_cache, req); - result[idx].n_warmup_req += 1; + local_cache->get(local_cache, req); + fetch_add(&result[idx].n_warmup_req, 1); read_one_req(warmup_cloned_reader, req); + req->obj_id = req->obj_id % (uint64_t)UINT32_MAX + t_idx * 1000000000ULL; } close_reader(warmup_cloned_reader); INFO("cache %s (size %" PRIu64 @@ -64,6 +69,7 @@ static void _simulate(gpointer data, gpointer user_data) { } read_one_req(cloned_reader, req); + req->obj_id = req->obj_id % (uint64_t)UINT32_MAX + t_idx * 1000000000ULL; int64_t start_ts = (int64_t)req->clock_time; /* using warmup_frac or warmup_sec of requests from reader to warm up */ @@ -72,11 +78,12 @@ static void _simulate(gpointer data, gpointer user_data) { while (req->valid && (n_warmup < params->n_warmup_req || req->clock_time - start_ts < params->warmup_sec)) { req->clock_time -= start_ts; - bool ck = local_cache->get(local_cache, req); + local_cache->get(local_cache, req); n_warmup += 1; read_one_req(cloned_reader, req); + req->obj_id = req->obj_id % (uint64_t)UINT32_MAX + t_idx * 1000000000ULL; } - result[idx].n_warmup_req += n_warmup; + fetch_add(&result[idx].n_warmup_req, n_warmup); INFO("cache %s (size %" PRIu64 ") finishes warm up using " "with %" PRIu64 " requests, %.2lf hour trace time\n", @@ -84,18 +91,27 @@ static void _simulate(gpointer data, gpointer user_data) { (double)(req->clock_time - start_ts) / 3600.0); } + double start_time = gettime(); while (req->valid) { - result[idx].n_req++; - result[idx].n_req_byte += req->obj_size; + if(result[idx].runtime != 0){ + free_request(req); + close_reader(cloned_reader); + return; + } + fetch_add(&result[idx].n_req, 1); + fetch_add(&result[idx].n_req_byte, req->obj_size); req->clock_time -= start_ts; if (local_cache->get(local_cache, req) == false) { - result[idx].n_miss++; - result[idx].n_miss_byte += req->obj_size; + fetch_add(&result[idx].n_miss, 1); + fetch_add(&result[idx].n_miss_byte, req->obj_size); } read_one_req(cloned_reader, req); + req->obj_id = req->obj_id % (uint64_t)UINT32_MAX + t_idx * 1000000000ULL; } + result[idx].runtime = gettime() - start_time; + /* disabled due to ARC and LeCaR use ghost entries in the hash table */ #if defined(SUPPORT_TTL) && defined(ENABLE_SCAN) /* get expiration information */ @@ -124,14 +140,16 @@ static void _simulate(gpointer data, gpointer user_data) { CACHE_NAME_ARRAY_LEN); // report progress - g_mutex_lock(&(params->mtx)); - (*(params->progress))++; - g_mutex_unlock(&(params->mtx)); + // g_mutex_lock(&(params->mtx)); + // (*(params->progress))++; + // g_mutex_unlock(&(params->mtx)); + + __atomic_fetch_add(params->progress, 1, __ATOMIC_ACQ_REL); // clean up - if (params->free_cache_when_finish) { - local_cache->cache_free(local_cache); - } + // if (params->free_cache_when_finish) { + // local_cache->cache_free(local_cache); + // } free_request(req); close_reader(cloned_reader); } @@ -191,7 +209,7 @@ cache_stat_t *simulate_at_multi_sizes(reader_t *reader, const cache_t *cache, params->result = result; params->free_cache_when_finish = true; params->progress = &progress; - g_mutex_init(&(params->mtx)); + // g_mutex_init(&(params->mtx)); // build the thread pool GThreadPool *gthread_pool = g_thread_pool_new( @@ -225,7 +243,7 @@ cache_stat_t *simulate_at_multi_sizes(reader_t *reader, const cache_t *cache, // clean up g_thread_pool_free(gthread_pool, FALSE, TRUE); - g_mutex_clear(&(params->mtx)); + // g_mutex_clear(&(params->mtx)); my_free(sizeof(cache_t *) * num_of_sizes, params->caches); my_free(sizeof(sim_mt_params_t), params); @@ -252,7 +270,7 @@ cache_stat_t *simulate_with_multi_caches(reader_t *reader, cache_t *caches[], int num_of_threads, bool free_cache_when_finish) { assert(num_of_caches > 0); - int i, progress = 0; + int i, j, progress = 0; cache_stat_t *result = my_malloc_n(cache_stat_t, num_of_caches); memset(result, 0, sizeof(cache_stat_t) * num_of_caches); @@ -261,6 +279,7 @@ cache_stat_t *simulate_with_multi_caches(reader_t *reader, cache_t *caches[], sim_mt_params_t *params = my_malloc(sim_mt_params_t); params->reader = reader; params->caches = caches; + params->n_caches = num_of_caches; params->warmup_reader = warmup_reader; params->warmup_sec = warmup_sec; if (warmup_frac > 1e-6) { @@ -272,19 +291,20 @@ cache_stat_t *simulate_with_multi_caches(reader_t *reader, cache_t *caches[], params->result = result; params->free_cache_when_finish = free_cache_when_finish; params->progress = &progress; - g_mutex_init(&(params->mtx)); + // g_mutex_init(&(params->mtx)); // build the thread pool GThreadPool *gthread_pool = g_thread_pool_new( - (GFunc)_simulate, (gpointer)params, num_of_threads, TRUE, NULL); + (GFunc)_simulate, (gpointer)params, num_of_caches * num_of_threads, TRUE, NULL); ASSERT_NOT_NULL(gthread_pool, "cannot create thread pool in simulator\n"); - // start computation - for (i = 1; i < num_of_caches + 1; i++) { + // start computation + for (i = 1; i < num_of_caches + 1; i++){ result[i - 1].cache_size = caches[i - 1]->cache_size; - - ASSERT_TRUE(g_thread_pool_push(gthread_pool, GSIZE_TO_POINTER(i), NULL), + for (j = 0; j < num_of_threads; j++) { + ASSERT_TRUE(g_thread_pool_push(gthread_pool, GSIZE_TO_POINTER(i + j * num_of_caches), NULL), "cannot push data into thread_pool in get_miss_ratio\n"); + } } char start_cache_size[64], end_cache_size[64]; @@ -299,13 +319,20 @@ cache_stat_t *simulate_with_multi_caches(reader_t *reader, cache_t *caches[], num_of_caches, num_of_threads); // wait for all simulations to finish - while (progress < (uint64_t)num_of_caches - 1) { - print_progress((double)progress / (double)(num_of_caches - 1) * 100); + while (progress < (uint64_t)num_of_caches) { + print_progress((double)progress / (double)(num_of_caches) * 100); } // clean up + g_thread_pool_free(gthread_pool, FALSE, TRUE); - g_mutex_clear(&(params->mtx)); + // g_mutex_clear(&(params->mtx)); + for (i = 0; i < num_of_caches; i++){ + if (params->free_cache_when_finish) { + cache_t *local_cache = params->caches[i]; + local_cache->cache_free(local_cache); + } + } my_free(sizeof(sim_mt_params_t), params); // user is responsible for free-ing the result diff --git a/libCacheSim/traceAnalyzer/analyzer.cpp b/libCacheSim/traceAnalyzer/analyzer.cpp index eab48568..e2b83dba 100644 --- a/libCacheSim/traceAnalyzer/analyzer.cpp +++ b/libCacheSim/traceAnalyzer/analyzer.cpp @@ -110,8 +110,9 @@ void traceAnalyzer::TraceAnalyzer::run() { if (curr_time_window_idx != time_to_window_idx(req->clock_time)) { ERROR( "The data is not ordered by time, please sort the trace first!" - "Current time %ld requested object %lu\n", - req->clock_time + start_ts_, req->obj_id); + "Current time %ld requested object %lu, obj size %lu\n", + (long)(req->clock_time + start_ts_), (unsigned long)req->obj_id, + (long)req->obj_size); } DEBUG_ASSERT(curr_time_window_idx == time_to_window_idx(req->clock_time)); diff --git a/libCacheSim/traceReader/CMakeLists.txt b/libCacheSim/traceReader/CMakeLists.txt index 25e9e3d9..7b749902 100644 --- a/libCacheSim/traceReader/CMakeLists.txt +++ b/libCacheSim/traceReader/CMakeLists.txt @@ -10,9 +10,9 @@ set(source sampling/temporal.c ) -if (SUPPORT_ZSTD_TRACE) +if (OPT_SUPPORT_ZSTD_TRACE) set(source ${source} generalReader/zstdReader.c) -endif (SUPPORT_ZSTD_TRACE) +endif (OPT_SUPPORT_ZSTD_TRACE) add_library(traceReader ${source}) diff --git a/libCacheSim/traceReader/customizedReader/binaryUtils.h b/libCacheSim/traceReader/customizedReader/binaryUtils.h index 59b2c5ef..780cabc1 100644 --- a/libCacheSim/traceReader/customizedReader/binaryUtils.h +++ b/libCacheSim/traceReader/customizedReader/binaryUtils.h @@ -4,6 +4,12 @@ #include "../generalReader/zstdReader.h" #endif +#include "../../include/libCacheSim/reader.h" + +#ifdef __cplusplus +extern "C" { +#endif + /* read decompressed data file */ static inline char *_read_bytes(reader_t *reader) { if (reader->mmap_offset >= reader->file_size) { @@ -44,4 +50,8 @@ static inline char *read_bytes(reader_t *reader) { start = _read_bytes(reader); } return start; -} \ No newline at end of file +} + +#ifdef __cplusplus +} +#endif diff --git a/libCacheSim/traceReader/generalReader/binary.c b/libCacheSim/traceReader/generalReader/binary.c index 86a11fab..78571eb6 100644 --- a/libCacheSim/traceReader/generalReader/binary.c +++ b/libCacheSim/traceReader/generalReader/binary.c @@ -56,8 +56,8 @@ int binaryReader_setup(reader_t *const reader) { params->fmt_str = strdup(init_params->binary_fmt_str + 1); const char *fmt_str = params->fmt_str; - uint32_t field_size = 0; - uint32_t curr_offset = 0; + // uint32_t field_size = 0; + // uint32_t curr_offset = 0; params->n_fields = strlen(fmt_str); diff --git a/libCacheSim/traceReader/generalReader/csv.c b/libCacheSim/traceReader/generalReader/csv.c index b872eb3f..acdc133b 100644 --- a/libCacheSim/traceReader/generalReader/csv.c +++ b/libCacheSim/traceReader/generalReader/csv.c @@ -79,7 +79,7 @@ static int read_first_line(const reader_t *reader, char *in_buf, */ static char csv_detect_delimiter(const reader_t *reader) { char first_line[1024] = {0}; - int _buf_size = read_first_line(reader, first_line, 1024); + read_first_line(reader, first_line, 1024); char possible_delims[4] = {'\t', ',', '|', ':'}; int possible_delim_counts[4] = {0, 0, 0, 0}; @@ -168,7 +168,7 @@ bool check_delimiter(const reader_t *reader, char delimiter) { bool is_delimiter_correct = true; size_t n = 0; - ssize_t read_size = getline(&buf, &n, ifile); + size_t _n = getline(&buf, &n, ifile); #define N_TEST 1024 for (int i = 0; i < N_TEST; i++) { if (strchr(buf, delimiter) == NULL) { @@ -338,7 +338,7 @@ void csv_reset_reader(reader_t *reader) { csv_set_delim(csv_params->csv_parser, csv_params->delimiter); if (csv_params->has_header) { - ssize_t _read_size = + size_t _n = getline(&reader->line_buf, &reader->line_buf_size, reader->file); } } diff --git a/libCacheSim/traceReader/generalReader/lcs.c b/libCacheSim/traceReader/generalReader/lcs.c index 1b65c406..5886d3ba 100644 --- a/libCacheSim/traceReader/generalReader/lcs.c +++ b/libCacheSim/traceReader/generalReader/lcs.c @@ -1,9 +1,8 @@ -#include "lcs.h" - #include #include "../customizedReader/binaryUtils.h" +#include "lcs.h" #include "readerInternal.h" #ifdef __cplusplus @@ -24,12 +23,13 @@ const char *print_lcs_trace_format(lcs_trace_header_t *header) { bool verify_LCS_trace_header(lcs_trace_header_t *header) { if (header->start_magic != LCS_TRACE_START_MAGIC) { ERROR("invalid trace file, start magic is wrong 0x%lx\n", - header->start_magic); + (unsigned long)header->start_magic); return false; } if (header->end_magic != LCS_TRACE_END_MAGIC) { - ERROR("invalid trace file, end magic is wrong 0x%lx\n", header->end_magic); + ERROR("invalid trace file, end magic is wrong 0x%lx\n", + (unsigned long)header->end_magic); return false; } diff --git a/libCacheSim/traceReader/reader.c b/libCacheSim/traceReader/reader.c index 190e7dbc..6680f410 100644 --- a/libCacheSim/traceReader/reader.c +++ b/libCacheSim/traceReader/reader.c @@ -52,8 +52,8 @@ reader_t *setup_reader(const char *const trace_path, * currently zstd reader only supports a few binary trace */ reader->is_zstd_file = false; reader->zstd_reader_p = NULL; - size_t slen = strlen(trace_path); #ifdef SUPPORT_ZSTD_TRACE + size_t slen = strlen(trace_path); if (strncmp(trace_path + (slen - 4), ".zst", 4) == 0 || strncmp(trace_path + (slen - 7), ".zst.22", 7) == 0) { reader->is_zstd_file = true; @@ -281,7 +281,7 @@ int read_one_req(reader_t *const reader, request_t *const req) { if (reader->cap_at_n_req > 1 && reader->n_read_req >= reader->cap_at_n_req) { DEBUG("read_one_req: processed %ld requests capped by the user\n", - reader->n_read_req); + (long) reader->n_read_req); req->valid = false; return 1; } diff --git a/libCacheSim/traceReader/sampling/spatial.c b/libCacheSim/traceReader/sampling/spatial.c index c3f57ad1..236e9e9e 100644 --- a/libCacheSim/traceReader/sampling/spatial.c +++ b/libCacheSim/traceReader/sampling/spatial.c @@ -28,7 +28,7 @@ sampler_t *clone_spatial_sampler(const sampler_t *sampler) { return cloned_sampler; } -sampler_t *free_spatial_sampler(sampler_t *sampler) { free(sampler); } +void free_spatial_sampler(sampler_t *sampler) { free(sampler); } sampler_t *create_spatial_sampler(double sampling_ratio) { if (sampling_ratio > 1 || sampling_ratio <= 0) { diff --git a/libCacheSim/traceReader/sampling/temporal.c b/libCacheSim/traceReader/sampling/temporal.c index 098242fd..f626a9dd 100644 --- a/libCacheSim/traceReader/sampling/temporal.c +++ b/libCacheSim/traceReader/sampling/temporal.c @@ -33,7 +33,7 @@ sampler_t *clone_temporal_sampler(const sampler_t *sampler) { return cloned_sampler; } -sampler_t *free_temporal_sampler(sampler_t *sampler) { +void free_temporal_sampler(sampler_t *sampler) { free(sampler->other_params); free(sampler); } diff --git a/libCacheSim/utils/include/mymath.h b/libCacheSim/utils/include/mymath.h index 6b9137ae..2c483ec6 100644 --- a/libCacheSim/utils/include/mymath.h +++ b/libCacheSim/utils/include/mymath.h @@ -24,11 +24,25 @@ void set_rand_seed(uint64_t seed); * random number generator from Knuth MMIX * @return */ -static inline uint64_t next_rand() { +static inline uint64_t next_rand(void) { rand_seed = 6364136223846793005 * rand_seed + 1442695040888963407; return rand_seed; } + +/** + * generate reproducible pseudo rand sequence, used to test concurrent hashtable. + * @method get_next_rand + * @author Chaos + * @date 2023-11-22 + * @param cur [current random number] + * @return [next random number in the sequence] + */ +static inline uint64_t get_next_rand(uint64_t cur) { + uint64_t next = 6364136223846793005 * cur + 1442695040888963407; + return next; +} + static inline long long next_power_of_2(long long N) { // if N is a power of two simply return it if (!(N & (N - 1))) return N; diff --git a/libCacheSim/utils/include/mymutex.h b/libCacheSim/utils/include/mymutex.h new file mode 100644 index 00000000..efeea7a0 --- /dev/null +++ b/libCacheSim/utils/include/mymutex.h @@ -0,0 +1,31 @@ +#pragma once + +// +// Created by Chaos on 11/20/23. +// + +#include +#include +#include + +typedef struct RWLocks +{ + uint64_t locksMask_; + pthread_rwlock_t* locks_; +}RWLocks_t; + + +RWLocks_t* init_RWLocks(uint32_t locksPower); + +void expand_RWLocks(RWLocks_t* rwlocks); + +void destory_RWLocks(RWLocks_t* rwlocks); + +pthread_rwlock_t* getRWLock(RWLocks_t* rwlocks, uint64_t hash); + +#define CAS(ptr, expected, desired) \ + __atomic_compare_exchange_n(ptr, expected, desired, false, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED) +#define fetch_add(ptr, val) __atomic_add_fetch(ptr, val, __ATOMIC_ACQ_REL) +#define fetch_sub(ptr, val) __atomic_sub_fetch(ptr, val, __ATOMIC_ACQ_REL) +#define fetch_and(ptr, val) __atomic_and_fetch(ptr, val, __ATOMIC_ACQ_REL) +#define fetch_or(ptr, val) __atomic_or_fetch(ptr, val, __ATOMIC_ACQ_REL) diff --git a/libCacheSim/utils/include/mysys.h b/libCacheSim/utils/include/mysys.h index eae3335a..f5786c6f 100644 --- a/libCacheSim/utils/include/mysys.h +++ b/libCacheSim/utils/include/mysys.h @@ -18,7 +18,7 @@ int get_n_cores(void); int n_cores(void); -double gettime(); +double gettime(void); void print_cwd(void); diff --git a/libCacheSim/utils/mymutex.c b/libCacheSim/utils/mymutex.c new file mode 100644 index 00000000..dc16511f --- /dev/null +++ b/libCacheSim/utils/mymutex.c @@ -0,0 +1,84 @@ +// +// Created by Chaos on 11/20/23. +// + +#ifdef __cplusplus +extern "C" { +#endif + +#include "include/mymutex.h" +#include "../include/libCacheSim/mem.h" +#include +/** + * [init_RWLocks: Initiate a rw_lock pool. The size of the pool is determined by locksPower. + * For example, if the locksPower is 4, the size is 2^4 = 16. ] + * @author Chaos + * @date 2023-11-20 + * @param locksPower [The power of the number of rwlocks.] + * @return [Handler of the created rw_locks. ] + */ +RWLocks_t* init_RWLocks(uint32_t locksPower){ + RWLocks_t* rwlocks = my_malloc(RWLocks_t); + rwlocks->locksMask_ = (1ULL << locksPower) - 1; + rwlocks->locks_ = my_malloc_n(pthread_rwlock_t, 1ULL << locksPower); + + for(uint64_t s = 0; s <= rwlocks->locksMask_; s++){ + pthread_rwlock_init(rwlocks->locks_ + s, NULL); + } + return rwlocks; +} + +/** + * [expand_RWLocks: Grow the rw_locks pool to the next power of 2..] + * @method expand_RWLocks + * @author Chaos + * @date 2023-11-21 + * @param rwlocks [Handler of operated rw_locks. ] + */ +void expand_RWLocks(RWLocks_t* rwlocks){ + uint64_t oldLocksMask = rwlocks->locksMask_; + rwlocks->locksMask_ = (oldLocksMask << 1) | 1; + pthread_rwlock_t* oldLocks = rwlocks->locks_; + rwlocks->locks_ = my_malloc_n(pthread_rwlock_t, rwlocks->locksMask_ + 1); + for(uint64_t s = 0; s <= oldLocksMask; s++){ + pthread_rwlock_init(rwlocks->locks_ + s, NULL); + } + for(uint64_t s = oldLocksMask + 1; s <= rwlocks->locksMask_; s++){ + pthread_rwlock_init(rwlocks->locks_ + s, NULL); + } + my_free(sizeof(pthread_rwlock_t) * (oldLocksMask + 1), oldLocks); +} + +/** + * [destory_RWLocks: Destory the rw_locks.] + * @method destory_RWLocks + * @author Chaos + * @date 2023-11-20 + * @param rwlocks [Handler of the destoryed rw_locks. ] + */ +void destory_RWLocks(RWLocks_t* rwlocks){ + for(uint64_t s = 0; s <= rwlocks->locksMask_; s++){ + pthread_rwlock_destroy(rwlocks->locks_ + s); + } + my_free(sizeof(pthread_rwlock_t) * (rwlocks->locksMask_ + 1), rwlocks->locks_); + my_free(sizeof(RWLocks_t), rwlocks); +} + +/** + * [getRWLock: Get a rw_lock by a random hash number.] + * @method getRWLock + * @author Chaos + * @date 2023-11-20 + * @param rwlocks [Handler] + * @param hash [A random hash number] + * @return [Current rw_lock.] + */ +pthread_rwlock_t* getRWLock(RWLocks_t* rwlocks, uint64_t hash) { + uint64_t index = hash & rwlocks->locksMask_; + return rwlocks->locks_ + index; +} + + +#ifdef __cplusplus +} +#endif \ No newline at end of file diff --git a/libCacheSim/utils/mysys.c b/libCacheSim/utils/mysys.c index 7af0efdb..893d47be 100644 --- a/libCacheSim/utils/mysys.c +++ b/libCacheSim/utils/mysys.c @@ -58,7 +58,7 @@ int get_n_cores(void) { return get_nprocs(); #else - return sysconf(_SC_NPROCESSORS_ONLN) + return sysconf(_SC_NPROCESSORS_ONLN); #endif WARN("Unknown system, use 4 threads as default\n"); return 4; @@ -88,9 +88,9 @@ void print_glib_ver(void) { // return ebx; // } -int n_cores() { return get_n_cores(); } +int n_cores(void) { return get_n_cores(); } -double gettime() { +double gettime(void) { struct timeval tv; gettimeofday(&tv, NULL); diff --git a/references.md b/references.md index e1982e25..eb05ea13 100644 --- a/references.md +++ b/references.md @@ -14,12 +14,28 @@ } @inproceedings{yang2023-s3fifo, - title={FIFO queues are all you need for cache eviction}, - author={Yang, Juncheng and Zhang, Yazhuo and Qiu, Ziyue and Yue, Yao and Rashmi, K.V.}, - booktitle={Symposium on Operating Systems Principles (SOSP'23)}, + title = {FIFO Queues Are All You Need for Cache Eviction}, + author = {Juncheng Yang and Yazhuo Zhang and Ziyue Qiu and Yao Yue and K.V. Rashmi}, + isbn = {9798400702297}, + publisher = {Association for Computing Machinery}, + booktitle = {Symposium on Operating Systems Principles (SOSP'23)}, + pages = {130–149}, + numpages = {20}, year={2023} } +@inproceedings{yang2023-qdlp, + author = {Juncheng Yang and Ziyue Qiu and Yazhuo Zhang and Yao Yue and K.V. Rashmi}, + title = {FIFO Can Be Better than LRU: The Power of Lazy Promotion and Quick Demotion}, + year = {2023}, + isbn = {9798400701955}, + publisher = {Association for Computing Machinery}, + doi = {10.1145/3593856.3595887}, + booktitle = {Proceedings of the 19th Workshop on Hot Topics in Operating Systems (HotOS23)}, + pages = {70–79}, + numpages = {10}, +} + @inproceedings{yazhuo2024-sieve, title={FIFO queues are all you need for cache eviction}, author={Yang, Juncheng and Zhang, Yazhuo and Qiu, Ziyue and Yue, Yao and Rashmi, K.V.}, diff --git a/scripts/install_dependency.sh b/scripts/install_dependency.sh index 653c61ba..a155d749 100644 --- a/scripts/install_dependency.sh +++ b/scripts/install_dependency.sh @@ -2,7 +2,7 @@ setup_ubuntu() { sudo apt update - sudo apt install -yqq libglib2.0-dev libgoogle-perftools-dev build-essential cmake google-perftools + sudo apt install -yqq libglib2.0-dev libgoogle-perftools-dev build-essential cmake google-perftools xxhash } setup_centos() { @@ -10,7 +10,7 @@ setup_centos() { } setup_macOS() { - brew install glib google-perftools + brew install glib google-perftools argp-standalone xxhash } setup_xgboost() { @@ -55,7 +55,6 @@ setup_zstd() { sudo make install } - CURR_DIR=$(pwd) if [ -n "$(uname -a | grep Ubuntu)" ]; then @@ -66,10 +65,11 @@ else setup_centos fi +setup_zstd + if [[ ! $GITHUB_ACTIONS == "true" ]]; then setup_xgboost setup_lightgbm fi -setup_zstd cd $CURR_DIR diff --git a/scripts/note b/scripts/note new file mode 100644 index 00000000..5688d7c9 --- /dev/null +++ b/scripts/note @@ -0,0 +1,9 @@ +## How to run Caffeine simulator +``` +git clone https://github.com/ben-manes/caffeine.git +./gradlew build +export GRADLE_OPTS="-Xmx204800m" +# modify simulator/src/main/resources/reference.conf, the lirs format is txt with only the object id +./gradlew run simulator:run + +``` diff --git a/test/common.h b/test/common.h index abb0bbed..6d89057c 100644 --- a/test/common.h +++ b/test/common.h @@ -10,7 +10,12 @@ #include #include #include +#ifdef __linux__ #include +#elif __APPLE__ +#include +#include +#endif #include "../libCacheSim/include/libCacheSim.h" #include "../libCacheSim/include/libCacheSim/prefetchAlgo.h" @@ -27,20 +32,30 @@ #define DEFAULT_TTL (300 * 86400) -static inline unsigned int _n_cores0() { - unsigned int eax = 11, ebx = 0, ecx = 1, edx = 0; +// static inline unsigned int _n_cores0() { +// unsigned int eax = 11, ebx = 0, ecx = 1, edx = 0; - asm volatile("cpuid" - : "=a"(eax), "=b"(ebx), "=c"(ecx), "=d"(edx) - : "0"(eax), "2"(ecx) - :); - // printf("Cores: %d\nThreads: %d\nActual thread: %d\n", eax, ebx, edx); - return ebx; -} +// asm volatile("cpuid" +// : "=a"(eax), "=b"(ebx), "=c"(ecx), "=d"(edx) +// : "0"(eax), "2"(ecx) +// :); +// // printf("Cores: %d\nThreads: %d\nActual thread: %d\n", eax, ebx, edx); +// return ebx; +// } -static inline unsigned int _n_cores() { - return get_nprocs(); +#ifdef __linux__ +static inline unsigned int _n_cores(void) { return get_nprocs(); } +#elif __APPLE__ +static inline unsigned int _n_cores(void) { + int count; + size_t count_len = sizeof(count); + sysctlbyname("hw.physicalcpu", &count, &count_len, NULL, 0); + // fprintf(stderr, "you have %i cpu cores", count); + return count; } +#else +#error "what platform is this" +#endif static void _detect_data_path(char *data_path, char *data_name) { sprintf(data_path, "data/%s", data_name); @@ -60,7 +75,7 @@ static void _detect_data_path(char *data_path, char *data_name) { static reader_t *setup_oracleGeneralBin_reader(void) { char data_path[1024]; - _detect_data_path(data_path, "trace.oracleGeneral.bin"); + _detect_data_path(data_path, "cloudPhysicsIO.oracleGeneral.bin"); reader_t *reader_oracle = setup_reader(data_path, ORACLE_GENERAL_TRACE, NULL); return reader_oracle; } @@ -86,7 +101,7 @@ static reader_t *setup_vscsi_reader_with_ignored_obj_size(void) { char data_path[1024]; reader_init_param_t *init_params = g_new0(reader_init_param_t, 1); init_params->ignore_obj_size = true; - _detect_data_path(data_path, "trace.vscsi"); + _detect_data_path(data_path, "cloudPhysicsIO.vscsi"); reader_t *reader_vscsi = setup_reader(data_path, VSCSI_TRACE, init_params); g_free(init_params); return reader_vscsi; @@ -94,14 +109,14 @@ static reader_t *setup_vscsi_reader_with_ignored_obj_size(void) { static reader_t *setup_vscsi_reader(void) { char data_path[1024]; - _detect_data_path(data_path, "trace.vscsi"); + _detect_data_path(data_path, "cloudPhysicsIO.vscsi"); reader_t *reader_vscsi = setup_reader(data_path, VSCSI_TRACE, NULL); return reader_vscsi; } static reader_t *setup_binary_reader(void) { char data_path[1024]; - _detect_data_path(data_path, "trace.vscsi"); + _detect_data_path(data_path, "cloudPhysicsIO.vscsi"); reader_init_param_t *init_params_bin = g_new0(reader_init_param_t, 1); init_params_bin->binary_fmt_str = "obj_size_field = 2; @@ -115,7 +130,7 @@ static reader_t *setup_binary_reader(void) { static reader_t *setup_csv_reader_obj_str(void) { char data_path[1024]; - _detect_data_path(data_path, "trace.csv"); + _detect_data_path(data_path, "cloudPhysicsIO.csv"); reader_init_param_t *init_params_csv = g_new0(reader_init_param_t, 1); init_params_csv->delimiter = ','; init_params_csv->time_field = 2; @@ -130,7 +145,7 @@ static reader_t *setup_csv_reader_obj_str(void) { static reader_t *setup_csv_reader_obj_num(void) { char data_path[1024]; - _detect_data_path(data_path, "trace.csv"); + _detect_data_path(data_path, "cloudPhysicsIO.csv"); reader_init_param_t *init_params_csv = g_new0(reader_init_param_t, 1); init_params_csv->delimiter = ','; init_params_csv->time_field = 2; @@ -145,14 +160,14 @@ static reader_t *setup_csv_reader_obj_num(void) { static reader_t *setup_plaintxt_reader_num(void) { char data_path[1024]; - _detect_data_path(data_path, "trace.txt"); + _detect_data_path(data_path, "cloudPhysicsIO.txt"); reader_init_param_t init_params = {.obj_id_is_num = true}; return setup_reader(data_path, PLAIN_TXT_TRACE, &init_params); } static reader_t *setup_plaintxt_reader_str(void) { char data_path[1024]; - _detect_data_path(data_path, "trace.txt"); + _detect_data_path(data_path, "cloudPhysicsIO.txt"); reader_init_param_t init_params = {.obj_id_is_num = false}; return setup_reader(data_path, PLAIN_TXT_TRACE, &init_params); } @@ -243,6 +258,12 @@ static cache_t *create_test_cache(const char *alg_name, cache = LRU_init(cc_params, NULL); cache->prefetcher = create_prefetcher("Mithril", NULL, cc_params.cache_size); + } else if (strcasecmp(alg_name, "OBL") == 0) { + cache = LRU_init(cc_params, NULL); + cache->prefetcher = create_prefetcher("OBL", NULL, cc_params.cache_size); + } else if (strcasecmp(alg_name, "PG") == 0) { + cache = LRU_init(cc_params, NULL); + cache->prefetcher = create_prefetcher("PG", NULL, cc_params.cache_size); } else { printf("cannot recognize algorithm %s\n", alg_name); exit(1); diff --git a/test/test_evictionAlgo.c b/test/test_evictionAlgo.c index 774f7b14..59a0befe 100644 --- a/test/test_evictionAlgo.c +++ b/test/test_evictionAlgo.c @@ -5,8 +5,7 @@ #include "../libCacheSim/utils/include/mymath.h" #include "common.h" -// static const uint64_t req_cnt_true = 113872, req_byte_true = 4205978112; -static const uint64_t req_cnt_true = 113872, req_byte_true = 4368040448; +static const uint64_t g_req_cnt_true = 113872, g_req_byte_true = 4368040448; static void _verify_profiler_results(const cache_stat_t *res, uint64_t num_of_sizes, @@ -24,23 +23,23 @@ static void _verify_profiler_results(const cache_stat_t *res, static void print_results(const cache_t *cache, const cache_stat_t *res) { printf("%s uint64_t cache_size[] = {", cache->cache_name); - printf("%ld", res[0].cache_size); + printf("%ld", (long)res[0].cache_size); for (uint64_t i = 1; i < CACHE_SIZE / STEP_SIZE; i++) { - printf(", %ld", res[i].cache_size); + printf(", %ld", (long)res[i].cache_size); } printf("};\n"); printf("uint64_t miss_cnt_true[] = {"); - printf("%ld", res[0].n_miss); + printf("%ld", (long)res[0].n_miss); for (uint64_t i = 1; i < CACHE_SIZE / STEP_SIZE; i++) { - printf(", %ld", res[i].n_miss); + printf(", %ld", (long)res[i].n_miss); } printf("};\n"); printf("uint64_t miss_byte_true[] = {"); - printf("%ld", res[0].n_miss_byte); + printf("%ld", (long)res[0].n_miss_byte); for (uint64_t i = 1; i < CACHE_SIZE / STEP_SIZE; i++) { - printf(", %ld", res[i].n_miss_byte); + printf(", %ld", (long)res[i].n_miss_byte); } printf("};\n"); } @@ -60,14 +59,13 @@ static void test_LRU(gconstpointer user_data) { reader, cache, STEP_SIZE, NULL, 0, 0, _n_cores()); print_results(cache, res); - _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, req_cnt_true, - miss_cnt_true, req_byte_true, miss_byte_true); + _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, g_req_cnt_true, + miss_cnt_true, g_req_byte_true, miss_byte_true); cache->cache_free(cache); my_free(sizeof(cache_stat_t), res); } static void test_Clock(gconstpointer user_data) { - /* myclock */ uint64_t miss_cnt_true[] = {93313, 89775, 83411, 81328, 74815, 72283, 71927, 64456}; uint64_t miss_byte_true[] = {4213887488, 4064512000, 3762650624, 3644467200, @@ -82,8 +80,8 @@ static void test_Clock(gconstpointer user_data) { reader, cache, STEP_SIZE, NULL, 0, 0, _n_cores()); print_results(cache, res); - _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, req_cnt_true, - miss_cnt_true, req_byte_true, miss_byte_true); + _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, g_req_cnt_true, + miss_cnt_true, g_req_byte_true, miss_byte_true); cache->cache_free(cache); my_free(sizeof(cache_stat_t), res); } @@ -103,8 +101,8 @@ static void test_FIFO(gconstpointer user_data) { reader, cache, STEP_SIZE, NULL, 0, 0, _n_cores()); print_results(cache, res); - _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, req_cnt_true, - miss_cnt_true, req_byte_true, miss_byte_true); + _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, g_req_cnt_true, + miss_cnt_true, g_req_byte_true, miss_byte_true); cache->cache_free(cache); my_free(sizeof(cache_stat_t), res); } @@ -128,8 +126,8 @@ static void test_Belady(gconstpointer user_data) { reader, cache, STEP_SIZE, NULL, 0, 0, _n_cores()); print_results(cache, res); - _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, req_cnt_true, - miss_cnt_true, req_byte_true, miss_byte_true); + _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, g_req_cnt_true, + miss_cnt_true, g_req_byte_true, miss_byte_true); cache->cache_free(cache); my_free(sizeof(cache_stat_t), res); } @@ -153,8 +151,8 @@ static void test_BeladySize(gconstpointer user_data) { reader, cache, STEP_SIZE, NULL, 0, 0, _n_cores()); print_results(cache, res); - _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, req_cnt_true, - miss_cnt_true, req_byte_true, miss_byte_true); + _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, g_req_cnt_true, + miss_cnt_true, g_req_byte_true, miss_byte_true); cache->cache_free(cache); my_free(sizeof(cache_stat_t), res); } @@ -174,8 +172,8 @@ static void test_Random(gconstpointer user_data) { reader, cache, STEP_SIZE, NULL, 0, 0, _n_cores()); print_results(cache, res); - _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, req_cnt_true, - miss_cnt_true, req_byte_true, miss_byte_true); + _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, g_req_cnt_true, + miss_cnt_true, g_req_byte_true, miss_byte_true); cache->cache_free(cache); my_free(sizeof(cache_stat_t), res); } @@ -195,8 +193,8 @@ static void test_LFU(gconstpointer user_data) { reader, cache, STEP_SIZE, NULL, 0, 0, _n_cores()); print_results(cache, res); - _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, req_cnt_true, - miss_cnt_true, req_byte_true, miss_byte_true); + _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, g_req_cnt_true, + miss_cnt_true, g_req_byte_true, miss_byte_true); cache->cache_free(cache); g_free(res); } @@ -216,8 +214,8 @@ static void test_LFUCpp(gconstpointer user_data) { reader, cache, STEP_SIZE, NULL, 0, 0, _n_cores()); print_results(cache, res); - _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, req_cnt_true, - miss_cnt_true, req_byte_true, miss_byte_true); + _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, g_req_cnt_true, + miss_cnt_true, g_req_byte_true, miss_byte_true); cache->cache_free(cache); my_free(sizeof(cache_stat_t), res); } @@ -237,8 +235,8 @@ static void test_GDSF(gconstpointer user_data) { reader, cache, STEP_SIZE, NULL, 0, 0, _n_cores()); print_results(cache, res); - _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, req_cnt_true, - miss_cnt_true, req_byte_true, miss_byte_true); + _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, g_req_cnt_true, + miss_cnt_true, g_req_byte_true, miss_byte_true); cache->cache_free(cache); my_free(sizeof(cache_stat_t), res); } @@ -258,8 +256,8 @@ static void test_LHD(gconstpointer user_data) { reader, cache, STEP_SIZE, NULL, 0, 0, _n_cores()); print_results(cache, res); - _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, req_cnt_true, - miss_cnt_true, req_byte_true, miss_byte_true); + _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, g_req_cnt_true, + miss_cnt_true, g_req_byte_true, miss_byte_true); cache->cache_free(cache); my_free(sizeof(cache_stat_t), res); } @@ -279,8 +277,8 @@ static void test_Hyperbolic(gconstpointer user_data) { reader, cache, STEP_SIZE, NULL, 0, 0, _n_cores()); print_results(cache, res); - _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, req_cnt_true, - miss_cnt_true, req_byte_true, miss_byte_true); + _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, g_req_cnt_true, + miss_cnt_true, g_req_byte_true, miss_byte_true); cache->cache_free(cache); my_free(sizeof(cache_stat_t), res); } @@ -300,8 +298,8 @@ static void test_LeCaR(gconstpointer user_data) { reader, cache, STEP_SIZE, NULL, 0, 0, _n_cores()); print_results(cache, res); - _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, req_cnt_true, - miss_cnt_true, req_byte_true, miss_byte_true); + _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, g_req_cnt_true, + miss_cnt_true, g_req_byte_true, miss_byte_true); cache->cache_free(cache); my_free(sizeof(cache_stat_t), res); } @@ -321,8 +319,8 @@ static void test_Cacheus(gconstpointer user_data) { reader, cache, STEP_SIZE, NULL, 0, 0, _n_cores()); print_results(cache, res); - _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, req_cnt_true, - miss_cnt_true, req_byte_true, miss_byte_true); + _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, g_req_cnt_true, + miss_cnt_true, g_req_byte_true, miss_byte_true); cache->cache_free(cache); my_free(sizeof(cache_stat_t), res); } @@ -342,8 +340,8 @@ static void test_SR_LRU(gconstpointer user_data) { reader, cache, STEP_SIZE, NULL, 0, 0, _n_cores()); print_results(cache, res); - _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, req_cnt_true, - miss_cnt_true, req_byte_true, miss_byte_true); + _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, g_req_cnt_true, + miss_cnt_true, g_req_byte_true, miss_byte_true); cache->cache_free(cache); my_free(sizeof(cache_stat_t), res); } @@ -363,8 +361,8 @@ static void test_CR_LFU(gconstpointer user_data) { reader, cache, STEP_SIZE, NULL, 0, 0, _n_cores()); print_results(cache, res); - _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, req_cnt_true, - miss_cnt_true, req_byte_true, miss_byte_true); + _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, g_req_cnt_true, + miss_cnt_true, g_req_byte_true, miss_byte_true); cache->cache_free(cache); my_free(sizeof(cache_stat_t), res); } @@ -384,8 +382,8 @@ static void test_LFUDA(gconstpointer user_data) { reader, cache, STEP_SIZE, NULL, 0, 0, _n_cores()); print_results(cache, res); - _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, req_cnt_true, - miss_cnt_true, req_byte_true, miss_byte_true); + _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, g_req_cnt_true, + miss_cnt_true, g_req_byte_true, miss_byte_true); cache->cache_free(cache); my_free(sizeof(cache_stat_t), res); } @@ -405,8 +403,8 @@ static void test_MRU(gconstpointer user_data) { reader, cache, STEP_SIZE, NULL, 0, 0, _n_cores()); print_results(cache, res); - _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, req_cnt_true, - miss_cnt_true, req_byte_true, miss_byte_true); + _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, g_req_cnt_true, + miss_cnt_true, g_req_byte_true, miss_byte_true); cache->cache_free(cache); my_free(sizeof(cache_stat_t), res); } @@ -426,8 +424,8 @@ static void test_ARC(gconstpointer user_data) { reader, cache, STEP_SIZE, NULL, 0, 0, _n_cores()); print_results(cache, res); - _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, req_cnt_true, - miss_cnt_true, req_byte_true, miss_byte_true); + _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, g_req_cnt_true, + miss_cnt_true, g_req_byte_true, miss_byte_true); cache->cache_free(cache); my_free(sizeof(cache_stat_t), res); } @@ -447,8 +445,8 @@ static void test_SLRU(gconstpointer user_data) { reader, cache, STEP_SIZE, NULL, 0, 0, _n_cores()); print_results(cache, res); - _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, req_cnt_true, - miss_cnt_true, req_byte_true, miss_byte_true); + _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, g_req_cnt_true, + miss_cnt_true, g_req_byte_true, miss_byte_true); cache->cache_free(cache); my_free(sizeof(cache_stat_t), res); } @@ -468,8 +466,8 @@ static void test_QDLP_FIFO(gconstpointer user_data) { reader, cache, STEP_SIZE, NULL, 0, 0, _n_cores()); print_results(cache, res); - _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, req_cnt_true, - miss_cnt_true, req_byte_true, miss_byte_true); + _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, g_req_cnt_true, + miss_cnt_true, g_req_byte_true, miss_byte_true); cache->cache_free(cache); my_free(sizeof(cache_stat_t), res); } @@ -489,8 +487,8 @@ static void test_S3FIFO(gconstpointer user_data) { reader, cache, STEP_SIZE, NULL, 0, 0, _n_cores()); print_results(cache, res); - _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, req_cnt_true, - miss_cnt_true, req_byte_true, miss_byte_true); + _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, g_req_cnt_true, + miss_cnt_true, g_req_byte_true, miss_byte_true); cache->cache_free(cache); my_free(sizeof(cache_stat_t), res); } @@ -510,8 +508,8 @@ static void test_Sieve(gconstpointer user_data) { reader, cache, STEP_SIZE, NULL, 0, 0, _n_cores()); print_results(cache, res); - _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, req_cnt_true, - miss_cnt_true, req_byte_true, miss_byte_true); + _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, g_req_cnt_true, + miss_cnt_true, g_req_byte_true, miss_byte_true); cache->cache_free(cache); my_free(sizeof(cache_stat_t), res); } @@ -535,8 +533,8 @@ static void test_LIRS(gconstpointer user_data) { reader, cache, STEP_SIZE, NULL, 0, 0, _n_cores()); print_results(cache, res); - _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, req_cnt_true, - miss_cnt_true, req_byte_true, miss_byte_true); + _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, g_req_cnt_true, + miss_cnt_true, g_req_byte_true, miss_byte_true); cache->cache_free(cache); my_free(sizeof(cache_stat_t), res); } diff --git a/test/test_prefetchAlgo.c b/test/test_prefetchAlgo.c index 190c6ec0..37b2397e 100644 --- a/test/test_prefetchAlgo.c +++ b/test/test_prefetchAlgo.c @@ -4,8 +4,8 @@ #include "../libCacheSim/utils/include/mymath.h" #include "common.h" -// static const uint64_t req_cnt_true = 113872, req_byte_true = 4205978112; -static const uint64_t req_cnt_true = 113872, req_byte_true = 4368040448; + +static const uint64_t g_req_cnt_true = 113872, g_req_byte_true = 4368040448; static void _verify_profiler_results(const cache_stat_t *res, uint64_t num_of_sizes, @@ -23,23 +23,23 @@ static void _verify_profiler_results(const cache_stat_t *res, static void print_results(const cache_t *cache, const cache_stat_t *res) { printf("%s uint64_t cache_size[] = {", cache->cache_name); - printf("%ld", res[0].cache_size); + printf("%ld", (long)res[0].cache_size); for (uint64_t i = 1; i < CACHE_SIZE / STEP_SIZE; i++) { - printf(", %ld", res[i].cache_size); + printf(", %ld", (long)res[i].cache_size); } printf("};\n"); printf("uint64_t miss_cnt_true[] = {"); - printf("%ld", res[0].n_miss); + printf("%ld", (long)res[0].n_miss); for (uint64_t i = 1; i < CACHE_SIZE / STEP_SIZE; i++) { - printf(", %ld", res[i].n_miss); + printf(", %ld", (long)res[i].n_miss); } printf("};\n"); printf("uint64_t miss_byte_true[] = {"); - printf("%ld", res[0].n_miss_byte); + printf("%ld", (long)res[0].n_miss_byte); for (uint64_t i = 1; i < CACHE_SIZE / STEP_SIZE; i++) { - printf(", %ld", res[i].n_miss_byte); + printf(", %ld", (long)res[i].n_miss_byte); } printf("};\n"); } @@ -59,8 +59,42 @@ static void test_Mithril(gconstpointer user_data) { reader, cache, STEP_SIZE, NULL, 0, 0, _n_cores()); print_results(cache, res); - _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, req_cnt_true, - miss_cnt_true, req_byte_true, miss_byte_true); + _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, g_req_cnt_true, + miss_cnt_true, g_req_byte_true, miss_byte_true); + cache->cache_free(cache); + my_free(sizeof(cache_stat_t), res); +} + +static void test_OBL(gconstpointer user_data) { + uint64_t miss_cnt_true[] = {92139, 88548, 82337, 80487, 71259, 70869, 70737, 70469}; + uint64_t miss_byte_true[] = {4213140480, 4060079616, 3776877568, 3659406848, + 3099764736, 3076965888, 3074241024, 3060499968}; + + reader_t *reader = (reader_t *)user_data; + common_cache_params_t cc_params = {.cache_size = CACHE_SIZE, .hashpower = 20, .default_ttl = DEFAULT_TTL}; + cache_t *cache = create_test_cache("OBL", cc_params, reader, NULL); + g_assert_true(cache != NULL); + cache_stat_t *res = simulate_at_multi_sizes_with_step_size(reader, cache, STEP_SIZE, NULL, 0, 0, _n_cores()); + + print_results(cache, res); + _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, g_req_cnt_true, miss_cnt_true, g_req_byte_true, miss_byte_true); + cache->cache_free(cache); + my_free(sizeof(cache_stat_t), res); +} + +static void test_PG(gconstpointer user_data) { + uint64_t miss_cnt_true[] = {92786, 89494, 83403, 81564, 72360, 71973, 71842, 71574}; + uint64_t miss_byte_true[] = {4195964416, 4054977024, 3776220672, 3659069952, + 3100251136, 3077595648, 3074874880, 3061133824}; + + reader_t *reader = (reader_t *)user_data; + common_cache_params_t cc_params = {.cache_size = CACHE_SIZE, .hashpower = 20, .default_ttl = DEFAULT_TTL}; + cache_t *cache = create_test_cache("PG", cc_params, reader, NULL); + g_assert_true(cache != NULL); + cache_stat_t *res = simulate_at_multi_sizes_with_step_size(reader, cache, STEP_SIZE, NULL, 0, 0, _n_cores()); + + print_results(cache, res); + _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, g_req_cnt_true, miss_cnt_true, g_req_byte_true, miss_byte_true); cache->cache_free(cache); my_free(sizeof(cache_stat_t), res); } @@ -80,6 +114,8 @@ int main(int argc, char *argv[]) { reader = setup_oracleGeneralBin_reader(); // reader = setup_vscsi_reader_with_ignored_obj_size(); g_test_add_data_func("/libCacheSim/cacheAlgo_Mithril", reader, test_Mithril); + g_test_add_data_func("/libCacheSim/cacheAlgo_OBL", reader, test_OBL); + g_test_add_data_func("/libCacheSim/cacheAlgo_PG", reader, test_PG); return g_test_run(); } \ No newline at end of file diff --git a/test/test_simulator.c b/test/test_simulator.c index 937cb959..8921d274 100644 --- a/test/test_simulator.c +++ b/test/test_simulator.c @@ -106,7 +106,8 @@ static void test_simulator(gconstpointer user_data) { g_assert_true(caches[i] != NULL); } - res = simulate_with_multi_caches(reader, caches, 4, NULL, 0, 0, _n_cores(), false); + res = simulate_with_multi_caches(reader, caches, 4, NULL, 0, 0, _n_cores(), + false); g_assert_cmpuint(res[0].cache_size, ==, STEP_SIZE); g_assert_cmpuint(res[1].n_req_byte, ==, req_byte_true); g_assert_cmpuint(res[3].n_req, ==, req_cnt_true); @@ -205,9 +206,10 @@ static void test_simulator_with_ttl(gconstpointer user_data) { reader, cache, STEP_SIZE, NULL, 0, 0, _n_cores()); for (uint64_t i = 0; i < CACHE_SIZE / STEP_SIZE; i++) { - printf("cache size: %lu, n_req: %lu, n_req_byte: %lu, n_miss: %8lu %16lu\n", - res[i].cache_size, res[i].n_req, res[i].n_req_byte, res[i].n_miss, - res[i].n_miss_byte); + printf("cache size: %lu, n_req: %ld, n_req_byte: %ld, n_miss: %8ld %16ld\n", + (unsigned long)res[i].cache_size, (long)res[i].n_req, + (long)res[i].n_req_byte, (long)res[i].n_miss, + (long)res[i].n_miss_byte); g_assert_cmpuint(res[i].cache_size, ==, STEP_SIZE * (i + 1)); g_assert_cmpuint(res[i].n_req, ==, req_cnt_true); g_assert_cmpuint(res[i].n_req_byte, ==, req_byte_true); @@ -224,13 +226,11 @@ int main(int argc, char *argv[]) { reader_t *reader; reader = setup_plaintxt_reader_num(); - g_test_add_data_func_full("/libCacheSim/simulator_no_size_plain_num", - reader, + g_test_add_data_func_full("/libCacheSim/simulator_no_size_plain_num", reader, test_simulator_no_size, test_teardown); reader = setup_plaintxt_reader_str(); - g_test_add_data_func_full("/libCacheSim/simulator_no_size_plain_str", - reader, + g_test_add_data_func_full("/libCacheSim/simulator_no_size_plain_str", reader, test_simulator_no_size, test_teardown); reader = setup_csv_reader_obj_num();