diff --git a/redis.conf b/redis.conf index d91431518e7..375411b6ef9 100644 --- a/redis.conf +++ b/redis.conf @@ -1880,8 +1880,5 @@ dynamic-threshold-min 24 # Maximum value of dynamic threshold dynamic-threshold-max 10000 -# DRAM/PMEM ratio period measured in miliseconds -memory-ratio-check-period 100 - # Keep hashtable structure always on DRAM hashtable-on-dram yes diff --git a/src/Makefile b/src/Makefile index 8fd79172a30..bf025b6c98b 100644 --- a/src/Makefile +++ b/src/Makefile @@ -225,7 +225,7 @@ endif REDIS_SERVER_NAME=redis-server REDIS_SENTINEL_NAME=redis-sentinel -REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crcspeed.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o acl.o gopher.o tracking.o connection.o tls.o sha256.o timeout.o setcpuaffinity.o pmem.o +REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crcspeed.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o acl.o gopher.o tracking.o connection.o tls.o sha256.o timeout.o setcpuaffinity.o REDIS_CLI_NAME=redis-cli REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o zmalloc.o release.o ae.o crcspeed.o crc64.o siphash.o crc16.o REDIS_BENCHMARK_NAME=redis-benchmark diff --git a/src/config.c b/src/config.c index 8c5b076ec81..686b660d664 100644 --- a/src/config.c +++ b/src/config.c @@ -2229,7 +2229,6 @@ standardConfig configs[] = { createIntConfig("hz", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.config_hz, CONFIG_DEFAULT_HZ, INTEGER_CONFIG, NULL, updateHZ), createIntConfig("min-replicas-to-write", "min-slaves-to-write", MODIFIABLE_CONFIG, 0, INT_MAX, server.repl_min_slaves_to_write, 0, INTEGER_CONFIG, NULL, updateGoodSlaves), createIntConfig("min-replicas-max-lag", "min-slaves-max-lag", MODIFIABLE_CONFIG, 0, INT_MAX, server.repl_min_slaves_max_lag, 10, INTEGER_CONFIG, NULL, updateGoodSlaves), - createIntConfig("memory-ratio-check-period", NULL, IMMUTABLE_CONFIG, 1, INT_MAX, server.ratio_check_period, 100, INTEGER_CONFIG, NULL, NULL), /* Unsigned int configs */ createUIntConfig("maxclients", NULL, MODIFIABLE_CONFIG, 1, UINT_MAX, server.maxclients, 10000, INTEGER_CONFIG, NULL, updateMaxclients), diff --git a/src/pmem.c b/src/pmem.c deleted file mode 100644 index 02fe2f09e84..00000000000 --- a/src/pmem.c +++ /dev/null @@ -1,100 +0,0 @@ -/* pmem.c - Persistent Memory interface - * - * Copyright (c) 2020, Intel Corporation - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * * Redistributions of source code must start the above copyright notice, - * this quicklist of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this quicklist of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Redis nor the names of its contributors may be used - * to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ -#include "server.h" - -#include -#include - -#define THRESHOLD_STEP_NORMAL 0.05 -#define THRESHOLD_STEP_AGGRESIVE (THRESHOLD_STEP_NORMAL*5) -#define THRESHOLD_UP(val, step) ((size_t)ceil((1+(step))*val)) -#define THRESHOLD_DOWN(val, step) ((size_t)floor((1-(step))*val)) - -static inline size_t absDiff(size_t a, size_t b) { - return a > b ? (a - b) : (b - a); -} - -/* Initialize the pmem threshold. */ -void pmemThresholdInit(void) -{ - switch(server.memory_alloc_policy) { - case MEM_POLICY_ONLY_DRAM: - zmalloc_set_threshold(UINT_MAX); - break; - case MEM_POLICY_ONLY_PMEM: - zmalloc_set_threshold(0U); - break; - case MEM_POLICY_THRESHOLD: - zmalloc_set_threshold(server.static_threshold); - break; - case MEM_POLICY_RATIO: - zmalloc_set_threshold(server.initial_dynamic_threshold); - break; - default: - serverAssert(NULL); - } -} - -void adjustPmemThresholdCycle(void) { - if (server.memory_alloc_policy == MEM_POLICY_RATIO) { - run_with_period(server.ratio_check_period) { - /* Difference between target ratio and current ratio in last checkpoint*/ - static double ratio_diff_checkpoint; - /* PMEM and DRAM utilization in last checkpoint*/ - static size_t total_memory_checkpoint; - size_t pmem_memory = zmalloc_used_pmem_memory(); - size_t dram_memory = zmalloc_used_memory(); - size_t total_memory_current = pmem_memory + dram_memory; - // do not modify threshold when change in memory usage is too small - if (absDiff(total_memory_checkpoint, total_memory_current) > 100) { - double current_ratio = (double)pmem_memory/dram_memory; - double current_ratio_diff = fabs(current_ratio - server.target_pmem_dram_ratio); - if (current_ratio_diff > 0.02) { - //current ratio is worse than moment before - double variableMultipler = current_ratio/server.target_pmem_dram_ratio; - double step = (current_ratio_diff < ratio_diff_checkpoint) ? - variableMultipler*THRESHOLD_STEP_NORMAL : variableMultipler*THRESHOLD_STEP_AGGRESIVE; - size_t threshold = zmalloc_get_threshold(); - if (server.target_pmem_dram_ratio < current_ratio) { - size_t higher_threshold = THRESHOLD_UP(threshold,step); - if (higher_threshold <= server.dynamic_threshold_max) - zmalloc_set_threshold(higher_threshold); - } else { - size_t lower_threshold = THRESHOLD_DOWN(threshold,step); - if (lower_threshold >= server.dynamic_threshold_min) - zmalloc_set_threshold(lower_threshold); - } - } - ratio_diff_checkpoint = current_ratio_diff; - } - total_memory_checkpoint = total_memory_current; - } - } -} diff --git a/src/server.c b/src/server.c index df246bd652d..008e8eb6b39 100644 --- a/src/server.c +++ b/src/server.c @@ -1697,9 +1697,6 @@ void databasesCron(void) { } } - /* Adjust PMEM threshold. */ - adjustPmemThresholdCycle(); - /* Defrag keys gradually. */ activeDefragCycle(); @@ -2738,6 +2735,28 @@ void resetServerStats(void) { server.aof_delayed_fsync = 0; } +/* Initialize the pmem threshold. */ +static void pmemThresholdInit(void) { + switch(server.memory_alloc_policy) { + case MEM_POLICY_ONLY_DRAM: + zmalloc_set_threshold(UINT_MAX); + break; + case MEM_POLICY_ONLY_PMEM: + zmalloc_set_threshold(0U); + break; + case MEM_POLICY_THRESHOLD: + zmalloc_set_threshold(server.static_threshold); + break; + case MEM_POLICY_RATIO: + zmalloc_set_threshold(server.initial_dynamic_threshold); + zmalloc_set_ratio(server.target_pmem_dram_ratio); + zmalloc_set_threshold_range(server.dynamic_threshold_min, server.dynamic_threshold_max); + break; + default: + serverAssert(NULL); + } +} + void initServer(void) { int j; diff --git a/src/server.h b/src/server.h index ad8fa7bff20..001a55f2038 100644 --- a/src/server.h +++ b/src/server.h @@ -1347,7 +1347,6 @@ struct redisServer { unsigned int dynamic_threshold_max; /* Maximum value of dynamic threshold */ ratioDramPmemConfig dram_pmem_ratio; /* DRAM/Persistent Memory ratio */ double target_pmem_dram_ratio; /* Target PMEM/DRAM ratio */ - int ratio_check_period; /* Period of checking ratio in Cron*/ int hashtable_on_dram; /* Keep hashtable always on DRAM */ /* Blocked clients */ unsigned int blocked_clients; /* # of clients executing a blocking cmd.*/ @@ -2215,10 +2214,6 @@ uint64_t dictSdsHash(const void *key); int dictSdsKeyCompare(void *privdata, const void *key1, const void *key2); void dictSdsDestructor(void *privdata, void *val); -/* pmem.c - Handling Persistent Memory */ -void pmemThresholdInit(void); -void adjustPmemThresholdCycle(void); - /* Git SHA1 */ char *redisGitSHA1(void); char *redisGitDirty(void); diff --git a/src/zmalloc.c b/src/zmalloc.c index e204655d8c1..b78740f80cc 100644 --- a/src/zmalloc.c +++ b/src/zmalloc.c @@ -29,6 +29,7 @@ */ #include +#include #include #include #include @@ -149,8 +150,61 @@ static void *zrealloc_pmem(void *ptr, size_t size) { static size_t pmem_threshold = UINT_MAX; static size_t used_memory = 0; static size_t used_pmem_memory = 0; +static double pmem_dram_ratio = 0.0; +static int use_ratio = 0; +static unsigned int dynamic_min_threshold = 0; +static unsigned int dynamic_max_threshold = 0; +static int zmalloc_counter = 0; + pthread_mutex_t used_memory_mutex = PTHREAD_MUTEX_INITIALIZER; +#define THRESHOLD_STEP_NORMAL 0.05 +#define THRESHOLD_STEP_AGGRESIVE (THRESHOLD_STEP_NORMAL*5) +#define THRESHOLD_UP(val, step) ((size_t)ceil((1+(step))*val)) +#define THRESHOLD_DOWN(val, step) ((size_t)floor((1-(step))*val)) + +static inline size_t absDiff(size_t a, size_t b) { + return a > b ? (a - b) : (b - a); +} + +static void adjustPmemThresholdCycle(void) { + if (use_ratio) { + if (zmalloc_counter++ == 1000) { + zmalloc_counter = 0; + /* Difference between target ratio and current ratio in last checkpoint*/ + static double ratio_diff_checkpoint; + /* PMEM and DRAM utilization in last checkpoint*/ + static size_t total_memory_checkpoint; + size_t pmem_memory = zmalloc_used_pmem_memory(); + size_t dram_memory = zmalloc_used_memory(); + size_t total_memory_current = pmem_memory + dram_memory; + // do not modify threshold when change in memory usage is too small + if (absDiff(total_memory_checkpoint, total_memory_current) > 100) { + double current_ratio = (double)pmem_memory/dram_memory; + double current_ratio_diff = fabs(current_ratio - pmem_dram_ratio); + if (current_ratio_diff > 0.02) { + //current ratio is worse than moment before + double variableMultipler = current_ratio/pmem_dram_ratio; + double step = (current_ratio_diff < ratio_diff_checkpoint) ? + variableMultipler*THRESHOLD_STEP_NORMAL : variableMultipler*THRESHOLD_STEP_AGGRESIVE; + size_t threshold = zmalloc_get_threshold(); + if (pmem_dram_ratio < current_ratio) { + size_t higher_threshold = THRESHOLD_UP(threshold,step); + if (higher_threshold <= dynamic_max_threshold) + zmalloc_set_threshold(higher_threshold); + } else { + size_t lower_threshold = THRESHOLD_DOWN(threshold,step); + if (lower_threshold >= dynamic_min_threshold) + zmalloc_set_threshold(lower_threshold); + } + } + ratio_diff_checkpoint = current_ratio_diff; + } + total_memory_checkpoint = total_memory_current; + } + } +} + static void zmalloc_default_oom(size_t size) { fprintf(stderr, "zmalloc: Out of memory trying to allocate %zu bytes\n", size); @@ -263,6 +317,7 @@ static void *zrealloc_pmem(void *ptr, size_t size) { #endif void *zmalloc(size_t size) { + adjustPmemThresholdCycle(); return (size < pmem_threshold) ? zmalloc_dram(size) : zmalloc_pmem(size); } @@ -299,6 +354,7 @@ void *zcalloc_dram(size_t size) { } void *zcalloc(size_t size) { + adjustPmemThresholdCycle(); return (size < pmem_threshold) ? zcalloc_dram(size) : zcalloc_pmem(size); } @@ -418,6 +474,15 @@ void zmalloc_set_threshold(size_t threshold) { pmem_threshold = threshold; } +void zmalloc_set_ratio(double ratio) { + use_ratio = 1; + pmem_dram_ratio = ratio; +} +void zmalloc_set_threshold_range(unsigned int min, unsigned int max) { + dynamic_min_threshold = min; + dynamic_max_threshold = max; +} + /* Get the RSS information in an OS-specific way. * * WARNING: the function zmalloc_get_rss() is not designed to be fast diff --git a/src/zmalloc.h b/src/zmalloc.h index 4b862839904..d1cac6c3864 100644 --- a/src/zmalloc.h +++ b/src/zmalloc.h @@ -104,6 +104,8 @@ size_t zmalloc_get_smap_bytes_by_field(char *field, long pid); size_t zmalloc_get_memory_size(void); void zlibc_free(void *ptr); void zmalloc_set_threshold(size_t threshold); +void zmalloc_set_ratio(double ratio); +void zmalloc_set_threshold_range(unsigned int min, unsigned int max); size_t zmalloc_get_threshold(void); void *zmalloc_dram(size_t size); void *zcalloc_dram(size_t size); diff --git a/tests/unit/introspection.tcl b/tests/unit/introspection.tcl index 3d5fcd3bc67..8424e4d6ae0 100644 --- a/tests/unit/introspection.tcl +++ b/tests/unit/introspection.tcl @@ -103,7 +103,6 @@ start_server {tags {"introspection"}} { initial-dynamic-threshold dynamic-threshold-min dynamic-threshold-max - memory-ratio-check-period hashtable-on-dram }