From b4dc1f209a93c851ee065bd543f4a2fea6bd3ef9 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Fri, 11 Feb 2022 10:26:35 +0000 Subject: [PATCH 1/2] Use a typedef for the block index, and reduce it from 64 to 32 bit which is plenty. Currently only 19 is used. --- .../src/vespamalloc/malloc/datasegment.h | 44 ++++++----- .../src/vespamalloc/malloc/datasegment.hpp | 76 +++++++++++-------- 2 files changed, 69 insertions(+), 51 deletions(-) diff --git a/vespamalloc/src/vespamalloc/malloc/datasegment.h b/vespamalloc/src/vespamalloc/malloc/datasegment.h index d03a585ccc29..e44f611201e2 100644 --- a/vespamalloc/src/vespamalloc/malloc/datasegment.h +++ b/vespamalloc/src/vespamalloc/malloc/datasegment.h @@ -13,7 +13,7 @@ template class DataSegment { public: - typedef unsigned FreeCountT; + using BlockIdT = uint32_t; enum { UNMAPPED_BLOCK=-4, UNUSED_BLOCK=-3, FREE_BLOCK=-2, SYSTEM_BLOCK=-1, NUM_ADMIN_CLASSES=4 }; DataSegment() __attribute__((noinline)); ~DataSegment() __attribute__((noinline)); @@ -21,7 +21,7 @@ class DataSegment void * getBlock(size_t & oldBlockSize, SizeClassT sc) __attribute__((noinline)); void returnBlock(void *ptr) __attribute__((noinline)); SizeClassT sizeClass(const void * ptr) const { return _blockList[blockId(ptr)].sizeClass(); } - bool containsPtr(const void * ptr) const { return blockId(ptr) < BlockCount; } + bool containsPtr(const void * ptr) const { return blockId(ptr) < BlockCount; } size_t getMaxSize(const void * ptr) const { return _blockList[blockId(ptr)].getMaxSize(); } const void * start() const { return _osMemory.getStart(); } const void * end() const { return _osMemory.getEnd(); } @@ -42,7 +42,7 @@ class DataSegment checkAndLogBigSegment(); } void enableThreadSupport() { _mutex.init(); } - static size_t blockId(const void * ptr) { + static uint32_t blockId(const void * ptr) { return (size_t(ptr) - Memory::getMinPreferredStartAddress())/BlockSize; } static void * fromBlockId(size_t id) { @@ -61,20 +61,22 @@ class DataSegment DataSegment(const DataSegment & rhs); DataSegment & operator = (const DataSegment & rhs); - enum { BlockSize=0x200000, BlockCount=0x80000 }; //1T + // Allow for 1T heap + static constexpr size_t BlockSize = 0x200000ul; + static constexpr BlockIdT BlockCount = 0x80000; class BlockT { public: - BlockT(SizeClassT szClass = UNUSED_BLOCK, FreeCountT numBlocks = 0) + BlockT(SizeClassT szClass = UNUSED_BLOCK, BlockIdT numBlocks = 0) : _sizeClass(szClass), _freeChainLength(0), _realNumBlocks(numBlocks) { } SizeClassT sizeClass() const { return _sizeClass; } - FreeCountT realNumBlocks() const { return _realNumBlocks; } - FreeCountT freeChainLength() const { return _freeChainLength; } + BlockIdT realNumBlocks() const { return _realNumBlocks; } + BlockIdT freeChainLength() const { return _freeChainLength; } void sizeClass(SizeClassT sc) { _sizeClass = sc; } - void realNumBlocks(FreeCountT fc) { _realNumBlocks = fc; } - void freeChainLength(FreeCountT fc) { _freeChainLength = fc; } + void realNumBlocks(BlockIdT fc) { _realNumBlocks = fc; } + void freeChainLength(BlockIdT fc) { _freeChainLength = fc; } size_t getMaxSize() const { return MemBlockPtrT::unAdjustSize(std::min(MemBlockPtrT::classSize(_sizeClass), size_t(_realNumBlocks) * BlockSize)); @@ -82,30 +84,36 @@ class DataSegment private: SizeClassT _sizeClass; /// Number of blocks free from here and on. For memory reuse, big blocks only. - FreeCountT _freeChainLength; + BlockIdT _freeChainLength; /// Real number of blocks used. Used to avoid rounding for big blocks. - FreeCountT _realNumBlocks; + BlockIdT _realNumBlocks; }; template class FreeListT { public: + using Index = BlockIdT; FreeListT(BlockT * blockList) __attribute__((noinline)); - void add(size_t startIndex) __attribute__((noinline)); - void * sub(size_t numBlocks) __attribute__((noinline)); - size_t lastBlock(size_t nextBlock) __attribute__((noinline)); + FreeListT(const FreeListT &) = delete; + FreeListT & operator =(const FreeListT &) = delete; + FreeListT(FreeListT &&) = delete; + FreeListT & operator =(FreeListT &&) = delete; + ~FreeListT(); + void add(Index startIndex) __attribute__((noinline)); + void * sub(Index numBlocks) __attribute__((noinline)); + Index lastBlock(Index nextBlock) __attribute__((noinline)); void removeLastBlock() { if (_count > 0) { _count--; } } - size_t numFreeBlocks() const; + Index numFreeBlocks() const; void info(FILE * os) __attribute__((noinline)); private: - void * linkOut(size_t findex, size_t left) __attribute__((noinline)); + void * linkOut(Index findex, Index left) __attribute__((noinline)); BlockT *_blockList; - size_t _count; - size_t _freeStartIndex[MaxCount]; + Index _count; + Index _freeStartIndex[MaxCount]; }; void checkAndLogBigSegment() __attribute__((noinline)); diff --git a/vespamalloc/src/vespamalloc/malloc/datasegment.hpp b/vespamalloc/src/vespamalloc/malloc/datasegment.hpp index 80a70b6b5bc8..051773df21fb 100644 --- a/vespamalloc/src/vespamalloc/malloc/datasegment.hpp +++ b/vespamalloc/src/vespamalloc/malloc/datasegment.hpp @@ -26,7 +26,7 @@ DataSegment::DataSegment() : size_t wanted(0x1000000000ul); //64G void * everything = _osMemory.reserve(wanted); if (everything) { - for (size_t i = blockId(everything), m = blockId(everything)+(wanted/BlockSize); i < m; i++) { + for (BlockIdT i = blockId(everything), m = blockId(everything) + (wanted / BlockSize); i < m; i++) { if (i > BlockCount) { abort(); } @@ -47,9 +47,9 @@ DataSegment::freeSize() const { template void * DataSegment::getBlock(size_t & oldBlockSize, SizeClassT sc) { - const size_t minBlockSize = std::max(size_t(BlockSize), _osMemory.getMinBlockSize()); + const size_t minBlockSize = std::max(BlockSize, _osMemory.getMinBlockSize()); oldBlockSize = ((oldBlockSize + (minBlockSize-1))/minBlockSize)*minBlockSize; - size_t numBlocks((oldBlockSize + (BlockSize-1))/BlockSize); + BlockIdT numBlocks((oldBlockSize + (BlockSize - 1)) / BlockSize); size_t blockSize = BlockSize * numBlocks; void * newBlock(nullptr); { @@ -58,8 +58,8 @@ void * DataSegment::getBlock(size_t & oldBlockSize, SizeClassT sc) if ( newBlock == nullptr ) { newBlock = _unMappedList.sub(numBlocks); if ( newBlock == nullptr ) { - size_t nextBlock(blockId(end())); - size_t startBlock = _freeList.lastBlock(nextBlock); + BlockIdT nextBlock = blockId(end()); + BlockIdT startBlock = _freeList.lastBlock(nextBlock); if (startBlock) { size_t adjustedBlockSize = blockSize - BlockSize*(nextBlock-startBlock); newBlock = _osMemory.get(adjustedBlockSize); @@ -91,7 +91,7 @@ void * DataSegment::getBlock(size_t & oldBlockSize, SizeClassT sc) // assumes _osMemory.get will always return a value that does not make // "i" overflow the _blockList array; this will break when hitting the // 2T address space boundary. - for (size_t i = blockId(newBlock), m = blockId(newBlock)+numBlocks; i < m; i++) { + for (BlockIdT i = blockId(newBlock), m = blockId(newBlock) + numBlocks; i < m; i++) { _blockList[i].sizeClass(sc); _blockList[i].freeChainLength(m-i); _blockList[i].realNumBlocks(m-i); @@ -129,17 +129,17 @@ void DataSegment::checkAndLogBigSegment() template void DataSegment::returnBlock(void *ptr) { - size_t bId(blockId(ptr)); + BlockIdT bId(blockId(ptr)); SizeClassT sc = _blockList[bId].sizeClass(); size_t bsz = MemBlockPtrT::classSize(sc); if (bsz >= BlockSize) { - size_t numBlocks = bsz / BlockSize; + BlockIdT numBlocks = bsz / BlockSize; if (numBlocks > _blockList[bId].realNumBlocks()) { numBlocks = _blockList[bId].realNumBlocks(); } assert(_blockList[bId].freeChainLength() >= numBlocks); if ((_unmapSize < bsz) && _osMemory.release(ptr, numBlocks*BlockSize)) { - for(size_t i=0; i < numBlocks; i++) { + for(BlockIdT i=0; i < numBlocks; i++) { BlockT & b = _blockList[bId + i]; b.sizeClass(UNMAPPED_BLOCK); b.freeChainLength(numBlocks - i); @@ -149,7 +149,7 @@ void DataSegment::returnBlock(void *ptr) _unMappedList.add(bId); } } else { - for(size_t i=0; i < numBlocks; i++) { + for(BlockIdT i=0; i < numBlocks; i++) { BlockT & b = _blockList[bId + i]; b.sizeClass(FREE_BLOCK); b.freeChainLength(numBlocks - i); @@ -349,12 +349,17 @@ DataSegment::FreeListT::FreeListT(BlockT * blockList) : template template -void DataSegment::FreeListT::add(size_t startIndex) +DataSegment::FreeListT::~FreeListT() = default; + +template +template +void +DataSegment::FreeListT::add(Index startIndex) { - size_t i(0); - size_t numBlocks(_blockList[startIndex].freeChainLength()); + Index i(0); + Index numBlocks(_blockList[startIndex].freeChainLength()); for (i=0; (i < _count) && (_freeStartIndex[i] < startIndex); i++) { } - size_t prevIndex(0), nextIndex(0); + Index prevIndex(0), nextIndex(0); BlockT * prev(nullptr), * next(nullptr); if (i > 0) { prevIndex = _freeStartIndex[i-1]; @@ -373,12 +378,12 @@ void DataSegment::FreeListT::add(size_t startIndex) // Join with freeChain that follows. _freeStartIndex[i] = startIndex; nextIndex = startIndex; - size_t oldNextCount = next->freeChainLength(); + Index oldNextCount = next->freeChainLength(); next = & _blockList[startIndex]; next->freeChainLength(oldNextCount + numBlocks); } else { // Insert. - for(size_t j=0; j < (_count-i); j++) { + for(Index j=0; j < (_count-i); j++) { _freeStartIndex[_count-j] = _freeStartIndex[_count-j-1]; } _count++; @@ -388,7 +393,7 @@ void DataSegment::FreeListT::add(size_t startIndex) if (prev && next && (prevIndex + prev->freeChainLength() == nextIndex)) { prev->freeChainLength(prev->freeChainLength() + next->freeChainLength()); _count--; - for(size_t j=i; j < _count; j++) { + for(Index j=i; j < _count; j++) { _freeStartIndex[j] = _freeStartIndex[j+1]; } _freeStartIndex[_count] = -1; @@ -397,7 +402,8 @@ void DataSegment::FreeListT::add(size_t startIndex) template template -void * DataSegment::FreeListT::sub(size_t numBlocks) +void * +DataSegment::FreeListT::sub(Index numBlocks) { void * block(nullptr); size_t bestFitIndex(_count); @@ -419,11 +425,12 @@ void * DataSegment::FreeListT::sub(size_t numBlocks) template template -size_t DataSegment::FreeListT::lastBlock(size_t nextBlock) +uint32_t +DataSegment::FreeListT::lastBlock(Index nextBlock) { - size_t lastIndex(0); + Index lastIndex(0); if (_count > 0) { - size_t index(_freeStartIndex[_count-1]); + Index index(_freeStartIndex[_count-1]); BlockT & b = _blockList[index]; if (index + b.freeChainLength() == nextBlock) { lastIndex = index; @@ -434,23 +441,25 @@ size_t DataSegment::FreeListT::lastBlock(size_t nextBloc template template -void DataSegment::FreeListT::info(FILE * os) +void +DataSegment::FreeListT::info(FILE * os) { - for (size_t i=0; i < _count; i++) { - size_t index(_freeStartIndex[i]); + for (Index i=0; i < _count; i++) { + Index index(_freeStartIndex[i]); const BlockT & b = _blockList[index]; - fprintf(os, "Free #%3ld block #%5ld chainlength %5d size %10lu\n", - i, index, b.freeChainLength(), size_t(b.freeChainLength())*BlockSize); + fprintf(os, "Free #%3d block #%5d chainlength %5d size %10lu\n", + i, index, b.freeChainLength(), b.freeChainLength()*BlockSize); } } template template -size_t DataSegment::FreeListT::numFreeBlocks() const +uint32_t +DataSegment::FreeListT::numFreeBlocks() const { - size_t freeBlockCount(0); - for (size_t i=0; i < _count; i++) { - size_t index(_freeStartIndex[i]); + Index freeBlockCount(0); + for (Index i=0; i < _count; i++) { + Index index(_freeStartIndex[i]); const BlockT & b = _blockList[index]; freeBlockCount += b.freeChainLength(); } @@ -459,17 +468,18 @@ size_t DataSegment::FreeListT::numFreeBlocks() const template template -void * DataSegment::FreeListT::linkOut(size_t findex, size_t left) +void * +DataSegment::FreeListT::linkOut(Index findex, Index left) { size_t index(_freeStartIndex[findex]); BlockT & b = _blockList[index]; - size_t startIndex = index + left; + Index startIndex = index + left; void *block = fromBlockId(startIndex); if (left > 0) { b.freeChainLength(left); } else { _count--; - for(size_t j=findex; j < (_count); j++) { + for(Index j=findex; j < (_count); j++) { _freeStartIndex[j] = _freeStartIndex[j+1]; } _freeStartIndex[_count] = -1; From a29e2add3a19bb80aeb3191202bd6850764ecdac Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Sat, 12 Feb 2022 10:57:50 +0000 Subject: [PATCH 2/2] Consistent use of types. --- vespamalloc/src/vespamalloc/malloc/datasegment.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/vespamalloc/src/vespamalloc/malloc/datasegment.h b/vespamalloc/src/vespamalloc/malloc/datasegment.h index e44f611201e2..cefa961852fd 100644 --- a/vespamalloc/src/vespamalloc/malloc/datasegment.h +++ b/vespamalloc/src/vespamalloc/malloc/datasegment.h @@ -42,7 +42,7 @@ class DataSegment checkAndLogBigSegment(); } void enableThreadSupport() { _mutex.init(); } - static uint32_t blockId(const void * ptr) { + static BlockIdT blockId(const void * ptr) { return (size_t(ptr) - Memory::getMinPreferredStartAddress())/BlockSize; } static void * fromBlockId(size_t id) { @@ -71,13 +71,13 @@ class DataSegment BlockT(SizeClassT szClass = UNUSED_BLOCK, BlockIdT numBlocks = 0) : _sizeClass(szClass), _freeChainLength(0), _realNumBlocks(numBlocks) { } - SizeClassT sizeClass() const { return _sizeClass; } + SizeClassT sizeClass() const { return _sizeClass; } BlockIdT realNumBlocks() const { return _realNumBlocks; } BlockIdT freeChainLength() const { return _freeChainLength; } - void sizeClass(SizeClassT sc) { _sizeClass = sc; } + void sizeClass(SizeClassT sc) { _sizeClass = sc; } void realNumBlocks(BlockIdT fc) { _realNumBlocks = fc; } void freeChainLength(BlockIdT fc) { _freeChainLength = fc; } - size_t getMaxSize() const { + size_t getMaxSize() const { return MemBlockPtrT::unAdjustSize(std::min(MemBlockPtrT::classSize(_sizeClass), size_t(_realNumBlocks) * BlockSize)); }