Skip to content

Commit

Permalink
Merge pull request #21156 from vespa-engine/balder/less-templates
Browse files Browse the repository at this point in the history 
Use a typedef for the block index, and reduce it from 64 to 32 bit wh…
  • Loading branch information
@baldersheim
baldersheim authored Feb 12, 2022
2 parents a24d92d + a29e2ad commit c6e08c8
Show file tree
Hide file tree
Showing 2 changed files with 72 additions and 54 deletions.
50 changes: 29 additions & 21 deletions vespamalloc/src/vespamalloc/malloc/datasegment.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,15 +13,15 @@ template<typename MemBlockPtrT>
class DataSegment
{
public:
typedef unsigned FreeCountT;
using BlockIdT = uint32_t;
enum { UNMAPPED_BLOCK=-4, UNUSED_BLOCK=-3, FREE_BLOCK=-2, SYSTEM_BLOCK=-1, NUM_ADMIN_CLASSES=4 };
DataSegment() __attribute__((noinline));
~DataSegment() __attribute__((noinline));

void * getBlock(size_t & oldBlockSize, SizeClassT sc) __attribute__((noinline));
void returnBlock(void *ptr) __attribute__((noinline));
SizeClassT sizeClass(const void * ptr) const { return _blockList[blockId(ptr)].sizeClass(); }
bool containsPtr(const void * ptr) const { return blockId(ptr) < BlockCount; }
bool containsPtr(const void * ptr) const { return blockId(ptr) < BlockCount; }
size_t getMaxSize(const void * ptr) const { return _blockList[blockId(ptr)].getMaxSize(); }
const void * start() const { return _osMemory.getStart(); }
const void * end() const { return _osMemory.getEnd(); }
Expand All @@ -42,7 +42,7 @@ class DataSegment
checkAndLogBigSegment();
}
void enableThreadSupport() { _mutex.init(); }
static size_t blockId(const void * ptr) {
static BlockIdT blockId(const void * ptr) {
return (size_t(ptr) - Memory::getMinPreferredStartAddress())/BlockSize;
}
static void * fromBlockId(size_t id) {
Expand All @@ -61,51 +61,59 @@ class DataSegment
DataSegment(const DataSegment & rhs);
DataSegment & operator = (const DataSegment & rhs);

enum { BlockSize=0x200000, BlockCount=0x80000 }; //1T
// Allow for 1T heap
static constexpr size_t BlockSize = 0x200000ul;
static constexpr BlockIdT BlockCount = 0x80000;

class BlockT
{
public:
BlockT(SizeClassT szClass = UNUSED_BLOCK, FreeCountT numBlocks = 0)
BlockT(SizeClassT szClass = UNUSED_BLOCK, BlockIdT numBlocks = 0)
: _sizeClass(szClass), _freeChainLength(0), _realNumBlocks(numBlocks)
{ }
SizeClassT sizeClass() const { return _sizeClass; }
FreeCountT realNumBlocks() const { return _realNumBlocks; }
FreeCountT freeChainLength() const { return _freeChainLength; }
void sizeClass(SizeClassT sc) { _sizeClass = sc; }
void realNumBlocks(FreeCountT fc) { _realNumBlocks = fc; }
void freeChainLength(FreeCountT fc) { _freeChainLength = fc; }
size_t getMaxSize() const {
SizeClassT sizeClass() const { return _sizeClass; }
BlockIdT realNumBlocks() const { return _realNumBlocks; }
BlockIdT freeChainLength() const { return _freeChainLength; }
void sizeClass(SizeClassT sc) { _sizeClass = sc; }
void realNumBlocks(BlockIdT fc) { _realNumBlocks = fc; }
void freeChainLength(BlockIdT fc) { _freeChainLength = fc; }
size_t getMaxSize() const {
return MemBlockPtrT::unAdjustSize(std::min(MemBlockPtrT::classSize(_sizeClass),
size_t(_realNumBlocks) * BlockSize));
}
private:
SizeClassT _sizeClass;
/// Number of blocks free from here and on. For memory reuse, big blocks only.
FreeCountT _freeChainLength;
BlockIdT _freeChainLength;
/// Real number of blocks used. Used to avoid rounding for big blocks.
FreeCountT _realNumBlocks;
BlockIdT _realNumBlocks;
};

template <int MaxCount>
class FreeListT {
public:
using Index = BlockIdT;
FreeListT(BlockT * blockList) __attribute__((noinline));
void add(size_t startIndex) __attribute__((noinline));
void * sub(size_t numBlocks) __attribute__((noinline));
size_t lastBlock(size_t nextBlock) __attribute__((noinline));
FreeListT(const FreeListT &) = delete;
FreeListT & operator =(const FreeListT &) = delete;
FreeListT(FreeListT &&) = delete;
FreeListT & operator =(FreeListT &&) = delete;
~FreeListT();
void add(Index startIndex) __attribute__((noinline));
void * sub(Index numBlocks) __attribute__((noinline));
Index lastBlock(Index nextBlock) __attribute__((noinline));
void removeLastBlock() {
if (_count > 0) {
_count--;
}
}
size_t numFreeBlocks() const;
Index numFreeBlocks() const;
void info(FILE * os) __attribute__((noinline));
private:
void * linkOut(size_t findex, size_t left) __attribute__((noinline));
void * linkOut(Index findex, Index left) __attribute__((noinline));
BlockT *_blockList;
size_t _count;
size_t _freeStartIndex[MaxCount];
Index _count;
Index _freeStartIndex[MaxCount];
};

void checkAndLogBigSegment() __attribute__((noinline));
Expand Down
76 changes: 43 additions & 33 deletions vespamalloc/src/vespamalloc/malloc/datasegment.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ DataSegment<MemBlockPtrT>::DataSegment() :
size_t wanted(0x1000000000ul); //64G
void * everything = _osMemory.reserve(wanted);
if (everything) {
for (size_t i = blockId(everything), m = blockId(everything)+(wanted/BlockSize); i < m; i++) {
for (BlockIdT i = blockId(everything), m = blockId(everything) + (wanted / BlockSize); i < m; i++) {
if (i > BlockCount) {
abort();
}
Expand All @@ -47,9 +47,9 @@ DataSegment<MemBlockPtrT>::freeSize() const {
template<typename MemBlockPtrT>
void * DataSegment<MemBlockPtrT>::getBlock(size_t & oldBlockSize, SizeClassT sc)
{
const size_t minBlockSize = std::max(size_t(BlockSize), _osMemory.getMinBlockSize());
const size_t minBlockSize = std::max(BlockSize, _osMemory.getMinBlockSize());
oldBlockSize = ((oldBlockSize + (minBlockSize-1))/minBlockSize)*minBlockSize;
size_t numBlocks((oldBlockSize + (BlockSize-1))/BlockSize);
BlockIdT numBlocks((oldBlockSize + (BlockSize - 1)) / BlockSize);
size_t blockSize = BlockSize * numBlocks;
void * newBlock(nullptr);
{
Expand All @@ -58,8 +58,8 @@ void * DataSegment<MemBlockPtrT>::getBlock(size_t & oldBlockSize, SizeClassT sc)
if ( newBlock == nullptr ) {
newBlock = _unMappedList.sub(numBlocks);
if ( newBlock == nullptr ) {
size_t nextBlock(blockId(end()));
size_t startBlock = _freeList.lastBlock(nextBlock);
BlockIdT nextBlock = blockId(end());
BlockIdT startBlock = _freeList.lastBlock(nextBlock);
if (startBlock) {
size_t adjustedBlockSize = blockSize - BlockSize*(nextBlock-startBlock);
newBlock = _osMemory.get(adjustedBlockSize);
Expand Down Expand Up @@ -91,7 +91,7 @@ void * DataSegment<MemBlockPtrT>::getBlock(size_t & oldBlockSize, SizeClassT sc)
// assumes _osMemory.get will always return a value that does not make
// "i" overflow the _blockList array; this will break when hitting the
// 2T address space boundary.
for (size_t i = blockId(newBlock), m = blockId(newBlock)+numBlocks; i < m; i++) {
for (BlockIdT i = blockId(newBlock), m = blockId(newBlock) + numBlocks; i < m; i++) {
_blockList[i].sizeClass(sc);
_blockList[i].freeChainLength(m-i);
_blockList[i].realNumBlocks(m-i);
Expand Down Expand Up @@ -129,17 +129,17 @@ void DataSegment<MemBlockPtrT>::checkAndLogBigSegment()
template<typename MemBlockPtrT>
void DataSegment<MemBlockPtrT>::returnBlock(void *ptr)
{
size_t bId(blockId(ptr));
BlockIdT bId(blockId(ptr));
SizeClassT sc = _blockList[bId].sizeClass();
size_t bsz = MemBlockPtrT::classSize(sc);
if (bsz >= BlockSize) {
size_t numBlocks = bsz / BlockSize;
BlockIdT numBlocks = bsz / BlockSize;
if (numBlocks > _blockList[bId].realNumBlocks()) {
numBlocks = _blockList[bId].realNumBlocks();
}
assert(_blockList[bId].freeChainLength() >= numBlocks);
if ((_unmapSize < bsz) && _osMemory.release(ptr, numBlocks*BlockSize)) {
for(size_t i=0; i < numBlocks; i++) {
for(BlockIdT i=0; i < numBlocks; i++) {
BlockT & b = _blockList[bId + i];
b.sizeClass(UNMAPPED_BLOCK);
b.freeChainLength(numBlocks - i);
Expand All @@ -149,7 +149,7 @@ void DataSegment<MemBlockPtrT>::returnBlock(void *ptr)
_unMappedList.add(bId);
}
} else {
for(size_t i=0; i < numBlocks; i++) {
for(BlockIdT i=0; i < numBlocks; i++) {
BlockT & b = _blockList[bId + i];
b.sizeClass(FREE_BLOCK);
b.freeChainLength(numBlocks - i);
Expand Down Expand Up @@ -349,12 +349,17 @@ DataSegment<MemBlockPtrT>::FreeListT<MaxCount>::FreeListT(BlockT * blockList) :

template<typename MemBlockPtrT>
template <int MaxCount>
void DataSegment<MemBlockPtrT>::FreeListT<MaxCount>::add(size_t startIndex)
DataSegment<MemBlockPtrT>::FreeListT<MaxCount>::~FreeListT() = default;

template<typename MemBlockPtrT>
template <int MaxCount>
void
DataSegment<MemBlockPtrT>::FreeListT<MaxCount>::add(Index startIndex)
{
size_t i(0);
size_t numBlocks(_blockList[startIndex].freeChainLength());
Index i(0);
Index numBlocks(_blockList[startIndex].freeChainLength());
for (i=0; (i < _count) && (_freeStartIndex[i] < startIndex); i++) { }
size_t prevIndex(0), nextIndex(0);
Index prevIndex(0), nextIndex(0);
BlockT * prev(nullptr), * next(nullptr);
if (i > 0) {
prevIndex = _freeStartIndex[i-1];
Expand All @@ -373,12 +378,12 @@ void DataSegment<MemBlockPtrT>::FreeListT<MaxCount>::add(size_t startIndex)
// Join with freeChain that follows.
_freeStartIndex[i] = startIndex;
nextIndex = startIndex;
size_t oldNextCount = next->freeChainLength();
Index oldNextCount = next->freeChainLength();
next = & _blockList[startIndex];
next->freeChainLength(oldNextCount + numBlocks);
} else {
// Insert.
for(size_t j=0; j < (_count-i); j++) {
for(Index j=0; j < (_count-i); j++) {
_freeStartIndex[_count-j] = _freeStartIndex[_count-j-1];
}
_count++;
Expand All @@ -388,7 +393,7 @@ void DataSegment<MemBlockPtrT>::FreeListT<MaxCount>::add(size_t startIndex)
if (prev && next && (prevIndex + prev->freeChainLength() == nextIndex)) {
prev->freeChainLength(prev->freeChainLength() + next->freeChainLength());
_count--;
for(size_t j=i; j < _count; j++) {
for(Index j=i; j < _count; j++) {
_freeStartIndex[j] = _freeStartIndex[j+1];
}
_freeStartIndex[_count] = -1;
Expand All @@ -397,7 +402,8 @@ void DataSegment<MemBlockPtrT>::FreeListT<MaxCount>::add(size_t startIndex)

template<typename MemBlockPtrT>
template <int MaxCount>
void * DataSegment<MemBlockPtrT>::FreeListT<MaxCount>::sub(size_t numBlocks)
void *
DataSegment<MemBlockPtrT>::FreeListT<MaxCount>::sub(Index numBlocks)
{
void * block(nullptr);
size_t bestFitIndex(_count);
Expand All @@ -419,11 +425,12 @@ void * DataSegment<MemBlockPtrT>::FreeListT<MaxCount>::sub(size_t numBlocks)

template<typename MemBlockPtrT>
template <int MaxCount>
size_t DataSegment<MemBlockPtrT>::FreeListT<MaxCount>::lastBlock(size_t nextBlock)
uint32_t
DataSegment<MemBlockPtrT>::FreeListT<MaxCount>::lastBlock(Index nextBlock)
{
size_t lastIndex(0);
Index lastIndex(0);
if (_count > 0) {
size_t index(_freeStartIndex[_count-1]);
Index index(_freeStartIndex[_count-1]);
BlockT & b = _blockList[index];
if (index + b.freeChainLength() == nextBlock) {
lastIndex = index;
Expand All @@ -434,23 +441,25 @@ size_t DataSegment<MemBlockPtrT>::FreeListT<MaxCount>::lastBlock(size_t nextBloc

template<typename MemBlockPtrT>
template <int MaxCount>
void DataSegment<MemBlockPtrT>::FreeListT<MaxCount>::info(FILE * os)
void
DataSegment<MemBlockPtrT>::FreeListT<MaxCount>::info(FILE * os)
{
for (size_t i=0; i < _count; i++) {
size_t index(_freeStartIndex[i]);
for (Index i=0; i < _count; i++) {
Index index(_freeStartIndex[i]);
const BlockT & b = _blockList[index];
fprintf(os, "Free #%3ld block #%5ld chainlength %5d size %10lu\n",
i, index, b.freeChainLength(), size_t(b.freeChainLength())*BlockSize);
fprintf(os, "Free #%3d block #%5d chainlength %5d size %10lu\n",
i, index, b.freeChainLength(), b.freeChainLength()*BlockSize);
}
}

template<typename MemBlockPtrT>
template <int MaxCount>
size_t DataSegment<MemBlockPtrT>::FreeListT<MaxCount>::numFreeBlocks() const
uint32_t
DataSegment<MemBlockPtrT>::FreeListT<MaxCount>::numFreeBlocks() const
{
size_t freeBlockCount(0);
for (size_t i=0; i < _count; i++) {
size_t index(_freeStartIndex[i]);
Index freeBlockCount(0);
for (Index i=0; i < _count; i++) {
Index index(_freeStartIndex[i]);
const BlockT & b = _blockList[index];
freeBlockCount += b.freeChainLength();
}
Expand All @@ -459,17 +468,18 @@ size_t DataSegment<MemBlockPtrT>::FreeListT<MaxCount>::numFreeBlocks() const

template<typename MemBlockPtrT>
template <int MaxCount>
void * DataSegment<MemBlockPtrT>::FreeListT<MaxCount>::linkOut(size_t findex, size_t left)
void *
DataSegment<MemBlockPtrT>::FreeListT<MaxCount>::linkOut(Index findex, Index left)
{
size_t index(_freeStartIndex[findex]);
BlockT & b = _blockList[index];
size_t startIndex = index + left;
Index startIndex = index + left;
void *block = fromBlockId(startIndex);
if (left > 0) {
b.freeChainLength(left);
} else {
_count--;
for(size_t j=findex; j < (_count); j++) {
for(Index j=findex; j < (_count); j++) {
_freeStartIndex[j] = _freeStartIndex[j+1];
}
_freeStartIndex[_count] = -1;
Expand Down

0 comments on commit c6e08c8

Please sign in to comment.