Skip to content

Commit

Permalink
Use a typedef for the block index, and reduce it from 64 to 32 bit wh…
Browse files Browse the repository at this point in the history 
…ich is plenty. Currently only 19 is used.
  • Loading branch information
@baldersheim
baldersheim committed Feb 11, 2022
1 parent 0f3e655 commit b4dc1f2
Show file tree
Hide file tree
Showing 2 changed files with 69 additions and 51 deletions.
44 changes: 26 additions & 18 deletions vespamalloc/src/vespamalloc/malloc/datasegment.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,15 +13,15 @@ template<typename MemBlockPtrT>
class DataSegment
{
public:
typedef unsigned FreeCountT;
using BlockIdT = uint32_t;
enum { UNMAPPED_BLOCK=-4, UNUSED_BLOCK=-3, FREE_BLOCK=-2, SYSTEM_BLOCK=-1, NUM_ADMIN_CLASSES=4 };
DataSegment() __attribute__((noinline));
~DataSegment() __attribute__((noinline));

void * getBlock(size_t & oldBlockSize, SizeClassT sc) __attribute__((noinline));
void returnBlock(void *ptr) __attribute__((noinline));
SizeClassT sizeClass(const void * ptr) const { return _blockList[blockId(ptr)].sizeClass(); }
bool containsPtr(const void * ptr) const { return blockId(ptr) < BlockCount; }
bool containsPtr(const void * ptr) const { return blockId(ptr) < BlockCount; }
size_t getMaxSize(const void * ptr) const { return _blockList[blockId(ptr)].getMaxSize(); }
const void * start() const { return _osMemory.getStart(); }
const void * end() const { return _osMemory.getEnd(); }
Expand All @@ -42,7 +42,7 @@ class DataSegment
checkAndLogBigSegment();
}
void enableThreadSupport() { _mutex.init(); }
static size_t blockId(const void * ptr) {
static uint32_t blockId(const void * ptr) {
return (size_t(ptr) - Memory::getMinPreferredStartAddress())/BlockSize;
}
static void * fromBlockId(size_t id) {
Expand All @@ -61,51 +61,59 @@ class DataSegment
DataSegment(const DataSegment & rhs);
DataSegment & operator = (const DataSegment & rhs);

enum { BlockSize=0x200000, BlockCount=0x80000 }; //1T
// Allow for 1T heap
static constexpr size_t BlockSize = 0x200000ul;
static constexpr BlockIdT BlockCount = 0x80000;

class BlockT
{
public:
BlockT(SizeClassT szClass = UNUSED_BLOCK, FreeCountT numBlocks = 0)
BlockT(SizeClassT szClass = UNUSED_BLOCK, BlockIdT numBlocks = 0)
: _sizeClass(szClass), _freeChainLength(0), _realNumBlocks(numBlocks)
{ }
SizeClassT sizeClass() const { return _sizeClass; }
FreeCountT realNumBlocks() const { return _realNumBlocks; }
FreeCountT freeChainLength() const { return _freeChainLength; }
BlockIdT realNumBlocks() const { return _realNumBlocks; }
BlockIdT freeChainLength() const { return _freeChainLength; }
void sizeClass(SizeClassT sc) { _sizeClass = sc; }
void realNumBlocks(FreeCountT fc) { _realNumBlocks = fc; }
void freeChainLength(FreeCountT fc) { _freeChainLength = fc; }
void realNumBlocks(BlockIdT fc) { _realNumBlocks = fc; }
void freeChainLength(BlockIdT fc) { _freeChainLength = fc; }
size_t getMaxSize() const {
return MemBlockPtrT::unAdjustSize(std::min(MemBlockPtrT::classSize(_sizeClass),
size_t(_realNumBlocks) * BlockSize));
}
private:
SizeClassT _sizeClass;
/// Number of blocks free from here and on. For memory reuse, big blocks only.
FreeCountT _freeChainLength;
BlockIdT _freeChainLength;
/// Real number of blocks used. Used to avoid rounding for big blocks.
FreeCountT _realNumBlocks;
BlockIdT _realNumBlocks;
};

template <int MaxCount>
class FreeListT {
public:
using Index = BlockIdT;
FreeListT(BlockT * blockList) __attribute__((noinline));
void add(size_t startIndex) __attribute__((noinline));
void * sub(size_t numBlocks) __attribute__((noinline));
size_t lastBlock(size_t nextBlock) __attribute__((noinline));
FreeListT(const FreeListT &) = delete;
FreeListT & operator =(const FreeListT &) = delete;
FreeListT(FreeListT &&) = delete;
FreeListT & operator =(FreeListT &&) = delete;
~FreeListT();
void add(Index startIndex) __attribute__((noinline));
void * sub(Index numBlocks) __attribute__((noinline));
Index lastBlock(Index nextBlock) __attribute__((noinline));
void removeLastBlock() {
if (_count > 0) {
_count--;
}
}
size_t numFreeBlocks() const;
Index numFreeBlocks() const;
void info(FILE * os) __attribute__((noinline));
private:
void * linkOut(size_t findex, size_t left) __attribute__((noinline));
void * linkOut(Index findex, Index left) __attribute__((noinline));
BlockT *_blockList;
size_t _count;
size_t _freeStartIndex[MaxCount];
Index _count;
Index _freeStartIndex[MaxCount];
};

void checkAndLogBigSegment() __attribute__((noinline));
Expand Down
76 changes: 43 additions & 33 deletions vespamalloc/src/vespamalloc/malloc/datasegment.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ DataSegment<MemBlockPtrT>::DataSegment() :
size_t wanted(0x1000000000ul); //64G
void * everything = _osMemory.reserve(wanted);
if (everything) {
for (size_t i = blockId(everything), m = blockId(everything)+(wanted/BlockSize); i < m; i++) {
for (BlockIdT i = blockId(everything), m = blockId(everything) + (wanted / BlockSize); i < m; i++) {
if (i > BlockCount) {
abort();
}
Expand All @@ -47,9 +47,9 @@ DataSegment<MemBlockPtrT>::freeSize() const {
template<typename MemBlockPtrT>
void * DataSegment<MemBlockPtrT>::getBlock(size_t & oldBlockSize, SizeClassT sc)
{
const size_t minBlockSize = std::max(size_t(BlockSize), _osMemory.getMinBlockSize());
const size_t minBlockSize = std::max(BlockSize, _osMemory.getMinBlockSize());
oldBlockSize = ((oldBlockSize + (minBlockSize-1))/minBlockSize)*minBlockSize;
size_t numBlocks((oldBlockSize + (BlockSize-1))/BlockSize);
BlockIdT numBlocks((oldBlockSize + (BlockSize - 1)) / BlockSize);
size_t blockSize = BlockSize * numBlocks;
void * newBlock(nullptr);
{
Expand All @@ -58,8 +58,8 @@ void * DataSegment<MemBlockPtrT>::getBlock(size_t & oldBlockSize, SizeClassT sc)
if ( newBlock == nullptr ) {
newBlock = _unMappedList.sub(numBlocks);
if ( newBlock == nullptr ) {
size_t nextBlock(blockId(end()));
size_t startBlock = _freeList.lastBlock(nextBlock);
BlockIdT nextBlock = blockId(end());
BlockIdT startBlock = _freeList.lastBlock(nextBlock);
if (startBlock) {
size_t adjustedBlockSize = blockSize - BlockSize*(nextBlock-startBlock);
newBlock = _osMemory.get(adjustedBlockSize);
Expand Down Expand Up @@ -91,7 +91,7 @@ void * DataSegment<MemBlockPtrT>::getBlock(size_t & oldBlockSize, SizeClassT sc)
// assumes _osMemory.get will always return a value that does not make
// "i" overflow the _blockList array; this will break when hitting the
// 2T address space boundary.
for (size_t i = blockId(newBlock), m = blockId(newBlock)+numBlocks; i < m; i++) {
for (BlockIdT i = blockId(newBlock), m = blockId(newBlock) + numBlocks; i < m; i++) {
_blockList[i].sizeClass(sc);
_blockList[i].freeChainLength(m-i);
_blockList[i].realNumBlocks(m-i);
Expand Down Expand Up @@ -129,17 +129,17 @@ void DataSegment<MemBlockPtrT>::checkAndLogBigSegment()
template<typename MemBlockPtrT>
void DataSegment<MemBlockPtrT>::returnBlock(void *ptr)
{
size_t bId(blockId(ptr));
BlockIdT bId(blockId(ptr));
SizeClassT sc = _blockList[bId].sizeClass();
size_t bsz = MemBlockPtrT::classSize(sc);
if (bsz >= BlockSize) {
size_t numBlocks = bsz / BlockSize;
BlockIdT numBlocks = bsz / BlockSize;
if (numBlocks > _blockList[bId].realNumBlocks()) {
numBlocks = _blockList[bId].realNumBlocks();
}
assert(_blockList[bId].freeChainLength() >= numBlocks);
if ((_unmapSize < bsz) && _osMemory.release(ptr, numBlocks*BlockSize)) {
for(size_t i=0; i < numBlocks; i++) {
for(BlockIdT i=0; i < numBlocks; i++) {
BlockT & b = _blockList[bId + i];
b.sizeClass(UNMAPPED_BLOCK);
b.freeChainLength(numBlocks - i);
Expand All @@ -149,7 +149,7 @@ void DataSegment<MemBlockPtrT>::returnBlock(void *ptr)
_unMappedList.add(bId);
}
} else {
for(size_t i=0; i < numBlocks; i++) {
for(BlockIdT i=0; i < numBlocks; i++) {
BlockT & b = _blockList[bId + i];
b.sizeClass(FREE_BLOCK);
b.freeChainLength(numBlocks - i);
Expand Down Expand Up @@ -349,12 +349,17 @@ DataSegment<MemBlockPtrT>::FreeListT<MaxCount>::FreeListT(BlockT * blockList) :

template<typename MemBlockPtrT>
template <int MaxCount>
void DataSegment<MemBlockPtrT>::FreeListT<MaxCount>::add(size_t startIndex)
DataSegment<MemBlockPtrT>::FreeListT<MaxCount>::~FreeListT() = default;

template<typename MemBlockPtrT>
template <int MaxCount>
void
DataSegment<MemBlockPtrT>::FreeListT<MaxCount>::add(Index startIndex)
{
size_t i(0);
size_t numBlocks(_blockList[startIndex].freeChainLength());
Index i(0);
Index numBlocks(_blockList[startIndex].freeChainLength());
for (i=0; (i < _count) && (_freeStartIndex[i] < startIndex); i++) { }
size_t prevIndex(0), nextIndex(0);
Index prevIndex(0), nextIndex(0);
BlockT * prev(nullptr), * next(nullptr);
if (i > 0) {
prevIndex = _freeStartIndex[i-1];
Expand All @@ -373,12 +378,12 @@ void DataSegment<MemBlockPtrT>::FreeListT<MaxCount>::add(size_t startIndex)
// Join with freeChain that follows.
_freeStartIndex[i] = startIndex;
nextIndex = startIndex;
size_t oldNextCount = next->freeChainLength();
Index oldNextCount = next->freeChainLength();
next = & _blockList[startIndex];
next->freeChainLength(oldNextCount + numBlocks);
} else {
// Insert.
for(size_t j=0; j < (_count-i); j++) {
for(Index j=0; j < (_count-i); j++) {
_freeStartIndex[_count-j] = _freeStartIndex[_count-j-1];
}
_count++;
Expand All @@ -388,7 +393,7 @@ void DataSegment<MemBlockPtrT>::FreeListT<MaxCount>::add(size_t startIndex)
if (prev && next && (prevIndex + prev->freeChainLength() == nextIndex)) {
prev->freeChainLength(prev->freeChainLength() + next->freeChainLength());
_count--;
for(size_t j=i; j < _count; j++) {
for(Index j=i; j < _count; j++) {
_freeStartIndex[j] = _freeStartIndex[j+1];
}
_freeStartIndex[_count] = -1;
Expand All @@ -397,7 +402,8 @@ void DataSegment<MemBlockPtrT>::FreeListT<MaxCount>::add(size_t startIndex)

template<typename MemBlockPtrT>
template <int MaxCount>
void * DataSegment<MemBlockPtrT>::FreeListT<MaxCount>::sub(size_t numBlocks)
void *
DataSegment<MemBlockPtrT>::FreeListT<MaxCount>::sub(Index numBlocks)
{
void * block(nullptr);
size_t bestFitIndex(_count);
Expand All @@ -419,11 +425,12 @@ void * DataSegment<MemBlockPtrT>::FreeListT<MaxCount>::sub(size_t numBlocks)

template<typename MemBlockPtrT>
template <int MaxCount>
size_t DataSegment<MemBlockPtrT>::FreeListT<MaxCount>::lastBlock(size_t nextBlock)
uint32_t
DataSegment<MemBlockPtrT>::FreeListT<MaxCount>::lastBlock(Index nextBlock)
{
size_t lastIndex(0);
Index lastIndex(0);
if (_count > 0) {
size_t index(_freeStartIndex[_count-1]);
Index index(_freeStartIndex[_count-1]);
BlockT & b = _blockList[index];
if (index + b.freeChainLength() == nextBlock) {
lastIndex = index;
Expand All @@ -434,23 +441,25 @@ size_t DataSegment<MemBlockPtrT>::FreeListT<MaxCount>::lastBlock(size_t nextBloc

template<typename MemBlockPtrT>
template <int MaxCount>
void DataSegment<MemBlockPtrT>::FreeListT<MaxCount>::info(FILE * os)
void
DataSegment<MemBlockPtrT>::FreeListT<MaxCount>::info(FILE * os)
{
for (size_t i=0; i < _count; i++) {
size_t index(_freeStartIndex[i]);
for (Index i=0; i < _count; i++) {
Index index(_freeStartIndex[i]);
const BlockT & b = _blockList[index];
fprintf(os, "Free #%3ld block #%5ld chainlength %5d size %10lu\n",
i, index, b.freeChainLength(), size_t(b.freeChainLength())*BlockSize);
fprintf(os, "Free #%3d block #%5d chainlength %5d size %10lu\n",
i, index, b.freeChainLength(), b.freeChainLength()*BlockSize);
}
}

template<typename MemBlockPtrT>
template <int MaxCount>
size_t DataSegment<MemBlockPtrT>::FreeListT<MaxCount>::numFreeBlocks() const
uint32_t
DataSegment<MemBlockPtrT>::FreeListT<MaxCount>::numFreeBlocks() const
{
size_t freeBlockCount(0);
for (size_t i=0; i < _count; i++) {
size_t index(_freeStartIndex[i]);
Index freeBlockCount(0);
for (Index i=0; i < _count; i++) {
Index index(_freeStartIndex[i]);
const BlockT & b = _blockList[index];
freeBlockCount += b.freeChainLength();
}
Expand All @@ -459,17 +468,18 @@ size_t DataSegment<MemBlockPtrT>::FreeListT<MaxCount>::numFreeBlocks() const

template<typename MemBlockPtrT>
template <int MaxCount>
void * DataSegment<MemBlockPtrT>::FreeListT<MaxCount>::linkOut(size_t findex, size_t left)
void *
DataSegment<MemBlockPtrT>::FreeListT<MaxCount>::linkOut(Index findex, Index left)
{
size_t index(_freeStartIndex[findex]);
BlockT & b = _blockList[index];
size_t startIndex = index + left;
Index startIndex = index + left;
void *block = fromBlockId(startIndex);
if (left > 0) {
b.freeChainLength(left);
} else {
_count--;
for(size_t j=findex; j < (_count); j++) {
for(Index j=findex; j < (_count); j++) {
_freeStartIndex[j] = _freeStartIndex[j+1];
}
_freeStartIndex[_count] = -1;
Expand Down

0 comments on commit b4dc1f2

Please sign in to comment.