Skip to content

Commit

Permalink
Memory map posting list files if cache size is -1.
Browse files Browse the repository at this point in the history
  • Loading branch information
@toregge
toregge committed Dec 5, 2024
1 parent 5dbc3c6 commit 2e104a8
Show file tree
Hide file tree
Showing 6 changed files with 67 additions and 53 deletions.
1 change: 1 addition & 0 deletions searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ BootstrapConfigManager::update(const ConfigSnapshot & snapshot)
tune._index._indexing._read.setFromConfig<ProtonConfig::Indexing::Read>(conf.indexing.read.io);
tune._attr._write.setFromConfig<ProtonConfig::Attribute::Write>(conf.attribute.write.io);
tune._index._search._read.setFromConfig<ProtonConfig::Search, ProtonConfig::Search::Mmap>(conf.search.io, conf.search.mmap);
tune._index._search.set_force_memory_map_posting_list(conf.index.cache.postinglist.maxbytes == -1);
tune._summary._write.setFromConfig<ProtonConfig::Summary::Write>(conf.summary.write.io);
tune._summary._seqRead.setFromConfig<ProtonConfig::Summary::Read>(conf.summary.read.io);
tune._summary._randRead.setFromConfig<ProtonConfig::Summary::Read, ProtonConfig::Summary::Read::Mmap>(conf.summary.read.io, conf.summary.read.mmap);
Expand Down
8 changes: 4 additions & 4 deletions searchcore/src/vespa/searchcore/proton/server/proton.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -166,11 +166,11 @@ void ensureWritableDir(const std::string &dirName) {
std::shared_ptr<IPostingListCache>
make_posting_list_cache(const ProtonConfig& cfg)
{
if (cfg.search.io == ProtonConfig::Search::Io::MMAP ||
(cfg.index.cache.postinglist.maxbytes == 0 && cfg.index.cache.bitvector.maxbytes == 0)) {
return {};
int64_t max_bytes = cfg.index.cache.postinglist.maxbytes;
if (max_bytes == -1) { // Force memory map posting lists, cf. BootstrapConfigManager::update
max_bytes = 0;
}
return std::make_shared<PostingListCache>(cfg.index.cache.postinglist.maxbytes, cfg.index.cache.bitvector.maxbytes);
return std::make_shared<PostingListCache>(max_bytes, cfg.index.cache.bitvector.maxbytes);
}

} // namespace <unnamed>
Expand Down
10 changes: 10 additions & 0 deletions searchlib/src/vespa/searchlib/common/tunefileinfo.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,14 @@

namespace search {

TuneFileRandRead
TuneFileRandRead::consider_force_memory_map(bool force_memory_map) const noexcept
{
TuneFileRandRead result = *this;
if (force_memory_map) {
result.setWantMemoryMap();
}
return result;
}

} // namespace search
83 changes: 46 additions & 37 deletions searchlib/src/vespa/searchlib/common/tunefileinfo.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,11 @@ class TuneFileSeqRead

public:
TuneFileSeqRead() noexcept : _tuneControl(NORMAL) { }
void setWantDirectIO() { _tuneControl = DIRECTIO; }
bool getWantDirectIO() const { return _tuneControl == DIRECTIO; }
void setWantDirectIO() noexcept { _tuneControl = DIRECTIO; }
bool getWantDirectIO() const noexcept { return _tuneControl == DIRECTIO; }

template <typename Config>
void setFromConfig(const enum Config::Io &config) {
void setFromConfig(const enum Config::Io &config) noexcept {
switch (config) {
case Config::Io::NORMAL:
_tuneControl = NORMAL;
Expand All @@ -38,11 +38,11 @@ class TuneFileSeqRead
}
}

bool operator==(const TuneFileSeqRead &rhs) const {
bool operator==(const TuneFileSeqRead &rhs) const noexcept {
return _tuneControl == rhs._tuneControl;
}

bool operator!=(const TuneFileSeqRead &rhs) const {
bool operator!=(const TuneFileSeqRead &rhs) const noexcept {
return _tuneControl != rhs._tuneControl;
}
};
Expand All @@ -63,12 +63,12 @@ class TuneFileSeqWrite

public:
TuneFileSeqWrite() noexcept : _tuneControl(NORMAL) { }
void setWantDirectIO() { _tuneControl = DIRECTIO; }
bool getWantDirectIO() const { return _tuneControl == DIRECTIO; }
bool getWantSyncWrites() const { return _tuneControl == OSYNC; }
void setWantDirectIO() noexcept { _tuneControl = DIRECTIO; }
bool getWantDirectIO() const noexcept { return _tuneControl == DIRECTIO; }
bool getWantSyncWrites() const noexcept { return _tuneControl == OSYNC; }

template <typename Config>
void setFromConfig(const enum Config::Io &config) {
void setFromConfig(const enum Config::Io &config) noexcept {
switch (config) {
case Config::Io::NORMAL:
_tuneControl = NORMAL;
Expand All @@ -85,8 +85,8 @@ class TuneFileSeqWrite
}
}

bool operator==(const TuneFileSeqWrite &rhs) const { return _tuneControl == rhs._tuneControl; }
bool operator!=(const TuneFileSeqWrite &rhs) const { return _tuneControl != rhs._tuneControl; }
bool operator==(const TuneFileSeqWrite &rhs) const noexcept { return _tuneControl == rhs._tuneControl; }
bool operator!=(const TuneFileSeqWrite &rhs) const noexcept { return _tuneControl != rhs._tuneControl; }
};


Expand All @@ -105,27 +105,28 @@ class TuneFileRandRead
_advise(0)
{ }

void setAdvise(int advise) { _advise = advise; }
void setWantMemoryMap() { _tuneControl = MMAP; }
void setWantDirectIO() { _tuneControl = DIRECTIO; }
void setWantNormal() { _tuneControl = NORMAL; }
bool getWantDirectIO() const { return _tuneControl == DIRECTIO; }
bool getWantMemoryMap() const { return _tuneControl == MMAP; }
int getMemoryMapFlags() const { return _mmapFlags; }
int getAdvise() const { return _advise; }
void setAdvise(int advise) noexcept { _advise = advise; }
void setWantMemoryMap() noexcept { _tuneControl = MMAP; }
void setWantDirectIO() noexcept { _tuneControl = DIRECTIO; }
void setWantNormal() noexcept { _tuneControl = NORMAL; }
bool getWantDirectIO() const noexcept { return _tuneControl == DIRECTIO; }
bool getWantMemoryMap() const noexcept { return _tuneControl == MMAP; }
int getMemoryMapFlags() const noexcept { return _mmapFlags; }
int getAdvise() const noexcept { return _advise; }

template <typename TuneControlConfig, typename MMapConfig>
void setFromConfig(const enum TuneControlConfig::Io & tuneControlConfig, const MMapConfig & mmapFlags);
void setFromConfig(const enum TuneControlConfig::Io & tuneControlConfig, const MMapConfig & mmapFlags) noexcept;
template <typename MMapConfig>
void setFromMmapConfig(const MMapConfig & mmapFlags);
void setFromMmapConfig(const MMapConfig & mmapFlags) noexcept;

bool operator==(const TuneFileRandRead &rhs) const {
bool operator==(const TuneFileRandRead &rhs) const noexcept {
return (_tuneControl == rhs._tuneControl) && (_mmapFlags == rhs._mmapFlags);
}

bool operator!=(const TuneFileRandRead &rhs) const {
bool operator!=(const TuneFileRandRead &rhs) const noexcept {
return (_tuneControl != rhs._tuneControl) && (_mmapFlags == rhs._mmapFlags);
}
TuneFileRandRead consider_force_memory_map(bool force_memory_map) const noexcept;
};


Expand All @@ -143,11 +144,11 @@ class TuneFileIndexing

TuneFileIndexing(const TuneFileSeqRead &r, const TuneFileSeqWrite &w) noexcept : _read(r), _write(w) { }

bool operator==(const TuneFileIndexing &rhs) const {
bool operator==(const TuneFileIndexing &rhs) const noexcept {
return _read == rhs._read && _write == rhs._write;
}

bool operator!=(const TuneFileIndexing &rhs) const {
bool operator!=(const TuneFileIndexing &rhs) const noexcept {
return _read != rhs._read || _write != rhs._write;
}
};
Expand All @@ -160,11 +161,19 @@ class TuneFileSearch
{
public:
TuneFileRandRead _read;
bool _force_memory_map_posting_list;

TuneFileSearch() noexcept : _read() { }
TuneFileSearch(const TuneFileRandRead &r) noexcept : _read(r) { }
bool operator==(const TuneFileSearch &rhs) const { return _read == rhs._read; }
bool operator!=(const TuneFileSearch &rhs) const { return _read != rhs._read; }
TuneFileSearch() noexcept : _read(), _force_memory_map_posting_list(false) { }
TuneFileSearch(const TuneFileRandRead &r) noexcept : _read(r), _force_memory_map_posting_list(false) { }
void set_force_memory_map_posting_list(bool value) noexcept { _force_memory_map_posting_list = value; }
TuneFileRandRead get_tune_file_search_posting_list() const noexcept {
return _read.consider_force_memory_map(_force_memory_map_posting_list);
}
bool operator==(const TuneFileSearch &rhs) const noexcept {
return _read == rhs._read &&
_force_memory_map_posting_list == rhs._force_memory_map_posting_list;
}
bool operator!=(const TuneFileSearch &rhs) const noexcept { return !operator==(rhs); }
};


Expand All @@ -180,11 +189,11 @@ class TuneFileIndexManager

TuneFileIndexManager() noexcept : _indexing(), _search() { }

bool operator==(const TuneFileIndexManager &rhs) const {
bool operator==(const TuneFileIndexManager &rhs) const noexcept {
return _indexing == rhs._indexing && _search == rhs._search;
}

bool operator!=(const TuneFileIndexManager &rhs) const {
bool operator!=(const TuneFileIndexManager &rhs) const noexcept {
return _indexing != rhs._indexing || _search != rhs._search;
}
};
Expand All @@ -200,11 +209,11 @@ class TuneFileAttributes

TuneFileAttributes() noexcept : _write() { }

bool operator==(const TuneFileAttributes &rhs) const {
bool operator==(const TuneFileAttributes &rhs) const noexcept {
return _write == rhs._write;
}

bool operator!=(const TuneFileAttributes &rhs) const {
bool operator!=(const TuneFileAttributes &rhs) const noexcept {
return _write != rhs._write;
}
};
Expand All @@ -222,13 +231,13 @@ class TuneFileSummary

TuneFileSummary() noexcept : _seqRead(), _write(), _randRead() { }

bool operator==(const TuneFileSummary &rhs) const {
bool operator==(const TuneFileSummary &rhs) const noexcept {
return _seqRead == rhs._seqRead &&
_write == rhs._write &&
_randRead == rhs._randRead;
}

bool operator!=(const TuneFileSummary &rhs) const {
bool operator!=(const TuneFileSummary &rhs) const noexcept {
return _seqRead != rhs._seqRead ||
_write != rhs._write ||
_randRead != rhs._randRead;
Expand All @@ -250,13 +259,13 @@ class TuneFileDocumentDB

TuneFileDocumentDB() noexcept : _index(), _attr(), _summary() { }

bool operator==(const TuneFileDocumentDB &rhs) const {
bool operator==(const TuneFileDocumentDB &rhs) const noexcept {
return _index == rhs._index &&
_attr == rhs._attr &&
_summary == rhs._summary;
}

bool operator!=(const TuneFileDocumentDB &rhs) const {
bool operator!=(const TuneFileDocumentDB &rhs) const noexcept {
return _index != rhs._index ||
_attr != rhs._attr ||
_summary != rhs._summary;
Expand Down
4 changes: 2 additions & 2 deletions searchlib/src/vespa/searchlib/common/tunefileinfo.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ namespace search {

template <typename TuneControlConfig, typename MMapConfig>
void
TuneFileRandRead::setFromConfig(const enum TuneControlConfig::Io & tuneControlConfig, const MMapConfig & mmapFlags) {
TuneFileRandRead::setFromConfig(const enum TuneControlConfig::Io & tuneControlConfig, const MMapConfig & mmapFlags) noexcept {
switch ( tuneControlConfig) {
case TuneControlConfig::Io::NORMAL: _tuneControl = NORMAL; break;
case TuneControlConfig::Io::DIRECTIO: _tuneControl = DIRECTIO; break;
Expand All @@ -22,7 +22,7 @@ TuneFileRandRead::setFromConfig(const enum TuneControlConfig::Io & tuneControlCo

template <typename MMapConfig>
void
TuneFileRandRead::setFromMmapConfig(const MMapConfig & mmapFlags) {
TuneFileRandRead::setFromMmapConfig(const MMapConfig & mmapFlags) noexcept {
for (size_t i(0), m(mmapFlags.options.size()); i < m; i++) {
#ifdef __linux__
switch (mmapFlags.options[i]) {
Expand Down
14 changes: 4 additions & 10 deletions searchlib/src/vespa/searchlib/diskindex/field_index.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,21 +138,15 @@ FieldIndex::open(const std::string& field_dir, const TuneFileSearch& tune_file_s
LOG(warning, "Could not detect format for posocc file read %s", postingName.c_str());
}
}
pFile.reset(dynamicK
? new DiskPostingFileDynamicKReal()
: new DiskPostingFileReal());
if (!pFile->open(postingName, tune_file_search._read)) {
pFile = dynamicK ? std::make_shared<DiskPostingFileDynamicKReal>() : std::make_shared<DiskPostingFileReal>();
auto tune_file_search_posting_list = tune_file_search.get_tune_file_search_posting_list();
if (!pFile->open(postingName, tune_file_search_posting_list)) {
LOG(warning, "Could not open posting list file '%s'", postingName.c_str());
return false;
}

bDict = std::make_shared<BitVectorDictionary>();
// memory map bitvectors unless bitvector cache is enabled
auto maybe_force_mmap = tune_file_search._read;
if (!_bitvector_cache_enabled) {
maybe_force_mmap.setWantMemoryMap();
}
if (!bDict->open(field_dir, maybe_force_mmap, BitVectorKeyScope::PERFIELD_WORDS)) {
if (!bDict->open(field_dir, tune_file_search._read, BitVectorKeyScope::PERFIELD_WORDS)) {
LOG(warning, "Could not open bit vector dictionary in '%s'", field_dir.c_str());
return false;
}
Expand Down

0 comments on commit 2e104a8

Please sign in to comment.