diff --git a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.cpp index a37aa462140..0239d0d4937 100644 --- a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.cpp @@ -86,6 +86,7 @@ BootstrapConfigManager::update(const ConfigSnapshot & snapshot) tune._index._indexing._read.setFromConfig(conf.indexing.read.io); tune._attr._write.setFromConfig(conf.attribute.write.io); tune._index._search._read.setFromConfig(conf.search.io, conf.search.mmap); + tune._index._search.set_force_memory_map_posting_list(conf.index.cache.postinglist.maxbytes == -1); tune._summary._write.setFromConfig(conf.summary.write.io); tune._summary._seqRead.setFromConfig(conf.summary.read.io); tune._summary._randRead.setFromConfig(conf.summary.read.io, conf.summary.read.mmap); diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index 7a70baf9eaa..b62236c6312 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -166,11 +166,11 @@ void ensureWritableDir(const std::string &dirName) { std::shared_ptr make_posting_list_cache(const ProtonConfig& cfg) { - if (cfg.search.io == ProtonConfig::Search::Io::MMAP || - (cfg.index.cache.postinglist.maxbytes == 0 && cfg.index.cache.bitvector.maxbytes == 0)) { - return {}; + int64_t max_bytes = cfg.index.cache.postinglist.maxbytes; + if (max_bytes == -1) { // Force memory map posting lists, cf. BootstrapConfigManager::update + max_bytes = 0; } - return std::make_shared(cfg.index.cache.postinglist.maxbytes, cfg.index.cache.bitvector.maxbytes); + return std::make_shared(max_bytes, cfg.index.cache.bitvector.maxbytes); } } // namespace diff --git a/searchlib/src/vespa/searchlib/common/tunefileinfo.cpp b/searchlib/src/vespa/searchlib/common/tunefileinfo.cpp index 7c95ef8f349..a588cab0d86 100644 --- a/searchlib/src/vespa/searchlib/common/tunefileinfo.cpp +++ b/searchlib/src/vespa/searchlib/common/tunefileinfo.cpp @@ -4,4 +4,14 @@ namespace search { +TuneFileRandRead +TuneFileRandRead::consider_force_memory_map(bool force_memory_map) const noexcept +{ + TuneFileRandRead result = *this; + if (force_memory_map) { + result.setWantMemoryMap(); + } + return result; +} + } // namespace search diff --git a/searchlib/src/vespa/searchlib/common/tunefileinfo.h b/searchlib/src/vespa/searchlib/common/tunefileinfo.h index f7030f463a4..0f3a2b7812e 100644 --- a/searchlib/src/vespa/searchlib/common/tunefileinfo.h +++ b/searchlib/src/vespa/searchlib/common/tunefileinfo.h @@ -20,11 +20,11 @@ class TuneFileSeqRead public: TuneFileSeqRead() noexcept : _tuneControl(NORMAL) { } - void setWantDirectIO() { _tuneControl = DIRECTIO; } - bool getWantDirectIO() const { return _tuneControl == DIRECTIO; } + void setWantDirectIO() noexcept { _tuneControl = DIRECTIO; } + bool getWantDirectIO() const noexcept { return _tuneControl == DIRECTIO; } template - void setFromConfig(const enum Config::Io &config) { + void setFromConfig(const enum Config::Io &config) noexcept { switch (config) { case Config::Io::NORMAL: _tuneControl = NORMAL; @@ -38,11 +38,11 @@ class TuneFileSeqRead } } - bool operator==(const TuneFileSeqRead &rhs) const { + bool operator==(const TuneFileSeqRead &rhs) const noexcept { return _tuneControl == rhs._tuneControl; } - bool operator!=(const TuneFileSeqRead &rhs) const { + bool operator!=(const TuneFileSeqRead &rhs) const noexcept { return _tuneControl != rhs._tuneControl; } }; @@ -63,12 +63,12 @@ class TuneFileSeqWrite public: TuneFileSeqWrite() noexcept : _tuneControl(NORMAL) { } - void setWantDirectIO() { _tuneControl = DIRECTIO; } - bool getWantDirectIO() const { return _tuneControl == DIRECTIO; } - bool getWantSyncWrites() const { return _tuneControl == OSYNC; } + void setWantDirectIO() noexcept { _tuneControl = DIRECTIO; } + bool getWantDirectIO() const noexcept { return _tuneControl == DIRECTIO; } + bool getWantSyncWrites() const noexcept { return _tuneControl == OSYNC; } template - void setFromConfig(const enum Config::Io &config) { + void setFromConfig(const enum Config::Io &config) noexcept { switch (config) { case Config::Io::NORMAL: _tuneControl = NORMAL; @@ -85,8 +85,8 @@ class TuneFileSeqWrite } } - bool operator==(const TuneFileSeqWrite &rhs) const { return _tuneControl == rhs._tuneControl; } - bool operator!=(const TuneFileSeqWrite &rhs) const { return _tuneControl != rhs._tuneControl; } + bool operator==(const TuneFileSeqWrite &rhs) const noexcept { return _tuneControl == rhs._tuneControl; } + bool operator!=(const TuneFileSeqWrite &rhs) const noexcept { return _tuneControl != rhs._tuneControl; } }; @@ -105,27 +105,28 @@ class TuneFileRandRead _advise(0) { } - void setAdvise(int advise) { _advise = advise; } - void setWantMemoryMap() { _tuneControl = MMAP; } - void setWantDirectIO() { _tuneControl = DIRECTIO; } - void setWantNormal() { _tuneControl = NORMAL; } - bool getWantDirectIO() const { return _tuneControl == DIRECTIO; } - bool getWantMemoryMap() const { return _tuneControl == MMAP; } - int getMemoryMapFlags() const { return _mmapFlags; } - int getAdvise() const { return _advise; } + void setAdvise(int advise) noexcept { _advise = advise; } + void setWantMemoryMap() noexcept { _tuneControl = MMAP; } + void setWantDirectIO() noexcept { _tuneControl = DIRECTIO; } + void setWantNormal() noexcept { _tuneControl = NORMAL; } + bool getWantDirectIO() const noexcept { return _tuneControl == DIRECTIO; } + bool getWantMemoryMap() const noexcept { return _tuneControl == MMAP; } + int getMemoryMapFlags() const noexcept { return _mmapFlags; } + int getAdvise() const noexcept { return _advise; } template - void setFromConfig(const enum TuneControlConfig::Io & tuneControlConfig, const MMapConfig & mmapFlags); + void setFromConfig(const enum TuneControlConfig::Io & tuneControlConfig, const MMapConfig & mmapFlags) noexcept; template - void setFromMmapConfig(const MMapConfig & mmapFlags); + void setFromMmapConfig(const MMapConfig & mmapFlags) noexcept; - bool operator==(const TuneFileRandRead &rhs) const { + bool operator==(const TuneFileRandRead &rhs) const noexcept { return (_tuneControl == rhs._tuneControl) && (_mmapFlags == rhs._mmapFlags); } - bool operator!=(const TuneFileRandRead &rhs) const { + bool operator!=(const TuneFileRandRead &rhs) const noexcept { return (_tuneControl != rhs._tuneControl) && (_mmapFlags == rhs._mmapFlags); } + TuneFileRandRead consider_force_memory_map(bool force_memory_map) const noexcept; }; @@ -143,11 +144,11 @@ class TuneFileIndexing TuneFileIndexing(const TuneFileSeqRead &r, const TuneFileSeqWrite &w) noexcept : _read(r), _write(w) { } - bool operator==(const TuneFileIndexing &rhs) const { + bool operator==(const TuneFileIndexing &rhs) const noexcept { return _read == rhs._read && _write == rhs._write; } - bool operator!=(const TuneFileIndexing &rhs) const { + bool operator!=(const TuneFileIndexing &rhs) const noexcept { return _read != rhs._read || _write != rhs._write; } }; @@ -160,11 +161,19 @@ class TuneFileSearch { public: TuneFileRandRead _read; + bool _force_memory_map_posting_list; - TuneFileSearch() noexcept : _read() { } - TuneFileSearch(const TuneFileRandRead &r) noexcept : _read(r) { } - bool operator==(const TuneFileSearch &rhs) const { return _read == rhs._read; } - bool operator!=(const TuneFileSearch &rhs) const { return _read != rhs._read; } + TuneFileSearch() noexcept : _read(), _force_memory_map_posting_list(false) { } + TuneFileSearch(const TuneFileRandRead &r) noexcept : _read(r), _force_memory_map_posting_list(false) { } + void set_force_memory_map_posting_list(bool value) noexcept { _force_memory_map_posting_list = value; } + TuneFileRandRead get_tune_file_search_posting_list() const noexcept { + return _read.consider_force_memory_map(_force_memory_map_posting_list); + } + bool operator==(const TuneFileSearch &rhs) const noexcept { + return _read == rhs._read && + _force_memory_map_posting_list == rhs._force_memory_map_posting_list; + } + bool operator!=(const TuneFileSearch &rhs) const noexcept { return !operator==(rhs); } }; @@ -180,11 +189,11 @@ class TuneFileIndexManager TuneFileIndexManager() noexcept : _indexing(), _search() { } - bool operator==(const TuneFileIndexManager &rhs) const { + bool operator==(const TuneFileIndexManager &rhs) const noexcept { return _indexing == rhs._indexing && _search == rhs._search; } - bool operator!=(const TuneFileIndexManager &rhs) const { + bool operator!=(const TuneFileIndexManager &rhs) const noexcept { return _indexing != rhs._indexing || _search != rhs._search; } }; @@ -200,11 +209,11 @@ class TuneFileAttributes TuneFileAttributes() noexcept : _write() { } - bool operator==(const TuneFileAttributes &rhs) const { + bool operator==(const TuneFileAttributes &rhs) const noexcept { return _write == rhs._write; } - bool operator!=(const TuneFileAttributes &rhs) const { + bool operator!=(const TuneFileAttributes &rhs) const noexcept { return _write != rhs._write; } }; @@ -222,13 +231,13 @@ class TuneFileSummary TuneFileSummary() noexcept : _seqRead(), _write(), _randRead() { } - bool operator==(const TuneFileSummary &rhs) const { + bool operator==(const TuneFileSummary &rhs) const noexcept { return _seqRead == rhs._seqRead && _write == rhs._write && _randRead == rhs._randRead; } - bool operator!=(const TuneFileSummary &rhs) const { + bool operator!=(const TuneFileSummary &rhs) const noexcept { return _seqRead != rhs._seqRead || _write != rhs._write || _randRead != rhs._randRead; @@ -250,13 +259,13 @@ class TuneFileDocumentDB TuneFileDocumentDB() noexcept : _index(), _attr(), _summary() { } - bool operator==(const TuneFileDocumentDB &rhs) const { + bool operator==(const TuneFileDocumentDB &rhs) const noexcept { return _index == rhs._index && _attr == rhs._attr && _summary == rhs._summary; } - bool operator!=(const TuneFileDocumentDB &rhs) const { + bool operator!=(const TuneFileDocumentDB &rhs) const noexcept { return _index != rhs._index || _attr != rhs._attr || _summary != rhs._summary; diff --git a/searchlib/src/vespa/searchlib/common/tunefileinfo.hpp b/searchlib/src/vespa/searchlib/common/tunefileinfo.hpp index 5e997276bb1..615efac9d1d 100644 --- a/searchlib/src/vespa/searchlib/common/tunefileinfo.hpp +++ b/searchlib/src/vespa/searchlib/common/tunefileinfo.hpp @@ -10,7 +10,7 @@ namespace search { template void -TuneFileRandRead::setFromConfig(const enum TuneControlConfig::Io & tuneControlConfig, const MMapConfig & mmapFlags) { +TuneFileRandRead::setFromConfig(const enum TuneControlConfig::Io & tuneControlConfig, const MMapConfig & mmapFlags) noexcept { switch ( tuneControlConfig) { case TuneControlConfig::Io::NORMAL: _tuneControl = NORMAL; break; case TuneControlConfig::Io::DIRECTIO: _tuneControl = DIRECTIO; break; @@ -22,7 +22,7 @@ TuneFileRandRead::setFromConfig(const enum TuneControlConfig::Io & tuneControlCo template void -TuneFileRandRead::setFromMmapConfig(const MMapConfig & mmapFlags) { +TuneFileRandRead::setFromMmapConfig(const MMapConfig & mmapFlags) noexcept { for (size_t i(0), m(mmapFlags.options.size()); i < m; i++) { #ifdef __linux__ switch (mmapFlags.options[i]) { diff --git a/searchlib/src/vespa/searchlib/diskindex/field_index.cpp b/searchlib/src/vespa/searchlib/diskindex/field_index.cpp index 1fe90d1131b..8606227cf0f 100644 --- a/searchlib/src/vespa/searchlib/diskindex/field_index.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/field_index.cpp @@ -138,21 +138,15 @@ FieldIndex::open(const std::string& field_dir, const TuneFileSearch& tune_file_s LOG(warning, "Could not detect format for posocc file read %s", postingName.c_str()); } } - pFile.reset(dynamicK - ? new DiskPostingFileDynamicKReal() - : new DiskPostingFileReal()); - if (!pFile->open(postingName, tune_file_search._read)) { + pFile = dynamicK ? std::make_shared() : std::make_shared(); + auto tune_file_search_posting_list = tune_file_search.get_tune_file_search_posting_list(); + if (!pFile->open(postingName, tune_file_search_posting_list)) { LOG(warning, "Could not open posting list file '%s'", postingName.c_str()); return false; } bDict = std::make_shared(); - // memory map bitvectors unless bitvector cache is enabled - auto maybe_force_mmap = tune_file_search._read; - if (!_bitvector_cache_enabled) { - maybe_force_mmap.setWantMemoryMap(); - } - if (!bDict->open(field_dir, maybe_force_mmap, BitVectorKeyScope::PERFIELD_WORDS)) { + if (!bDict->open(field_dir, tune_file_search._read, BitVectorKeyScope::PERFIELD_WORDS)) { LOG(warning, "Could not open bit vector dictionary in '%s'", field_dir.c_str()); return false; }