From 75783d3fb82e88220b90b5d9c898563793f22bcc Mon Sep 17 00:00:00 2001 From: Tor Egge Date: Fri, 15 Nov 2024 14:24:42 +0100 Subject: [PATCH 1/2] Move DiskIoMetrics out of IndexMetricsEntry. --- .../searchcore/proton/metrics/CMakeLists.txt | 1 + .../proton/metrics/disk_io_metrics.cpp | 44 +++++++++++++++++++ .../proton/metrics/disk_io_metrics.h | 34 ++++++++++++++ .../proton/metrics/index_metrics_entry.cpp | 32 -------------- .../proton/metrics/index_metrics_entry.h | 21 +-------- 5 files changed, 80 insertions(+), 52 deletions(-) create mode 100644 searchcore/src/vespa/searchcore/proton/metrics/disk_io_metrics.cpp create mode 100644 searchcore/src/vespa/searchcore/proton/metrics/disk_io_metrics.h diff --git a/searchcore/src/vespa/searchcore/proton/metrics/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/metrics/CMakeLists.txt index dae64398293f..6ea9a976cc4d 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/metrics/CMakeLists.txt @@ -5,6 +5,7 @@ vespa_add_library(searchcore_proton_metrics STATIC attribute_metrics_entry.cpp cache_metrics.cpp content_proton_metrics.cpp + disk_io_metrics.cpp documentdb_job_trackers.cpp documentdb_tagged_metrics.cpp document_db_commit_metrics.cpp diff --git a/searchcore/src/vespa/searchcore/proton/metrics/disk_io_metrics.cpp b/searchcore/src/vespa/searchcore/proton/metrics/disk_io_metrics.cpp new file mode 100644 index 000000000000..4cd5e4d3c35e --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/metrics/disk_io_metrics.cpp @@ -0,0 +1,44 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "disk_io_metrics.h" +#include + +using search::CacheDiskIoStats; +using search::DiskIoStats; + +namespace proton { + +namespace { + +void update_helper(metrics::LongValueMetric &metric, const DiskIoStats &stats) { + metric.addTotalValueBatch(stats.read_bytes_total(), stats.read_operations(), + stats.read_bytes_min(), stats.read_bytes_max()); +} + +} + +DiskIoMetrics::SearchMetrics::SearchMetrics(metrics::MetricSet* parent) + : MetricSet("search", {}, "The search io for a given component", parent), + _read_bytes("read_bytes", {}, "Bytes read in posting list files as part of search", this), + _cached_read_bytes("cached_read_bytes", {}, "Bytes read from posting list files cache as part of search", this) +{ +} + +DiskIoMetrics::SearchMetrics::~SearchMetrics() = default; + +void +DiskIoMetrics::SearchMetrics::update(const CacheDiskIoStats& cache_disk_io_stats) +{ + update_helper(_read_bytes, cache_disk_io_stats.read()); + update_helper(_cached_read_bytes, cache_disk_io_stats.cached_read()); +} + +DiskIoMetrics::DiskIoMetrics(metrics::MetricSet* parent) + : MetricSet("io", {}, "The disk usage for a given component", parent), + _search(this) +{ +} + +DiskIoMetrics::~DiskIoMetrics() = default; + +} diff --git a/searchcore/src/vespa/searchcore/proton/metrics/disk_io_metrics.h b/searchcore/src/vespa/searchcore/proton/metrics/disk_io_metrics.h new file mode 100644 index 000000000000..b03969aee75e --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/metrics/disk_io_metrics.h @@ -0,0 +1,34 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include +#include + +namespace search { class CacheDiskIoStats; } + +namespace proton { + +/* + * Class containing disk io metrics, e.g. per index field or + * aggregated at document type level. + */ +class DiskIoMetrics : public metrics::MetricSet { + class SearchMetrics : public metrics::MetricSet { + metrics::LongValueMetric _read_bytes; + metrics::LongValueMetric _cached_read_bytes; + public: + explicit SearchMetrics(metrics::MetricSet* parent); + ~SearchMetrics() override; + void update(const search::CacheDiskIoStats& cache_disk_io_stats); + }; + + SearchMetrics _search; + +public: + explicit DiskIoMetrics(metrics::MetricSet* parent); + ~DiskIoMetrics() override; + void update(const search::CacheDiskIoStats& cache_disk_io_stats) { _search.update(cache_disk_io_stats); } +}; + +} diff --git a/searchcore/src/vespa/searchcore/proton/metrics/index_metrics_entry.cpp b/searchcore/src/vespa/searchcore/proton/metrics/index_metrics_entry.cpp index 3c3617cf14ba..d62f22a52710 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/index_metrics_entry.cpp +++ b/searchcore/src/vespa/searchcore/proton/metrics/index_metrics_entry.cpp @@ -3,9 +3,6 @@ #include "index_metrics_entry.h" #include -using search::CacheDiskIoStats; -using search::DiskIoStats; - namespace proton { namespace { @@ -13,37 +10,8 @@ namespace { const std::string entry_name("index"); const std::string entry_description("Metrics for indexes for a given field"); -void update_helper(metrics::LongValueMetric &metric, const DiskIoStats &stats) { - metric.addTotalValueBatch(stats.read_bytes_total(), stats.read_operations(), - stats.read_bytes_min(), stats.read_bytes_max()); -} - -} - -IndexMetricsEntry::DiskIoMetrics::SearchMetrics::SearchMetrics(metrics::MetricSet* parent) - : MetricSet("search", {}, "The search io for a given component", parent), - _read_bytes("read_bytes", {}, "Bytes read in posting list files as part of search", this), - _cached_read_bytes("cached_read_bytes", {}, "Bytes read from posting list files cache as part of search", this) -{ -} - -IndexMetricsEntry::DiskIoMetrics::SearchMetrics::~SearchMetrics() = default; - -void -IndexMetricsEntry::DiskIoMetrics::SearchMetrics::update(const CacheDiskIoStats& cache_disk_io_stats) -{ - update_helper(_read_bytes, cache_disk_io_stats.read()); - update_helper(_cached_read_bytes, cache_disk_io_stats.cached_read()); } -IndexMetricsEntry::DiskIoMetrics::DiskIoMetrics(metrics::MetricSet* parent) - : MetricSet("io", {}, "The disk usage for a given component", parent), - _search(this) -{ -} - -IndexMetricsEntry::DiskIoMetrics::~DiskIoMetrics() = default; - IndexMetricsEntry::IndexMetricsEntry(const std::string& field_name) : FieldMetricsEntry(entry_name, field_name, entry_description), _disk_io(this) diff --git a/searchcore/src/vespa/searchcore/proton/metrics/index_metrics_entry.h b/searchcore/src/vespa/searchcore/proton/metrics/index_metrics_entry.h index c2783d99e0e4..03d9bb7d5506 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/index_metrics_entry.h +++ b/searchcore/src/vespa/searchcore/proton/metrics/index_metrics_entry.h @@ -2,10 +2,9 @@ #pragma once +#include "disk_io_metrics.h" #include "field_metrics_entry.h" -namespace search { class CacheDiskIoStats; } - namespace proton { /* @@ -13,24 +12,6 @@ namespace proton { * disk indexes and memory indexes. */ class IndexMetricsEntry : public FieldMetricsEntry { - class DiskIoMetrics : public metrics::MetricSet { - class SearchMetrics : public metrics::MetricSet { - metrics::LongValueMetric _read_bytes; - metrics::LongValueMetric _cached_read_bytes; - public: - explicit SearchMetrics(metrics::MetricSet* parent); - ~SearchMetrics() override; - void update(const search::CacheDiskIoStats& cache_disk_io_stats); - }; - - SearchMetrics _search; - - public: - explicit DiskIoMetrics(metrics::MetricSet* parent); - ~DiskIoMetrics() override; - void update(const search::CacheDiskIoStats& cache_disk_io_stats) { _search.update(cache_disk_io_stats); } - }; - DiskIoMetrics _disk_io; public: From 46f0ad495e9b1c28aa2d6dcc598bc57ac5d53ec3 Mon Sep 17 00:00:00 2001 From: Tor Egge Date: Fri, 15 Nov 2024 14:36:36 +0100 Subject: [PATCH 2/2] Add disk io metrics for document type. --- .../searchcore/proton/metrics/documentdb_tagged_metrics.cpp | 3 ++- .../searchcore/proton/metrics/documentdb_tagged_metrics.h | 1 + .../searchcore/proton/server/documentdb_metrics_updater.cpp | 3 +++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.cpp b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.cpp index c1e87a834b6c..af03133edcd3 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.cpp +++ b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.cpp @@ -93,7 +93,8 @@ DocumentDBTaggedMetrics::IndexMetrics::IndexMetrics(MetricSet *parent) : MetricSet("index", {}, "Index metrics (memory and disk) for this document db", parent), diskUsage("disk_usage", {}, "Disk space usage in bytes", this), memoryUsage(this), - docsInMemory("docs_in_memory", {}, "Number of documents in memory index", this) + docsInMemory("docs_in_memory", {}, "Number of documents in memory index", this), + disk_io(this) { } diff --git a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h index 6191678c2bda..66fed96263fd 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h +++ b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h @@ -97,6 +97,7 @@ struct DocumentDBTaggedMetrics : metrics::MetricSet metrics::LongValueMetric diskUsage; MemoryUsageMetrics memoryUsage; metrics::LongValueMetric docsInMemory; + DiskIoMetrics disk_io; IndexMetrics(metrics::MetricSet *parent); ~IndexMetrics() override; diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb_metrics_updater.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb_metrics_updater.cpp index 73fa84685cb0..02633cffeb36 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb_metrics_updater.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb_metrics_updater.cpp @@ -80,6 +80,7 @@ updateIndexMetrics(DocumentDBTaggedMetrics &metrics, const search::SearchableSta updateMemoryUsageMetrics(indexMetrics.memoryUsage, stats.memoryUsage(), totalStats); indexMetrics.docsInMemory.set(stats.docsInMemory()); auto& field_metrics = metrics.ready.index; + search::CacheDiskIoStats disk_io; for (auto& field : stats.get_field_stats()) { auto entry = field_metrics.get_field_metrics_entry(field.first); if (entry) { @@ -87,7 +88,9 @@ updateIndexMetrics(DocumentDBTaggedMetrics &metrics, const search::SearchableSta entry->size_on_disk.set(field.second.size_on_disk()); entry->update_disk_io(field.second.cache_disk_io_stats()); } + disk_io.merge(field.second.cache_disk_io_stats()); } + indexMetrics.disk_io.update(disk_io); } struct TempAttributeMetric