From dd50aa1c25e9012023ef3609d0bbc7dfa2e6e26b Mon Sep 17 00:00:00 2001 From: Matt Kereczman Date: Fri, 6 Dec 2019 22:24:04 -0800 Subject: [PATCH 1/4] Add quorum value to DRBD metrics drbd_metrics.go: Added quorum boolean to metrics. drbd_metrics_test.go: Added test for quorum boolean and added some missing tests (missing from PR#106) for second resource. test/drbd.metrics: added appropriate test values for new metric. doc/metric_spec.md: added documentation for new metric. --- doc/metric_spec.md | 27 ++++++++++++++++++++------- drbd_metrics.go | 8 ++++++++ drbd_metrics_test.go | 24 ++++++++++++++++++++++++ test/drbd.metrics | 4 ++++ 4 files changed, 56 insertions(+), 7 deletions(-) diff --git a/doc/metric_spec.md b/doc/metric_spec.md index dede282..4ddce21 100644 --- a/doc/metric_spec.md +++ b/doc/metric_spec.md @@ -211,13 +211,14 @@ The DRBD subsystems collect devices stats by parsing its configuration the JSON 5. [`ha_cluster_drbd_bm_writes`](#ha_cluster_bm_writes) 6. [`ha_cluster_drbd_upper_pending`](#ha_cluster_drbd_upper_pending) 7. [`ha_cluster_drbd_lower_pending`](#ha_cluster_drbd_lower_pending) -8. [`ha_cluster_drbd_connections`](#ha_cluster_drbd_connections) -9. [`ha_cluster_drbd_connections_sync`](#ha_cluster_drbd_connections_sync) -10. [`ha_cluster_drbd_connections_received`](#ha_cluster_drbd_connections_received) -11. [`ha_cluster_drbd_connections_sent`](#ha_cluster_drbd_connections_sent) -12. [`ha_cluster_drbd_connections_pending`](#ha_cluster_drbd_connections_pending) -13. [`ha_cluster_drbd_connections_unacked`](#ha_cluster_drbd_connections_unacked) -14. [`ha_cluster_drbd_split_brain`](#ha_cluster_drbd_split_brain) +8. [`ha_cluster_drbd_quorum`](#ha_cluster_drbd_quorum) +9. [`ha_cluster_drbd_connections`](#ha_cluster_drbd_connections) +10. [`ha_cluster_drbd_connections_sync`](#ha_cluster_drbd_connections_sync) +11. [`ha_cluster_drbd_connections_received`](#ha_cluster_drbd_connections_received) +12. [`ha_cluster_drbd_connections_sent`](#ha_cluster_drbd_connections_sent) +13. [`ha_cluster_drbd_connections_pending`](#ha_cluster_drbd_connections_pending) +14. [`ha_cluster_drbd_connections_unacked`](#ha_cluster_drbd_connections_unacked) +15. [`ha_cluster_drbd_split_brain`](#ha_cluster_drbd_split_brain) ### `ha_cluster_drbd_connections` @@ -388,6 +389,18 @@ Value is an integer greater than or equal to `0`. - `resource`: the name of the resource. - `volume`: the volume number +### `ha_cluster_drbd_quorum` + +#### Description + +Quorum status of the DRBD resource according to it's configured quorum policies; 1 line per `resource`, per `volume` +Value is `1` when quorate, or `0` when inquorate. + +#### Labels + +- `resource`: the name of the resource. +- `volume`: the volume number + ### `ha_cluster_drbd_split_brain` #### Description diff --git a/drbd_metrics.go b/drbd_metrics.go index b417947..5f373d7 100644 --- a/drbd_metrics.go +++ b/drbd_metrics.go @@ -25,6 +25,7 @@ type drbdStatus struct { BmWrites int `json:"bm-writes"` UpPending int `json:"upper-pending"` LoPending int `json:"lower-pending"` + Quorum bool `json:"quorum"` DiskState string `json:"disk-state"` } `json:"devices"` Connections []struct { @@ -53,6 +54,7 @@ var ( "bm_writes": NewMetricDesc("drbd", "bm_writes", "Writes to bitmap; 1 line per res, per volume", []string{"resource", "volume"}), "upper_pending": NewMetricDesc("drbd", "upper_pending", "Upper pending; 1 line per res, per volume", []string{"resource", "volume"}), "lower_pending": NewMetricDesc("drbd", "lower_pending", "Lower pending; 1 line per res, per volume", []string{"resource", "volume"}), + "quorum": NewMetricDesc("drbd", "quorum", "Quorum status; 1 line per res, per volume", []string{"resource", "volume"}), "connections": NewMetricDesc("drbd", "connections", "The DRBD resource connections; 1 line per per resource, per peer_node_id", []string{"resource", "peer_node_id", "peer_role", "volume", "peer_disk_state"}), "connections_sync": NewMetricDesc("drbd", "connections_sync", "The in sync percentage value for DRBD resource connections", []string{"resource", "peer_node_id", "volume"}), "connections_received": NewMetricDesc("drbd", "connections_received", "KiB received per connection", []string{"resource", "peer_node_id", "volume"}), @@ -121,6 +123,12 @@ func (c *drbdCollector) Collect(ch chan<- prometheus.Metric) { ch <- c.makeGaugeMetric("upper_pending", float64(device.UpPending), resource.Name, strconv.Itoa(device.Volume)) ch <- c.makeGaugeMetric("lower_pending", float64(device.LoPending), resource.Name, strconv.Itoa(device.Volume)) + + if bool(device.Quorum) == true { + ch <- c.makeGaugeMetric("quorum", float64(1), resource.Name, strconv.Itoa(device.Volume)) + } else { + ch <- c.makeGaugeMetric("quorum", float64(0), resource.Name, strconv.Itoa(device.Volume)) + } } if len(resource.Connections) == 0 { log.Warnf("Could not retrieve connection info for resource '%s'\n", resource.Name) diff --git a/drbd_metrics_test.go b/drbd_metrics_test.go index 372f885..8547633 100644 --- a/drbd_metrics_test.go +++ b/drbd_metrics_test.go @@ -166,22 +166,46 @@ func TestDrbdParsing(t *testing.T) { t.Errorf("lower-pending should be 2") } + if true != drbdDevs[0].Devices[0].Quorum { + t.Errorf("quorum should be true") + } + + if true != drbdDevs[1].Devices[0].Quorum { + t.Errorf("quorum should be true") + } + if 456 != drbdDevs[0].Connections[0].PeerDevices[0].Received { t.Errorf("received should be 456") } + if 456 != drbdDevs[1].Connections[0].PeerDevices[0].Received { + t.Errorf("received should be 456") + } + if 654 != drbdDevs[0].Connections[0].PeerDevices[0].Sent { t.Errorf("sent should be 654") } + if 654 != drbdDevs[1].Connections[0].PeerDevices[0].Sent { + t.Errorf("sent should be 654") + } + if 3 != drbdDevs[0].Connections[0].PeerDevices[0].Pending { t.Errorf("pending should be 3") } + if 3 != drbdDevs[1].Connections[0].PeerDevices[0].Pending { + t.Errorf("pending should be 3") + } + if 4 != drbdDevs[0].Connections[0].PeerDevices[0].Unacked { t.Errorf("unacked should be 4") } + if 4 != drbdDevs[1].Connections[0].PeerDevices[0].Unacked { + t.Errorf("unacked should be 4") + } + if 100 != drbdDevs[0].Connections[0].PeerDevices[0].PercentInSync { t.Errorf("PercentInSync doesn't correspond! fail got %f", drbdDevs[0].Connections[0].PeerDevices[0].PercentInSync) } diff --git a/test/drbd.metrics b/test/drbd.metrics index c39e0e0..3c51802 100644 --- a/test/drbd.metrics +++ b/test/drbd.metrics @@ -46,6 +46,10 @@ ha_cluster_drbd_upper_pending{resource="1-single-1",volume="0"} 1 1234 # TYPE ha_cluster_drbd_lower_pending gauge ha_cluster_drbd_lower_pending{resource="1-single-0",volume="0"} 2 1234 ha_cluster_drbd_lower_pending{resource="1-single-1",volume="0"} 2 1234 +# HELP ha_cluster_drbd_quorum Quorum status; 1 line per res, per volume +# TYPE ha_cluster_drbd_quorum gauge +ha_cluster_drbd_quorum{resource="1-single-0",volume="0"} 1 1234 +ha_cluster_drbd_quorum{resource="1-single-1",volume="0"} 1 1234 # HELP ha_cluster_drbd_resources The DRBD resources; 1 line per name, per volume # TYPE ha_cluster_drbd_resources gauge ha_cluster_drbd_resources{disk_state="uptodate",resource="1-single-0",role="Secondary",volume="0"} 1 1234 From 4775461dafdf4225d4dd37f064287ebd9348bd7c Mon Sep 17 00:00:00 2001 From: Matt Kereczman Date: Sat, 7 Dec 2019 11:45:48 -0800 Subject: [PATCH 2/4] Removing changes to UTs unrelated to PR#108 drbd_metrics_test.go: I'll put the removed changes into a new PR for better history/organization as suggested by @diegoakechi. --- drbd_metrics_test.go | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/drbd_metrics_test.go b/drbd_metrics_test.go index 8547633..7234ce9 100644 --- a/drbd_metrics_test.go +++ b/drbd_metrics_test.go @@ -178,34 +178,18 @@ func TestDrbdParsing(t *testing.T) { t.Errorf("received should be 456") } - if 456 != drbdDevs[1].Connections[0].PeerDevices[0].Received { - t.Errorf("received should be 456") - } - if 654 != drbdDevs[0].Connections[0].PeerDevices[0].Sent { t.Errorf("sent should be 654") } - if 654 != drbdDevs[1].Connections[0].PeerDevices[0].Sent { - t.Errorf("sent should be 654") - } - if 3 != drbdDevs[0].Connections[0].PeerDevices[0].Pending { t.Errorf("pending should be 3") } - if 3 != drbdDevs[1].Connections[0].PeerDevices[0].Pending { - t.Errorf("pending should be 3") - } - if 4 != drbdDevs[0].Connections[0].PeerDevices[0].Unacked { t.Errorf("unacked should be 4") } - if 4 != drbdDevs[1].Connections[0].PeerDevices[0].Unacked { - t.Errorf("unacked should be 4") - } - if 100 != drbdDevs[0].Connections[0].PeerDevices[0].PercentInSync { t.Errorf("PercentInSync doesn't correspond! fail got %f", drbdDevs[0].Connections[0].PeerDevices[0].PercentInSync) } From 14f824310e0a01f1d5e3305ee812835d0149622c Mon Sep 17 00:00:00 2001 From: Matt Kereczman Date: Sat, 7 Dec 2019 11:58:07 -0800 Subject: [PATCH 3/4] Test for both quorate and inquorate DRBD resources drbd_metrics_test.go: check for quorate and inquorate DRBD resources. test/drbd.metrics: set second fake resource to inquorate. test/fake_drbdsetup.sh: set fake DRBD status to inquorate on second resource. --- drbd_metrics_test.go | 6 +++--- test/drbd.metrics | 2 +- test/fake_drbdsetup.sh | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/drbd_metrics_test.go b/drbd_metrics_test.go index 7234ce9..5d12269 100644 --- a/drbd_metrics_test.go +++ b/drbd_metrics_test.go @@ -71,7 +71,7 @@ func TestDrbdParsing(t *testing.T) { "minor": 3, "disk-state": "UpToDate", "client": false, - "quorum": true, + "quorum": false, "size": 10200, "read": 654321, "written": 123456, @@ -170,8 +170,8 @@ func TestDrbdParsing(t *testing.T) { t.Errorf("quorum should be true") } - if true != drbdDevs[1].Devices[0].Quorum { - t.Errorf("quorum should be true") + if false != drbdDevs[1].Devices[0].Quorum { + t.Errorf("quorum should be false") } if 456 != drbdDevs[0].Connections[0].PeerDevices[0].Received { diff --git a/test/drbd.metrics b/test/drbd.metrics index 3c51802..c676810 100644 --- a/test/drbd.metrics +++ b/test/drbd.metrics @@ -49,7 +49,7 @@ ha_cluster_drbd_lower_pending{resource="1-single-1",volume="0"} 2 1234 # HELP ha_cluster_drbd_quorum Quorum status; 1 line per res, per volume # TYPE ha_cluster_drbd_quorum gauge ha_cluster_drbd_quorum{resource="1-single-0",volume="0"} 1 1234 -ha_cluster_drbd_quorum{resource="1-single-1",volume="0"} 1 1234 +ha_cluster_drbd_quorum{resource="1-single-1",volume="0"} 0 1234 # HELP ha_cluster_drbd_resources The DRBD resources; 1 line per name, per volume # TYPE ha_cluster_drbd_resources gauge ha_cluster_drbd_resources{disk_state="uptodate",resource="1-single-0",role="Secondary",volume="0"} 1 1234 diff --git a/test/fake_drbdsetup.sh b/test/fake_drbdsetup.sh index 7fe8428..d959caf 100755 --- a/test/fake_drbdsetup.sh +++ b/test/fake_drbdsetup.sh @@ -65,7 +65,7 @@ cat < Date: Mon, 9 Dec 2019 08:09:32 -0800 Subject: [PATCH 4/4] Improved quorum `help` message --- drbd_metrics.go | 2 +- test/drbd.metrics | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drbd_metrics.go b/drbd_metrics.go index 5f373d7..06cd7f4 100644 --- a/drbd_metrics.go +++ b/drbd_metrics.go @@ -54,7 +54,7 @@ var ( "bm_writes": NewMetricDesc("drbd", "bm_writes", "Writes to bitmap; 1 line per res, per volume", []string{"resource", "volume"}), "upper_pending": NewMetricDesc("drbd", "upper_pending", "Upper pending; 1 line per res, per volume", []string{"resource", "volume"}), "lower_pending": NewMetricDesc("drbd", "lower_pending", "Lower pending; 1 line per res, per volume", []string{"resource", "volume"}), - "quorum": NewMetricDesc("drbd", "quorum", "Quorum status; 1 line per res, per volume", []string{"resource", "volume"}), + "quorum": NewMetricDesc("drbd", "quorum", "Quorum status per resource and per volume", []string{"resource", "volume"}), "connections": NewMetricDesc("drbd", "connections", "The DRBD resource connections; 1 line per per resource, per peer_node_id", []string{"resource", "peer_node_id", "peer_role", "volume", "peer_disk_state"}), "connections_sync": NewMetricDesc("drbd", "connections_sync", "The in sync percentage value for DRBD resource connections", []string{"resource", "peer_node_id", "volume"}), "connections_received": NewMetricDesc("drbd", "connections_received", "KiB received per connection", []string{"resource", "peer_node_id", "volume"}), diff --git a/test/drbd.metrics b/test/drbd.metrics index c676810..443ac1f 100644 --- a/test/drbd.metrics +++ b/test/drbd.metrics @@ -46,7 +46,7 @@ ha_cluster_drbd_upper_pending{resource="1-single-1",volume="0"} 1 1234 # TYPE ha_cluster_drbd_lower_pending gauge ha_cluster_drbd_lower_pending{resource="1-single-0",volume="0"} 2 1234 ha_cluster_drbd_lower_pending{resource="1-single-1",volume="0"} 2 1234 -# HELP ha_cluster_drbd_quorum Quorum status; 1 line per res, per volume +# HELP ha_cluster_drbd_quorum Quorum status per resource and per volume # TYPE ha_cluster_drbd_quorum gauge ha_cluster_drbd_quorum{resource="1-single-0",volume="0"} 1 1234 ha_cluster_drbd_quorum{resource="1-single-1",volume="0"} 0 1234