Merge pull request #11 from treydock/mmgetstate

Add mmgetstate collector and metrics

README.md

@@ -16,6 +16,7 @@ Collectors are enabled or disabled via `--collector.<name>` and `--no-collector.
 
 Name | Description | Default
 -----|-------------|--------
+mmgetstate | Collect state via mmgetstate | Enabled
 mmpmon| Collect metrics from `mmpmon` using `fs_io_s` | Enabled
 mount | Check status of GPFS mounts. | Enabled
 verbs | Test if GPFS is using verbs interface | Disabled
@@ -55,6 +56,8 @@ The following sudo config assumes `gpfs_exporter` is running as `gpfs_exporter`.
 ```
 Defaults:gpfs_exporter !syslog
 Defaults:gpfs_exporter !requiretty
+# mmgetstate collector
+gpfs_exporter ALL=(ALL) NOPASSWD:/usr/lpp/mmfs/bin/mmgetstate -Y
 # mmpmon collector
 gpfs_exporter ALL=(ALL) NOPASSWD:/usr/lpp/mmfs/bin/mmpmon -s -p
 # mmhealth collector

collectors/mmgetstate.go

@@ -0,0 +1,142 @@
+// Copyright 2020 Trey Dockendorf
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package collectors
+
+import (
+	"bytes"
+	"context"
+	"strings"
+	"time"
+
+	"github.com/go-kit/kit/log"
+	"github.com/go-kit/kit/log/level"
+	"github.com/prometheus/client_golang/prometheus"
+	"gopkg.in/alecthomas/kingpin.v2"
+)
+
+var (
+	mmgetstateTimeout = kingpin.Flag("collector.mmgetstate.timeout", "Timeout for executing mmgetstate").Default("5").Int()
+	mmgetstateStates  = []string{"active", "arbitrating", "down"}
+)
+
+type MmgetstateMetrics struct {
+	state string
+}
+
+type MmgetstateCollector struct {
+	state  *prometheus.Desc
+	logger log.Logger
+}
+
+func init() {
+	registerCollector("mmgetstate", true, NewMmgetstateCollector)
+}
+
+func NewMmgetstateCollector(logger log.Logger) Collector {
+	return &MmgetstateCollector{
+		state: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "state"),
+			"GPFS state", []string{"state"}, nil),
+		logger: logger,
+	}
+}
+
+func (c *MmgetstateCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- c.state
+}
+
+func (c *MmgetstateCollector) Collect(ch chan<- prometheus.Metric) {
+	level.Debug(c.logger).Log("msg", "Collecting mmgetstate metrics")
+	err := c.collect(ch)
+	if err != nil {
+		ch <- prometheus.MustNewConstMetric(collectError, prometheus.GaugeValue, 1, "mmgetstate")
+	} else {
+		ch <- prometheus.MustNewConstMetric(collectError, prometheus.GaugeValue, 0, "mmgetstate")
+	}
+}
+
+func (c *MmgetstateCollector) collect(ch chan<- prometheus.Metric) error {
+	collectTime := time.Now()
+	ctx, cancel := context.WithTimeout(context.Background(), time.Duration(*mmgetstateTimeout)*time.Second)
+	defer cancel()
+	out, err := mmgetstate(ctx)
+	if ctx.Err() == context.DeadlineExceeded {
+		ch <- prometheus.MustNewConstMetric(collecTimeout, prometheus.GaugeValue, 1, "mmgetstate")
+		level.Error(c.logger).Log("msg", "Timeout executing mmgetstate")
+		return nil
+	}
+	ch <- prometheus.MustNewConstMetric(collecTimeout, prometheus.GaugeValue, 0, "mmgetstate")
+	if err != nil {
+		level.Error(c.logger).Log("msg", err)
+		return err
+	}
+	metric, err := mmgetstate_parse(out)
+	if err != nil {
+		level.Error(c.logger).Log("msg", err)
+		return err
+	}
+	for _, state := range mmgetstateStates {
+		if state == metric.state {
+			ch <- prometheus.MustNewConstMetric(c.state, prometheus.GaugeValue, 1, state)
+		} else {
+			ch <- prometheus.MustNewConstMetric(c.state, prometheus.GaugeValue, 0, state)
+		}
+	}
+	if !SliceContains(mmgetstateStates, metric.state) {
+		ch <- prometheus.MustNewConstMetric(c.state, prometheus.GaugeValue, 1, "unknown")
+	} else {
+		ch <- prometheus.MustNewConstMetric(c.state, prometheus.GaugeValue, 0, "unknown")
+	}
+	ch <- prometheus.MustNewConstMetric(collectDuration, prometheus.GaugeValue, time.Since(collectTime).Seconds(), "mmgetstate")
+	return nil
+}
+
+func mmgetstate(ctx context.Context) (string, error) {
+	cmd := execCommand(ctx, "sudo", "/usr/lpp/mmfs/bin/mmgetstate", "-Y")
+	var out bytes.Buffer
+	cmd.Stdout = &out
+	err := cmd.Run()
+	if err != nil {
+		return "", err
+	}
+	return out.String(), nil
+}
+
+func mmgetstate_parse(out string) (MmgetstateMetrics, error) {
+	metric := MmgetstateMetrics{}
+	lines := strings.Split(out, "\n")
+	var headers []string
+	for _, l := range lines {
+		if !strings.HasPrefix(l, "mmgetstate") {
+			continue
+		}
+		items := strings.Split(l, ":")
+		if len(items) < 3 {
+			continue
+		}
+		var values []string
+		if items[2] == "HEADER" {
+			headers = append(headers, items...)
+			continue
+		} else {
+			values = append(values, items...)
+		}
+		for i, h := range headers {
+			switch h {
+			case "state":
+				metric.state = values[i]
+			}
+		}
+	}
+	return metric, nil
+}

collectors/mmgetstate_test.go

@@ -0,0 +1,64 @@
+// Copyright 2020 Trey Dockendorf
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package collectors
+
+import (
+	"github.com/go-kit/kit/log"
+	"github.com/prometheus/client_golang/prometheus/testutil"
+	"os/exec"
+	"strings"
+	"testing"
+)
+
+func TestParseMmgetstate(t *testing.T) {
+	execCommand = fakeExecCommand
+	mockedStdout = `
+mmgetstate::HEADER:version:reserved:reserved:nodeName:nodeNumber:state:quorum:nodesUp:totalNodes:remarks:cnfsState:
+mmgetstate::0:1:::ib-proj-nsd05.domain:11:active:4:7:1122::(undefined):
+`
+	defer func() { execCommand = exec.CommandContext }()
+	metric, err := mmgetstate_parse(mockedStdout)
+	if err != nil {
+		t.Errorf("Unexpected error: %s", err.Error())
+	}
+	if val := metric.state; val != "active" {
+		t.Errorf("Unexpected state got %s", val)
+	}
+}
+
+func TestMmgetstateCollector(t *testing.T) {
+	execCommand = fakeExecCommand
+	mockedStdout = `
+mmgetstate::HEADER:version:reserved:reserved:nodeName:nodeNumber:state:quorum:nodesUp:totalNodes:remarks:cnfsState:
+mmgetstate::0:1:::ib-proj-nsd05.domain:11:active:4:7:1122::(undefined):
+`
+	defer func() { execCommand = exec.CommandContext }()
+	metadata := `
+		# HELP gpfs_state GPFS state
+		# TYPE gpfs_state gauge`
+	expected := `
+		gpfs_state{state="active"} 1
+		gpfs_state{state="arbitrating"} 0
+		gpfs_state{state="down"} 0
+		gpfs_state{state="unknown"} 0
+	`
+	collector := NewMmgetstateCollector(log.NewNopLogger())
+	gatherers := setupGatherer(collector)
+	if val := testutil.CollectAndCount(collector); val != 7 {
+		t.Errorf("Unexpected collection count %d, expected 7", val)
+	}
+	if err := testutil.GatherAndCompare(gatherers, strings.NewReader(metadata+expected), "gpfs_state"); err != nil {
+		t.Errorf("unexpected collecting result:\n%s", err)
+	}
+}