Self-hosted system: Support disk statistics for software RAID devices

These statistics are summarized across all component disk devices and
then tracked for the parent software RAID device as one. Note that this
is only done in case these statistics are not yet set (which is the case
for the typical Linux software RAID setup).

go.mod

@@ -50,6 +50,8 @@ require (
 	gopkg.in/mcuadros/go-syslog.v2 v2.3.0
 )
 
+require github.com/prometheus/procfs v0.7.3
+
 require (
 	github.com/Azure/go-amqp v0.16.0 // indirect
 	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
@@ -60,7 +62,7 @@ require (
 	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
 	github.com/devigned/tab v0.1.1 // indirect
 	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
-	github.com/google/go-cmp v0.5.3 // indirect
+	github.com/google/go-cmp v0.5.4 // indirect
 	github.com/googleapis/gax-go/v2 v2.0.5 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.1 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
@@ -75,7 +77,7 @@ require (
 	golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect
 	golang.org/x/mod v0.3.0 // indirect
 	golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43 // indirect
-	golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 // indirect
+	golang.org/x/sync v0.0.0-20201207232520-09787c993a3a // indirect
 	golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71 // indirect
 	golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 // indirect
 	golang.org/x/text v0.3.6 // indirect

go.sum

@@ -172,8 +172,9 @@ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.3 h1:x95R7cp+rSeeqAMI2knLtQ0DKlaBhv2NrtrOvafPHRo=
 github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M=
+github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
@@ -262,6 +263,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU=
+github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/satori/go.uuid v0.0.0-20160713180306-0aa62d5ddceb h1:1r/p6yT1FfHR1+qBm7UYBPgfqCmzz/8mpNvfc+iKlfU=
 github.com/satori/go.uuid v0.0.0-20160713180306-0aa62d5ddceb/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
@@ -384,8 +387,9 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 h1:qwRHBd0NqMbJxfbotnDhm2ByMI1Shq4Y6oRJo21SGJA=
 golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs=
+golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -419,6 +423,7 @@ golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71 h1:ikCpsnYR+Ew0vu99XlDp55lGgDJdIMx3f4a18jfse/s=

input/system/selfhosted/system.go

@@ -10,6 +10,7 @@ import (
 	"github.com/pganalyze/collector/config"
 	"github.com/pganalyze/collector/state"
 	"github.com/pganalyze/collector/util"
+	"github.com/prometheus/procfs"
 	"github.com/shirou/gopsutil/cpu"
 	"github.com/shirou/gopsutil/disk"
 	"github.com/shirou/gopsutil/host"
@@ -195,6 +196,29 @@ func GetSystemState(config config.ServerConfig, logger *util.Logger) (system sta
 		}
 	}
 
+	// Remember disk components for software RAID
+	fs, err := procfs.NewFS("/proc")
+	if err != nil {
+		logger.PrintVerbose("Selfhosted/System: Could not access /proc: %s", err)
+	} else {
+		mdstats, err := fs.MDStat()
+		if err != nil {
+			logger.PrintVerbose("Selfhosted/System: Failed to get mdstat: %s", err)
+		} else {
+			for _, mdstat := range mdstats {
+				mdDisk, exists := system.Disks[mdstat.Name]
+				if exists {
+					mdDisk.ComponentDisks = mdstat.Devices
+					system.Disks[mdstat.Name] = mdDisk
+				} else {
+					system.Disks[mdstat.Name] = state.Disk{
+						ComponentDisks: mdstat.Devices,
+					}
+				}
+			}
+		}
+	}
+
 	diskPartitions, err := disk.Partitions(true)
 	if err != nil {
 		logger.PrintVerbose("Selfhosted/System: Failed to get disk partitions: %s", err)

output/transform/system.go

@@ -224,6 +224,34 @@ func transformSystem(systemState state.SystemState, diffState state.DiffState) *
 				AvgQueueSize:             diskStats.AvgQueueSize,
 				UtilizationPercent:       diskStats.UtilizationPercent,
 			})
+		} else if len(disk.ComponentDisks) > 0 {
+			summaryStats := &snapshot.DiskStatistic{DiskIdx: idx}
+			summaryStatsCount := 0
+			for _, componentDisk := range disk.ComponentDisks {
+				componentDiskStats, exists := diffState.SystemDiskStats[componentDisk]
+				if exists {
+					summaryStats.ReadOperationsPerSecond += componentDiskStats.ReadOperationsPerSecond
+					summaryStats.ReadsMergedPerSecond += componentDiskStats.ReadsMergedPerSecond
+					summaryStats.BytesReadPerSecond += componentDiskStats.BytesReadPerSecond
+					summaryStats.WriteOperationsPerSecond += componentDiskStats.WriteOperationsPerSecond
+					summaryStats.WritesMergedPerSecond += componentDiskStats.WritesMergedPerSecond
+					summaryStats.BytesWrittenPerSecond += componentDiskStats.BytesWrittenPerSecond
+
+					// These averages will be divided by the total count at the end
+					summaryStats.AvgReadLatency += componentDiskStats.AvgReadLatency
+					summaryStats.AvgWriteLatency += componentDiskStats.AvgWriteLatency
+					summaryStats.AvgQueueSize += componentDiskStats.AvgQueueSize
+					summaryStats.UtilizationPercent += componentDiskStats.UtilizationPercent
+					summaryStatsCount += 1
+				}
+			}
+			if summaryStatsCount > 0 {
+				summaryStats.AvgReadLatency = summaryStats.AvgReadLatency / float64(summaryStatsCount)
+				summaryStats.AvgWriteLatency = summaryStats.AvgWriteLatency / float64(summaryStatsCount)
+				summaryStats.AvgQueueSize = summaryStats.AvgQueueSize / int32(summaryStatsCount)
+				summaryStats.UtilizationPercent = summaryStats.UtilizationPercent / float64(summaryStatsCount)
+			}
+			system.DiskStatistics = append(system.DiskStatistics, summaryStats)
 		}
 	}
 

state/system.go

@@ -199,6 +199,8 @@ type Disk struct {
 	Scheduler       string // Linux Scheduler (noop/anticipatory/deadline/cfq)
 	ProvisionedIOPS uint32 // If applicable, how many IOPS are provisioned for this device
 	Encrypted       bool   // If applicable, is this device encrypted? (default false)
+
+	ComponentDisks []string // Identifiers for component disks (e.g. for a software RAID)
 }
 
 // DiskStats - Statistics about an individual disk device in the system