kafkacluster-prometheus-autoscale.yaml

---
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  creationTimestamp: null
  labels:
    prometheus: kafka-rules
  name: kafka-alerts
spec:
  groups:
  - name: KafkaAlerts
    rules:
    - alert: BrokerOverLoaded
      expr: avg(sum by(brokerId, kafka_cr, namespace) (rate(kafka_network_requestmetrics_requests_total[15m]))) > 30
      for: 5m
      labels:
        severity: alert
      annotations:
        description: 'broker {{ $labels.brokerId }} overloaded (current value is: {{ $value }})'
        summary: 'broker overloaded'
        # If brokerConfigGroup is defined it will override scaled broker config
        # brokerConfigGroup: 'default_group'
        storageClass: 'standard'
        mountPath: '/kafkalog'
        diskSize: '2G'
        image: 'wurstmeister/kafka:2.12-2.3.0'
        command: 'upScale'
    - alert: BrokerUnderReplicated
      expr: kafka_server_replicamanager_underreplicatedpartitions > 0
      for: 5m
      labels:
        severity: alert
      annotations:
        description: 'broker {{ $labels.brokerId }} underreplicated'
        summary: 'broker underreplicated'
        storageClass: 'standard'
        mountPath: '/kafkalog'
        diskSize: '2G'
        image: 'wurstmeister/kafka:2.12-2.3.0'
        command: 'upScale'
    - alert: PartitionCountHigh
      expr: max(kafka_server_replicamanager_partitioncount)  by (namespace, kafka_cr) > 100
      for: 3m
      labels:
        severity: alert
      annotations:
        description: 'broker {{ $labels.brokerId }} has high partition count'
        summary: 'high partition count'
        storageClass: 'standard'
        mountPath: '/kafkalog'
        diskSize: '2G'
        image: 'wurstmeister/kafka:2.12-2.3.0'
        command: 'upScale'
    - alert: PartitionCountLow
      expr: min(kafka_server_replicamanager_partitioncount)  by (namespace, kafka_cr) < 5
      for: 3m
      labels:
        severity: alert
      annotations:
        description: 'broker {{ $labels.brokerId }} has low partition count'
        summary: 'low partition count'
        command: 'downScale'
    - alert: RemainingDiskSpaceLow
      expr: kubelet_volume_stats_available_bytes{persistentvolumeclaim=~"kafka-.*"} < 1 * 1000 * 1000 * 1000
      for: 2m
      labels:
        severity: alert
      annotations:
        description: 'broker {{ $labels.brokerId }} has low disk space'
        summary: 'low diskspace'
        storageClass: 'standard'
        mountPath: '/kafkalog'
        diskSize: '2G'
        command: 'addPVC'