212 lines
8.1 KiB
YAML
212 lines
8.1 KiB
YAML
apiVersion: monitoring.coreos.com/v1
|
|
kind: PrometheusRule
|
|
metadata:
|
|
name: snmp-alerts
|
|
namespace: monitoring
|
|
labels:
|
|
release: kube-prometheus-stack
|
|
|
|
spec:
|
|
groups:
|
|
- name: snmp-targets
|
|
rules:
|
|
- alert: SNMPTargetDown
|
|
expr: up{job=~"snmp.*"} == 0
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "SNMP target down"
|
|
description: "SNMP target {{ $labels.instance }} in job {{ $labels.job }} is down."
|
|
|
|
- alert: SNMPScrapeSlow
|
|
expr: scrape_duration_seconds{job=~"snmp.*"} > 20
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "SNMP scrape is slow"
|
|
description: "SNMP scrape for {{ $labels.instance }} in job {{ $labels.job }} takes more than 20 seconds."
|
|
|
|
- alert: SNMPScrapeVerySlow
|
|
expr: scrape_duration_seconds{job=~"snmp.*"} > 60
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "SNMP scrape is very slow"
|
|
description: "SNMP scrape for {{ $labels.instance }} in job {{ $labels.job }} takes more than 60 seconds."
|
|
|
|
- alert: SNMPLowSamples
|
|
expr: scrape_samples_scraped{job=~"snmp.*"} < 5
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "SNMP scrape returns very few samples"
|
|
description: "SNMP target {{ $labels.instance }} in job {{ $labels.job }} returns fewer than 5 samples."
|
|
|
|
- name: snmp-interfaces
|
|
rules:
|
|
- alert: InterfaceAdminUpButDown
|
|
expr: ifAdminStatus{job=~"snmp.*"} == 1 and ifOperStatus{job=~"snmp.*"} == 2
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Interface admin up but operationally down"
|
|
description: "{{ $labels.instance }} {{ $labels.ifName }} is enabled but operationally down."
|
|
|
|
- alert: InterfaceDown
|
|
expr: ifOperStatus{job=~"snmp.*"} == 2
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Interface down"
|
|
description: "{{ $labels.instance }} {{ $labels.ifName }} is down."
|
|
|
|
- alert: InterfaceHighRXUtilization
|
|
expr: ((rate(ifHCInOctets{job=~"snmp.*"}[5m]) * 8) / (ifHighSpeed{job=~"snmp.*"} * 1000000)) > 0.80
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "High RX interface utilization"
|
|
description: "{{ $labels.instance }} {{ $labels.ifName }} RX utilization is above 80%."
|
|
|
|
- alert: InterfaceHighTXUtilization
|
|
expr: ((rate(ifHCOutOctets{job=~"snmp.*"}[5m]) * 8) / (ifHighSpeed{job=~"snmp.*"} * 1000000)) > 0.80
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "High TX interface utilization"
|
|
description: "{{ $labels.instance }} {{ $labels.ifName }} TX utilization is above 80%."
|
|
|
|
- alert: InterfaceCriticalRXUtilization
|
|
expr: ((rate(ifHCInOctets{job=~"snmp.*"}[5m]) * 8) / (ifHighSpeed{job=~"snmp.*"} * 1000000)) > 0.95
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Critical RX interface utilization"
|
|
description: "{{ $labels.instance }} {{ $labels.ifName }} RX utilization is above 95%."
|
|
|
|
- alert: InterfaceCriticalTXUtilization
|
|
expr: ((rate(ifHCOutOctets{job=~"snmp.*"}[5m]) * 8) / (ifHighSpeed{job=~"snmp.*"} * 1000000)) > 0.95
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Critical TX interface utilization"
|
|
description: "{{ $labels.instance }} {{ $labels.ifName }} TX utilization is above 95%."
|
|
|
|
- alert: InterfaceHighRXFixedThreshold
|
|
expr: (rate(ifHCInOctets{job=~"snmp.*"}[5m]) * 8) > 800000000
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "High RX traffic"
|
|
description: "{{ $labels.instance }} {{ $labels.ifName }} RX traffic is above 800 Mbit/s."
|
|
|
|
- alert: InterfaceHighTXFixedThreshold
|
|
expr: (rate(ifHCOutOctets{job=~"snmp.*"}[5m]) * 8) > 800000000
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "High TX traffic"
|
|
description: "{{ $labels.instance }} {{ $labels.ifName }} TX traffic is above 800 Mbit/s."
|
|
|
|
- alert: InterfaceRXErrors
|
|
expr: increase(ifInErrors{job=~"snmp.*"}[10m]) > 10
|
|
for: 2m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Interface RX errors"
|
|
description: "{{ $labels.instance }} {{ $labels.ifName }} has more than 10 RX errors in 10 minutes."
|
|
|
|
- alert: InterfaceTXErrors
|
|
expr: increase(ifOutErrors{job=~"snmp.*"}[10m]) > 10
|
|
for: 2m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Interface TX errors"
|
|
description: "{{ $labels.instance }} {{ $labels.ifName }} has more than 10 TX errors in 10 minutes."
|
|
|
|
- alert: InterfaceRXDiscards
|
|
expr: increase(ifInDiscards{job=~"snmp.*"}[10m]) > 50
|
|
for: 2m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Interface RX discards"
|
|
description: "{{ $labels.instance }} {{ $labels.ifName }} has more than 50 RX discards in 10 minutes."
|
|
|
|
- alert: InterfaceTXDiscards
|
|
expr: increase(ifOutDiscards{job=~"snmp.*"}[10m]) > 50
|
|
for: 2m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Interface TX discards"
|
|
description: "{{ $labels.instance }} {{ $labels.ifName }} has more than 50 TX discards in 10 minutes."
|
|
|
|
- alert: InterfaceBroadcastStorm
|
|
expr: increase(ifHCInBroadcastPkts{job=~"snmp.*"}[5m]) > 10000
|
|
for: 2m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Possible broadcast storm"
|
|
description: "{{ $labels.instance }} {{ $labels.ifName }} received more than 10000 broadcast packets in 5 minutes."
|
|
|
|
- alert: InterfaceMulticastStorm
|
|
expr: increase(ifHCInMulticastPkts{job=~"snmp.*"}[5m]) > 50000
|
|
for: 2m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Possible multicast storm"
|
|
description: "{{ $labels.instance }} {{ $labels.ifName }} received more than 50000 multicast packets in 5 minutes."
|
|
|
|
- alert: InterfaceNoTraffic
|
|
expr: rate(ifHCInOctets{job=~"snmp.*"}[30m]) == 0 and rate(ifHCOutOctets{job=~"snmp.*"}[30m]) == 0 and ifOperStatus{job=~"snmp.*"} == 1
|
|
for: 30m
|
|
labels:
|
|
severity: info
|
|
annotations:
|
|
summary: "Interface has no traffic"
|
|
description: "{{ $labels.instance }} {{ $labels.ifName }} is up but has no RX/TX traffic for 30 minutes."
|
|
|
|
- alert: InterfaceCounterReset
|
|
expr: resets(ifHCInOctets{job=~"snmp.*"}[15m]) > 0 or resets(ifHCOutOctets{job=~"snmp.*"}[15m]) > 0
|
|
for: 1m
|
|
labels:
|
|
severity: info
|
|
annotations:
|
|
summary: "Interface counter reset"
|
|
description: "{{ $labels.instance }} {{ $labels.ifName }} interface counter reset detected."
|
|
|
|
- alert: InterfaceSpeedUnknown
|
|
expr: ifHighSpeed{job=~"snmp.*"} == 0
|
|
for: 10m
|
|
labels:
|
|
severity: info
|
|
annotations:
|
|
summary: "Interface speed unknown"
|
|
description: "{{ $labels.instance }} {{ $labels.ifName }} reports unknown interface speed."
|
|
|
|
- alert: InterfacePossibleFlapping
|
|
expr: changes(ifOperStatus{job=~"snmp.*"}[10m]) > 2
|
|
for: 1m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Possible interface flapping"
|
|
description: "{{ $labels.instance }} {{ $labels.ifName }} changed operational state more than twice in 10 minutes."
|