apiVersion: monitoring.coreos.com/v1 kind: PrometheusRule metadata: name: snmp-alerts namespace: monitoring labels: release: kube-prometheus-stack spec: groups: - name: snmp-targets rules: - alert: SNMPTargetDown expr: up{job=~"snmp.*"} == 0 for: 2m labels: severity: critical annotations: summary: "SNMP target down" description: "SNMP target {{ $labels.instance }} in job {{ $labels.job }} is down." - alert: SNMPScrapeSlow expr: scrape_duration_seconds{job=~"snmp.*"} > 20 for: 5m labels: severity: warning annotations: summary: "SNMP scrape is slow" description: "SNMP scrape for {{ $labels.instance }} in job {{ $labels.job }} takes more than 20 seconds." - alert: SNMPScrapeVerySlow expr: scrape_duration_seconds{job=~"snmp.*"} > 60 for: 2m labels: severity: critical annotations: summary: "SNMP scrape is very slow" description: "SNMP scrape for {{ $labels.instance }} in job {{ $labels.job }} takes more than 60 seconds." - alert: SNMPLowSamples expr: scrape_samples_scraped{job=~"snmp.*"} < 5 for: 10m labels: severity: warning annotations: summary: "SNMP scrape returns very few samples" description: "SNMP target {{ $labels.instance }} in job {{ $labels.job }} returns fewer than 5 samples." - name: snmp-interfaces rules: - alert: InterfaceAdminUpButDown expr: ifAdminStatus{job=~"snmp.*"} == 1 and ifOperStatus{job=~"snmp.*"} == 2 for: 2m labels: severity: critical annotations: summary: "Interface admin up but operationally down" description: "{{ $labels.instance }} {{ $labels.ifName }} is enabled but operationally down." - alert: InterfaceDown expr: ifOperStatus{job=~"snmp.*"} == 2 for: 5m labels: severity: warning annotations: summary: "Interface down" description: "{{ $labels.instance }} {{ $labels.ifName }} is down." - alert: InterfaceHighRXUtilization expr: ((rate(ifHCInOctets{job=~"snmp.*"}[5m]) * 8) / (ifHighSpeed{job=~"snmp.*"} * 1000000)) > 0.80 for: 5m labels: severity: warning annotations: summary: "High RX interface utilization" description: "{{ $labels.instance }} {{ $labels.ifName }} RX utilization is above 80%." - alert: InterfaceHighTXUtilization expr: ((rate(ifHCOutOctets{job=~"snmp.*"}[5m]) * 8) / (ifHighSpeed{job=~"snmp.*"} * 1000000)) > 0.80 for: 5m labels: severity: warning annotations: summary: "High TX interface utilization" description: "{{ $labels.instance }} {{ $labels.ifName }} TX utilization is above 80%." - alert: InterfaceCriticalRXUtilization expr: ((rate(ifHCInOctets{job=~"snmp.*"}[5m]) * 8) / (ifHighSpeed{job=~"snmp.*"} * 1000000)) > 0.95 for: 2m labels: severity: critical annotations: summary: "Critical RX interface utilization" description: "{{ $labels.instance }} {{ $labels.ifName }} RX utilization is above 95%." - alert: InterfaceCriticalTXUtilization expr: ((rate(ifHCOutOctets{job=~"snmp.*"}[5m]) * 8) / (ifHighSpeed{job=~"snmp.*"} * 1000000)) > 0.95 for: 2m labels: severity: critical annotations: summary: "Critical TX interface utilization" description: "{{ $labels.instance }} {{ $labels.ifName }} TX utilization is above 95%." - alert: InterfaceHighRXFixedThreshold expr: (rate(ifHCInOctets{job=~"snmp.*"}[5m]) * 8) > 800000000 for: 5m labels: severity: warning annotations: summary: "High RX traffic" description: "{{ $labels.instance }} {{ $labels.ifName }} RX traffic is above 800 Mbit/s." - alert: InterfaceHighTXFixedThreshold expr: (rate(ifHCOutOctets{job=~"snmp.*"}[5m]) * 8) > 800000000 for: 5m labels: severity: warning annotations: summary: "High TX traffic" description: "{{ $labels.instance }} {{ $labels.ifName }} TX traffic is above 800 Mbit/s." - alert: InterfaceRXErrors expr: increase(ifInErrors{job=~"snmp.*"}[10m]) > 10 for: 2m labels: severity: warning annotations: summary: "Interface RX errors" description: "{{ $labels.instance }} {{ $labels.ifName }} has more than 10 RX errors in 10 minutes." - alert: InterfaceTXErrors expr: increase(ifOutErrors{job=~"snmp.*"}[10m]) > 10 for: 2m labels: severity: warning annotations: summary: "Interface TX errors" description: "{{ $labels.instance }} {{ $labels.ifName }} has more than 10 TX errors in 10 minutes." - alert: InterfaceRXDiscards expr: increase(ifInDiscards{job=~"snmp.*"}[10m]) > 50 for: 2m labels: severity: warning annotations: summary: "Interface RX discards" description: "{{ $labels.instance }} {{ $labels.ifName }} has more than 50 RX discards in 10 minutes." - alert: InterfaceTXDiscards expr: increase(ifOutDiscards{job=~"snmp.*"}[10m]) > 50 for: 2m labels: severity: warning annotations: summary: "Interface TX discards" description: "{{ $labels.instance }} {{ $labels.ifName }} has more than 50 TX discards in 10 minutes." - alert: InterfaceBroadcastStorm expr: increase(ifHCInBroadcastPkts{job=~"snmp.*"}[5m]) > 10000 for: 2m labels: severity: warning annotations: summary: "Possible broadcast storm" description: "{{ $labels.instance }} {{ $labels.ifName }} received more than 10000 broadcast packets in 5 minutes." - alert: InterfaceMulticastStorm expr: increase(ifHCInMulticastPkts{job=~"snmp.*"}[5m]) > 50000 for: 2m labels: severity: warning annotations: summary: "Possible multicast storm" description: "{{ $labels.instance }} {{ $labels.ifName }} received more than 50000 multicast packets in 5 minutes." - alert: InterfaceNoTraffic expr: rate(ifHCInOctets{job=~"snmp.*"}[30m]) == 0 and rate(ifHCOutOctets{job=~"snmp.*"}[30m]) == 0 and ifOperStatus{job=~"snmp.*"} == 1 for: 30m labels: severity: info annotations: summary: "Interface has no traffic" description: "{{ $labels.instance }} {{ $labels.ifName }} is up but has no RX/TX traffic for 30 minutes." - alert: InterfaceCounterReset expr: resets(ifHCInOctets{job=~"snmp.*"}[15m]) > 0 or resets(ifHCOutOctets{job=~"snmp.*"}[15m]) > 0 for: 1m labels: severity: info annotations: summary: "Interface counter reset" description: "{{ $labels.instance }} {{ $labels.ifName }} interface counter reset detected." - alert: InterfaceSpeedUnknown expr: ifHighSpeed{job=~"snmp.*"} == 0 for: 10m labels: severity: info annotations: summary: "Interface speed unknown" description: "{{ $labels.instance }} {{ $labels.ifName }} reports unknown interface speed." - alert: InterfacePossibleFlapping expr: changes(ifOperStatus{job=~"snmp.*"}[10m]) > 2 for: 1m labels: severity: warning annotations: summary: "Possible interface flapping" description: "{{ $labels.instance }} {{ $labels.ifName }} changed operational state more than twice in 10 minutes."