Added a BUNCH of configs.
This commit is contained in:
@@ -0,0 +1,331 @@
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: ipmi-alerts
|
||||
namespace: monitoring
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
|
||||
spec:
|
||||
groups:
|
||||
- name: ipmi-targets
|
||||
rules:
|
||||
- alert: IPMITargetDown
|
||||
expr: up{job=~"ipmi.*|.*ipmi.*"} == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "IPMI target down"
|
||||
description: "{{ $labels.instance }} IPMI scrape target is down."
|
||||
|
||||
- alert: IPMIExporterCollectorDown
|
||||
expr: ipmi_up{job=~"ipmi.*|.*ipmi.*"} == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "IPMI collector down"
|
||||
description: "{{ $labels.instance }} IPMI collector {{ $labels.collector }} is down."
|
||||
|
||||
- alert: IPMIScrapeSlow
|
||||
expr: scrape_duration_seconds{job=~"ipmi.*|.*ipmi.*"} > 30
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "IPMI scrape slow"
|
||||
description: "{{ $labels.instance }} IPMI scrape takes more than 30 seconds."
|
||||
|
||||
- alert: IPMIScrapeVerySlow
|
||||
expr: scrape_duration_seconds{job=~"ipmi.*|.*ipmi.*"} > 90
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "IPMI scrape very slow"
|
||||
description: "{{ $labels.instance }} IPMI scrape takes more than 90 seconds."
|
||||
|
||||
- alert: IPMILowSamples
|
||||
expr: scrape_samples_scraped{job=~"ipmi.*|.*ipmi.*"} < 10
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "IPMI low sample count"
|
||||
description: "{{ $labels.instance }} IPMI scrape returns fewer than 10 samples."
|
||||
|
||||
- name: ipmi-chassis
|
||||
rules:
|
||||
- alert: IPMIChassisPowerOff
|
||||
expr: ipmi_chassis_power_state{job=~"ipmi.*|.*ipmi.*"} == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Server chassis power is off"
|
||||
description: "{{ $labels.instance }} chassis power state is OFF."
|
||||
|
||||
- alert: IPMIChassisCoolingFault
|
||||
expr: ipmi_chassis_cooling_fault_state{job=~"ipmi.*|.*ipmi.*"} == 1
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Chassis cooling fault"
|
||||
description: "{{ $labels.instance }} reports chassis cooling fault."
|
||||
|
||||
- alert: IPMIChassisDriveFault
|
||||
expr: ipmi_chassis_drive_fault_state{job=~"ipmi.*|.*ipmi.*"} == 1
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Chassis drive fault"
|
||||
description: "{{ $labels.instance }} reports chassis drive fault."
|
||||
|
||||
- alert: IPMIChassisPowerControlFault
|
||||
expr: ipmi_chassis_power_control_fault_state{job=~"ipmi.*|.*ipmi.*"} == 1
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Power control fault"
|
||||
description: "{{ $labels.instance }} reports power control fault."
|
||||
|
||||
- name: ipmi-sensors
|
||||
rules:
|
||||
- alert: IPMISensorWarning
|
||||
expr: ipmi_sensor_state{job=~"ipmi.*|.*ipmi.*"} == 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "IPMI sensor warning"
|
||||
description: "{{ $labels.instance }} sensor {{ $labels.name }} reports WARNING state."
|
||||
|
||||
- alert: IPMISensorCritical
|
||||
expr: ipmi_sensor_state{job=~"ipmi.*|.*ipmi.*"} >= 2
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "IPMI sensor critical"
|
||||
description: "{{ $labels.instance }} sensor {{ $labels.name }} reports CRITICAL state."
|
||||
|
||||
- alert: IPMISensorMissingValue
|
||||
expr: ipmi_sensor_value{job=~"ipmi.*|.*ipmi.*"} != ipmi_sensor_value{job=~"ipmi.*|.*ipmi.*"}
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "IPMI sensor value is NaN"
|
||||
description: "{{ $labels.instance }} sensor {{ $labels.name }} returns NaN."
|
||||
|
||||
- name: ipmi-fans
|
||||
rules:
|
||||
- alert: IPMIFanStateWarning
|
||||
expr: ipmi_fan_speed_state{job=~"ipmi.*|.*ipmi.*"} == 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Fan warning state"
|
||||
description: "{{ $labels.instance }} fan {{ $labels.name }} reports WARNING state."
|
||||
|
||||
- alert: IPMIFanStateCritical
|
||||
expr: ipmi_fan_speed_state{job=~"ipmi.*|.*ipmi.*"} >= 2
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Fan critical state"
|
||||
description: "{{ $labels.instance }} fan {{ $labels.name }} reports CRITICAL state."
|
||||
|
||||
- alert: IPMIFanStopped
|
||||
expr: ipmi_fan_speed_rpm{job=~"ipmi.*|.*ipmi.*"} == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Fan stopped"
|
||||
description: "{{ $labels.instance }} fan {{ $labels.name }} reports 0 RPM."
|
||||
|
||||
- alert: IPMIFanLowRPM
|
||||
expr: ipmi_fan_speed_rpm{job=~"ipmi.*|.*ipmi.*"} > 0 and ipmi_fan_speed_rpm{job=~"ipmi.*|.*ipmi.*"} < 1000
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Fan low RPM"
|
||||
description: "{{ $labels.instance }} fan {{ $labels.name }} is below 1000 RPM."
|
||||
|
||||
- alert: IPMIFanVeryHighRPM
|
||||
expr: ipmi_fan_speed_rpm{job=~"ipmi.*|.*ipmi.*"} > 10000
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Fan very high RPM"
|
||||
description: "{{ $labels.instance }} fan {{ $labels.name }} is above 10000 RPM."
|
||||
|
||||
- name: ipmi-temperature
|
||||
rules:
|
||||
- alert: IPMITemperatureHigh
|
||||
expr: ipmi_temperature_celsius{job=~"ipmi.*|.*ipmi.*"} > 70
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High IPMI temperature"
|
||||
description: "{{ $labels.instance }} sensor {{ $labels.name }} is above 70°C."
|
||||
|
||||
- alert: IPMITemperatureCritical
|
||||
expr: ipmi_temperature_celsius{job=~"ipmi.*|.*ipmi.*"} > 85
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Critical IPMI temperature"
|
||||
description: "{{ $labels.instance }} sensor {{ $labels.name }} is above 85°C."
|
||||
|
||||
- alert: IPMISensorTemperatureHigh
|
||||
expr: ipmi_sensor_value{type=~"Temperature|temperature",job=~"ipmi.*|.*ipmi.*"} > 70
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High sensor temperature"
|
||||
description: "{{ $labels.instance }} sensor {{ $labels.name }} is above 70°C."
|
||||
|
||||
- alert: IPMISensorTemperatureCritical
|
||||
expr: ipmi_sensor_value{type=~"Temperature|temperature",job=~"ipmi.*|.*ipmi.*"} > 85
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Critical sensor temperature"
|
||||
description: "{{ $labels.instance }} sensor {{ $labels.name }} is above 85°C."
|
||||
|
||||
- name: ipmi-power-current-voltage
|
||||
rules:
|
||||
- alert: IPMIPowerHigh
|
||||
expr: ipmi_power_watts{job=~"ipmi.*|.*ipmi.*"} > 600
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High power usage"
|
||||
description: "{{ $labels.instance }} power usage is above 600 W."
|
||||
|
||||
- alert: IPMIPowerVeryHigh
|
||||
expr: ipmi_power_watts{job=~"ipmi.*|.*ipmi.*"} > 800
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Very high power usage"
|
||||
description: "{{ $labels.instance }} power usage is above 800 W."
|
||||
|
||||
- alert: IPMIPowerStateWarning
|
||||
expr: ipmi_power_state{job=~"ipmi.*|.*ipmi.*"} == 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Power sensor warning"
|
||||
description: "{{ $labels.instance }} power sensor {{ $labels.name }} reports WARNING state."
|
||||
|
||||
- alert: IPMIPowerStateCritical
|
||||
expr: ipmi_power_state{job=~"ipmi.*|.*ipmi.*"} >= 2
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Power sensor critical"
|
||||
description: "{{ $labels.instance }} power sensor {{ $labels.name }} reports CRITICAL state."
|
||||
|
||||
- alert: IPMICurrentStateWarning
|
||||
expr: ipmi_current_state{job=~"ipmi.*|.*ipmi.*"} == 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Current sensor warning"
|
||||
description: "{{ $labels.instance }} current sensor {{ $labels.name }} reports WARNING state."
|
||||
|
||||
- alert: IPMICurrentStateCritical
|
||||
expr: ipmi_current_state{job=~"ipmi.*|.*ipmi.*"} >= 2
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Current sensor critical"
|
||||
description: "{{ $labels.instance }} current sensor {{ $labels.name }} reports CRITICAL state."
|
||||
|
||||
- alert: IPMIVoltageStateWarning
|
||||
expr: ipmi_voltage_state{job=~"ipmi.*|.*ipmi.*"} == 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Voltage sensor warning"
|
||||
description: "{{ $labels.instance }} voltage sensor {{ $labels.name }} reports WARNING state."
|
||||
|
||||
- alert: IPMIVoltageStateCritical
|
||||
expr: ipmi_voltage_state{job=~"ipmi.*|.*ipmi.*"} >= 2
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Voltage sensor critical"
|
||||
description: "{{ $labels.instance }} voltage sensor {{ $labels.name }} reports CRITICAL state."
|
||||
|
||||
- name: ipmi-generic-thresholds
|
||||
rules:
|
||||
- alert: IPMISensorVoltageVeryLow
|
||||
expr: ipmi_sensor_value{type=~"Voltage|voltage",job=~"ipmi.*|.*ipmi.*"} > 0 and ipmi_sensor_value{type=~"Voltage|voltage",job=~"ipmi.*|.*ipmi.*"} < 0.8
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Voltage sensor very low"
|
||||
description: "{{ $labels.instance }} voltage sensor {{ $labels.name }} is very low."
|
||||
|
||||
- alert: IPMISensorVoltageVeryHigh
|
||||
expr: ipmi_sensor_value{type=~"Voltage|voltage",job=~"ipmi.*|.*ipmi.*"} > 14
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Voltage sensor very high"
|
||||
description: "{{ $labels.instance }} voltage sensor {{ $labels.name }} is very high."
|
||||
|
||||
- alert: IPMISensorCurrentHigh
|
||||
expr: ipmi_current_amperes{job=~"ipmi.*|.*ipmi.*"} > 20
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High current"
|
||||
description: "{{ $labels.instance }} current sensor {{ $labels.name }} is above 20 A."
|
||||
|
||||
- name: ipmi-bmc-info
|
||||
rules:
|
||||
- alert: IPMIBMCInfoMissing
|
||||
expr: absent(ipmi_bmc_info{job=~"ipmi.*|.*ipmi.*"})
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "IPMI BMC info missing"
|
||||
description: "No ipmi_bmc_info metrics are present."
|
||||
|
||||
- alert: IPMIBMCFirmwareUnknown
|
||||
expr: ipmi_bmc_info{firmware_revision="",job=~"ipmi.*|.*ipmi.*"} == 1
|
||||
for: 10m
|
||||
labels:
|
||||
severity: info
|
||||
annotations:
|
||||
summary: "BMC firmware revision unknown"
|
||||
description: "{{ $labels.instance }} does not report BMC firmware revision."
|
||||
Reference in New Issue
Block a user