332 lines
12 KiB
YAML
332 lines
12 KiB
YAML
apiVersion: monitoring.coreos.com/v1
|
|
kind: PrometheusRule
|
|
metadata:
|
|
name: ipmi-alerts
|
|
namespace: monitoring
|
|
labels:
|
|
release: kube-prometheus-stack
|
|
|
|
spec:
|
|
groups:
|
|
- name: ipmi-targets
|
|
rules:
|
|
- alert: IPMITargetDown
|
|
expr: up{job=~"ipmi.*|.*ipmi.*"} == 0
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "IPMI target down"
|
|
description: "{{ $labels.instance }} IPMI scrape target is down."
|
|
|
|
- alert: IPMIExporterCollectorDown
|
|
expr: ipmi_up{job=~"ipmi.*|.*ipmi.*"} == 0
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "IPMI collector down"
|
|
description: "{{ $labels.instance }} IPMI collector {{ $labels.collector }} is down."
|
|
|
|
- alert: IPMIScrapeSlow
|
|
expr: scrape_duration_seconds{job=~"ipmi.*|.*ipmi.*"} > 30
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "IPMI scrape slow"
|
|
description: "{{ $labels.instance }} IPMI scrape takes more than 30 seconds."
|
|
|
|
- alert: IPMIScrapeVerySlow
|
|
expr: scrape_duration_seconds{job=~"ipmi.*|.*ipmi.*"} > 90
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "IPMI scrape very slow"
|
|
description: "{{ $labels.instance }} IPMI scrape takes more than 90 seconds."
|
|
|
|
- alert: IPMILowSamples
|
|
expr: scrape_samples_scraped{job=~"ipmi.*|.*ipmi.*"} < 10
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "IPMI low sample count"
|
|
description: "{{ $labels.instance }} IPMI scrape returns fewer than 10 samples."
|
|
|
|
- name: ipmi-chassis
|
|
rules:
|
|
- alert: IPMIChassisPowerOff
|
|
expr: ipmi_chassis_power_state{job=~"ipmi.*|.*ipmi.*"} == 0
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Server chassis power is off"
|
|
description: "{{ $labels.instance }} chassis power state is OFF."
|
|
|
|
- alert: IPMIChassisCoolingFault
|
|
expr: ipmi_chassis_cooling_fault_state{job=~"ipmi.*|.*ipmi.*"} == 1
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Chassis cooling fault"
|
|
description: "{{ $labels.instance }} reports chassis cooling fault."
|
|
|
|
- alert: IPMIChassisDriveFault
|
|
expr: ipmi_chassis_drive_fault_state{job=~"ipmi.*|.*ipmi.*"} == 1
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Chassis drive fault"
|
|
description: "{{ $labels.instance }} reports chassis drive fault."
|
|
|
|
- alert: IPMIChassisPowerControlFault
|
|
expr: ipmi_chassis_power_control_fault_state{job=~"ipmi.*|.*ipmi.*"} == 1
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Power control fault"
|
|
description: "{{ $labels.instance }} reports power control fault."
|
|
|
|
- name: ipmi-sensors
|
|
rules:
|
|
- alert: IPMISensorWarning
|
|
expr: ipmi_sensor_state{job=~"ipmi.*|.*ipmi.*"} == 1
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "IPMI sensor warning"
|
|
description: "{{ $labels.instance }} sensor {{ $labels.name }} reports WARNING state."
|
|
|
|
- alert: IPMISensorCritical
|
|
expr: ipmi_sensor_state{job=~"ipmi.*|.*ipmi.*"} >= 2
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "IPMI sensor critical"
|
|
description: "{{ $labels.instance }} sensor {{ $labels.name }} reports CRITICAL state."
|
|
|
|
- alert: IPMISensorMissingValue
|
|
expr: ipmi_sensor_value{job=~"ipmi.*|.*ipmi.*"} != ipmi_sensor_value{job=~"ipmi.*|.*ipmi.*"}
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "IPMI sensor value is NaN"
|
|
description: "{{ $labels.instance }} sensor {{ $labels.name }} returns NaN."
|
|
|
|
- name: ipmi-fans
|
|
rules:
|
|
- alert: IPMIFanStateWarning
|
|
expr: ipmi_fan_speed_state{job=~"ipmi.*|.*ipmi.*"} == 1
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Fan warning state"
|
|
description: "{{ $labels.instance }} fan {{ $labels.name }} reports WARNING state."
|
|
|
|
- alert: IPMIFanStateCritical
|
|
expr: ipmi_fan_speed_state{job=~"ipmi.*|.*ipmi.*"} >= 2
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Fan critical state"
|
|
description: "{{ $labels.instance }} fan {{ $labels.name }} reports CRITICAL state."
|
|
|
|
- alert: IPMIFanStopped
|
|
expr: ipmi_fan_speed_rpm{job=~"ipmi.*|.*ipmi.*"} == 0
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Fan stopped"
|
|
description: "{{ $labels.instance }} fan {{ $labels.name }} reports 0 RPM."
|
|
|
|
- alert: IPMIFanLowRPM
|
|
expr: ipmi_fan_speed_rpm{job=~"ipmi.*|.*ipmi.*"} > 0 and ipmi_fan_speed_rpm{job=~"ipmi.*|.*ipmi.*"} < 1000
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Fan low RPM"
|
|
description: "{{ $labels.instance }} fan {{ $labels.name }} is below 1000 RPM."
|
|
|
|
- alert: IPMIFanVeryHighRPM
|
|
expr: ipmi_fan_speed_rpm{job=~"ipmi.*|.*ipmi.*"} > 10000
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Fan very high RPM"
|
|
description: "{{ $labels.instance }} fan {{ $labels.name }} is above 10000 RPM."
|
|
|
|
- name: ipmi-temperature
|
|
rules:
|
|
- alert: IPMITemperatureHigh
|
|
expr: ipmi_temperature_celsius{job=~"ipmi.*|.*ipmi.*"} > 70
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "High IPMI temperature"
|
|
description: "{{ $labels.instance }} sensor {{ $labels.name }} is above 70°C."
|
|
|
|
- alert: IPMITemperatureCritical
|
|
expr: ipmi_temperature_celsius{job=~"ipmi.*|.*ipmi.*"} > 85
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Critical IPMI temperature"
|
|
description: "{{ $labels.instance }} sensor {{ $labels.name }} is above 85°C."
|
|
|
|
- alert: IPMISensorTemperatureHigh
|
|
expr: ipmi_sensor_value{type=~"Temperature|temperature",job=~"ipmi.*|.*ipmi.*"} > 70
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "High sensor temperature"
|
|
description: "{{ $labels.instance }} sensor {{ $labels.name }} is above 70°C."
|
|
|
|
- alert: IPMISensorTemperatureCritical
|
|
expr: ipmi_sensor_value{type=~"Temperature|temperature",job=~"ipmi.*|.*ipmi.*"} > 85
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Critical sensor temperature"
|
|
description: "{{ $labels.instance }} sensor {{ $labels.name }} is above 85°C."
|
|
|
|
- name: ipmi-power-current-voltage
|
|
rules:
|
|
- alert: IPMIPowerHigh
|
|
expr: ipmi_power_watts{job=~"ipmi.*|.*ipmi.*"} > 600
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "High power usage"
|
|
description: "{{ $labels.instance }} power usage is above 600 W."
|
|
|
|
- alert: IPMIPowerVeryHigh
|
|
expr: ipmi_power_watts{job=~"ipmi.*|.*ipmi.*"} > 800
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Very high power usage"
|
|
description: "{{ $labels.instance }} power usage is above 800 W."
|
|
|
|
- alert: IPMIPowerStateWarning
|
|
expr: ipmi_power_state{job=~"ipmi.*|.*ipmi.*"} == 1
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Power sensor warning"
|
|
description: "{{ $labels.instance }} power sensor {{ $labels.name }} reports WARNING state."
|
|
|
|
- alert: IPMIPowerStateCritical
|
|
expr: ipmi_power_state{job=~"ipmi.*|.*ipmi.*"} >= 2
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Power sensor critical"
|
|
description: "{{ $labels.instance }} power sensor {{ $labels.name }} reports CRITICAL state."
|
|
|
|
- alert: IPMICurrentStateWarning
|
|
expr: ipmi_current_state{job=~"ipmi.*|.*ipmi.*"} == 1
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Current sensor warning"
|
|
description: "{{ $labels.instance }} current sensor {{ $labels.name }} reports WARNING state."
|
|
|
|
- alert: IPMICurrentStateCritical
|
|
expr: ipmi_current_state{job=~"ipmi.*|.*ipmi.*"} >= 2
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Current sensor critical"
|
|
description: "{{ $labels.instance }} current sensor {{ $labels.name }} reports CRITICAL state."
|
|
|
|
- alert: IPMIVoltageStateWarning
|
|
expr: ipmi_voltage_state{job=~"ipmi.*|.*ipmi.*"} == 1
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Voltage sensor warning"
|
|
description: "{{ $labels.instance }} voltage sensor {{ $labels.name }} reports WARNING state."
|
|
|
|
- alert: IPMIVoltageStateCritical
|
|
expr: ipmi_voltage_state{job=~"ipmi.*|.*ipmi.*"} >= 2
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Voltage sensor critical"
|
|
description: "{{ $labels.instance }} voltage sensor {{ $labels.name }} reports CRITICAL state."
|
|
|
|
- name: ipmi-generic-thresholds
|
|
rules:
|
|
- alert: IPMISensorVoltageVeryLow
|
|
expr: ipmi_sensor_value{type=~"Voltage|voltage",job=~"ipmi.*|.*ipmi.*"} > 0 and ipmi_sensor_value{type=~"Voltage|voltage",job=~"ipmi.*|.*ipmi.*"} < 0.8
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Voltage sensor very low"
|
|
description: "{{ $labels.instance }} voltage sensor {{ $labels.name }} is very low."
|
|
|
|
- alert: IPMISensorVoltageVeryHigh
|
|
expr: ipmi_sensor_value{type=~"Voltage|voltage",job=~"ipmi.*|.*ipmi.*"} > 14
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Voltage sensor very high"
|
|
description: "{{ $labels.instance }} voltage sensor {{ $labels.name }} is very high."
|
|
|
|
- alert: IPMISensorCurrentHigh
|
|
expr: ipmi_current_amperes{job=~"ipmi.*|.*ipmi.*"} > 20
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "High current"
|
|
description: "{{ $labels.instance }} current sensor {{ $labels.name }} is above 20 A."
|
|
|
|
- name: ipmi-bmc-info
|
|
rules:
|
|
- alert: IPMIBMCInfoMissing
|
|
expr: absent(ipmi_bmc_info{job=~"ipmi.*|.*ipmi.*"})
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "IPMI BMC info missing"
|
|
description: "No ipmi_bmc_info metrics are present."
|
|
|
|
- alert: IPMIBMCFirmwareUnknown
|
|
expr: ipmi_bmc_info{firmware_revision="",job=~"ipmi.*|.*ipmi.*"} == 1
|
|
for: 10m
|
|
labels:
|
|
severity: info
|
|
annotations:
|
|
summary: "BMC firmware revision unknown"
|
|
description: "{{ $labels.instance }} does not report BMC firmware revision."
|