Added a BUNCH of configs.
This commit is contained in:
@@ -0,0 +1,30 @@
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: blackbox-alerts
|
||||
namespace: monitoring
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
|
||||
spec:
|
||||
groups:
|
||||
- name: blackbox
|
||||
rules:
|
||||
|
||||
- alert: DeviceDown
|
||||
expr: probe_success == 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Device down"
|
||||
description: "{{ $labels.instance }} is DOWN"
|
||||
|
||||
- alert: HighPing
|
||||
expr: probe_duration_seconds > 0.5
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High ping"
|
||||
description: "{{ $labels.instance }} has high latency"
|
||||
@@ -0,0 +1,331 @@
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: ipmi-alerts
|
||||
namespace: monitoring
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
|
||||
spec:
|
||||
groups:
|
||||
- name: ipmi-targets
|
||||
rules:
|
||||
- alert: IPMITargetDown
|
||||
expr: up{job=~"ipmi.*|.*ipmi.*"} == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "IPMI target down"
|
||||
description: "{{ $labels.instance }} IPMI scrape target is down."
|
||||
|
||||
- alert: IPMIExporterCollectorDown
|
||||
expr: ipmi_up{job=~"ipmi.*|.*ipmi.*"} == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "IPMI collector down"
|
||||
description: "{{ $labels.instance }} IPMI collector {{ $labels.collector }} is down."
|
||||
|
||||
- alert: IPMIScrapeSlow
|
||||
expr: scrape_duration_seconds{job=~"ipmi.*|.*ipmi.*"} > 30
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "IPMI scrape slow"
|
||||
description: "{{ $labels.instance }} IPMI scrape takes more than 30 seconds."
|
||||
|
||||
- alert: IPMIScrapeVerySlow
|
||||
expr: scrape_duration_seconds{job=~"ipmi.*|.*ipmi.*"} > 90
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "IPMI scrape very slow"
|
||||
description: "{{ $labels.instance }} IPMI scrape takes more than 90 seconds."
|
||||
|
||||
- alert: IPMILowSamples
|
||||
expr: scrape_samples_scraped{job=~"ipmi.*|.*ipmi.*"} < 10
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "IPMI low sample count"
|
||||
description: "{{ $labels.instance }} IPMI scrape returns fewer than 10 samples."
|
||||
|
||||
- name: ipmi-chassis
|
||||
rules:
|
||||
- alert: IPMIChassisPowerOff
|
||||
expr: ipmi_chassis_power_state{job=~"ipmi.*|.*ipmi.*"} == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Server chassis power is off"
|
||||
description: "{{ $labels.instance }} chassis power state is OFF."
|
||||
|
||||
- alert: IPMIChassisCoolingFault
|
||||
expr: ipmi_chassis_cooling_fault_state{job=~"ipmi.*|.*ipmi.*"} == 1
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Chassis cooling fault"
|
||||
description: "{{ $labels.instance }} reports chassis cooling fault."
|
||||
|
||||
- alert: IPMIChassisDriveFault
|
||||
expr: ipmi_chassis_drive_fault_state{job=~"ipmi.*|.*ipmi.*"} == 1
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Chassis drive fault"
|
||||
description: "{{ $labels.instance }} reports chassis drive fault."
|
||||
|
||||
- alert: IPMIChassisPowerControlFault
|
||||
expr: ipmi_chassis_power_control_fault_state{job=~"ipmi.*|.*ipmi.*"} == 1
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Power control fault"
|
||||
description: "{{ $labels.instance }} reports power control fault."
|
||||
|
||||
- name: ipmi-sensors
|
||||
rules:
|
||||
- alert: IPMISensorWarning
|
||||
expr: ipmi_sensor_state{job=~"ipmi.*|.*ipmi.*"} == 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "IPMI sensor warning"
|
||||
description: "{{ $labels.instance }} sensor {{ $labels.name }} reports WARNING state."
|
||||
|
||||
- alert: IPMISensorCritical
|
||||
expr: ipmi_sensor_state{job=~"ipmi.*|.*ipmi.*"} >= 2
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "IPMI sensor critical"
|
||||
description: "{{ $labels.instance }} sensor {{ $labels.name }} reports CRITICAL state."
|
||||
|
||||
- alert: IPMISensorMissingValue
|
||||
expr: ipmi_sensor_value{job=~"ipmi.*|.*ipmi.*"} != ipmi_sensor_value{job=~"ipmi.*|.*ipmi.*"}
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "IPMI sensor value is NaN"
|
||||
description: "{{ $labels.instance }} sensor {{ $labels.name }} returns NaN."
|
||||
|
||||
- name: ipmi-fans
|
||||
rules:
|
||||
- alert: IPMIFanStateWarning
|
||||
expr: ipmi_fan_speed_state{job=~"ipmi.*|.*ipmi.*"} == 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Fan warning state"
|
||||
description: "{{ $labels.instance }} fan {{ $labels.name }} reports WARNING state."
|
||||
|
||||
- alert: IPMIFanStateCritical
|
||||
expr: ipmi_fan_speed_state{job=~"ipmi.*|.*ipmi.*"} >= 2
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Fan critical state"
|
||||
description: "{{ $labels.instance }} fan {{ $labels.name }} reports CRITICAL state."
|
||||
|
||||
- alert: IPMIFanStopped
|
||||
expr: ipmi_fan_speed_rpm{job=~"ipmi.*|.*ipmi.*"} == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Fan stopped"
|
||||
description: "{{ $labels.instance }} fan {{ $labels.name }} reports 0 RPM."
|
||||
|
||||
- alert: IPMIFanLowRPM
|
||||
expr: ipmi_fan_speed_rpm{job=~"ipmi.*|.*ipmi.*"} > 0 and ipmi_fan_speed_rpm{job=~"ipmi.*|.*ipmi.*"} < 1000
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Fan low RPM"
|
||||
description: "{{ $labels.instance }} fan {{ $labels.name }} is below 1000 RPM."
|
||||
|
||||
- alert: IPMIFanVeryHighRPM
|
||||
expr: ipmi_fan_speed_rpm{job=~"ipmi.*|.*ipmi.*"} > 10000
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Fan very high RPM"
|
||||
description: "{{ $labels.instance }} fan {{ $labels.name }} is above 10000 RPM."
|
||||
|
||||
- name: ipmi-temperature
|
||||
rules:
|
||||
- alert: IPMITemperatureHigh
|
||||
expr: ipmi_temperature_celsius{job=~"ipmi.*|.*ipmi.*"} > 70
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High IPMI temperature"
|
||||
description: "{{ $labels.instance }} sensor {{ $labels.name }} is above 70°C."
|
||||
|
||||
- alert: IPMITemperatureCritical
|
||||
expr: ipmi_temperature_celsius{job=~"ipmi.*|.*ipmi.*"} > 85
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Critical IPMI temperature"
|
||||
description: "{{ $labels.instance }} sensor {{ $labels.name }} is above 85°C."
|
||||
|
||||
- alert: IPMISensorTemperatureHigh
|
||||
expr: ipmi_sensor_value{type=~"Temperature|temperature",job=~"ipmi.*|.*ipmi.*"} > 70
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High sensor temperature"
|
||||
description: "{{ $labels.instance }} sensor {{ $labels.name }} is above 70°C."
|
||||
|
||||
- alert: IPMISensorTemperatureCritical
|
||||
expr: ipmi_sensor_value{type=~"Temperature|temperature",job=~"ipmi.*|.*ipmi.*"} > 85
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Critical sensor temperature"
|
||||
description: "{{ $labels.instance }} sensor {{ $labels.name }} is above 85°C."
|
||||
|
||||
- name: ipmi-power-current-voltage
|
||||
rules:
|
||||
- alert: IPMIPowerHigh
|
||||
expr: ipmi_power_watts{job=~"ipmi.*|.*ipmi.*"} > 600
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High power usage"
|
||||
description: "{{ $labels.instance }} power usage is above 600 W."
|
||||
|
||||
- alert: IPMIPowerVeryHigh
|
||||
expr: ipmi_power_watts{job=~"ipmi.*|.*ipmi.*"} > 800
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Very high power usage"
|
||||
description: "{{ $labels.instance }} power usage is above 800 W."
|
||||
|
||||
- alert: IPMIPowerStateWarning
|
||||
expr: ipmi_power_state{job=~"ipmi.*|.*ipmi.*"} == 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Power sensor warning"
|
||||
description: "{{ $labels.instance }} power sensor {{ $labels.name }} reports WARNING state."
|
||||
|
||||
- alert: IPMIPowerStateCritical
|
||||
expr: ipmi_power_state{job=~"ipmi.*|.*ipmi.*"} >= 2
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Power sensor critical"
|
||||
description: "{{ $labels.instance }} power sensor {{ $labels.name }} reports CRITICAL state."
|
||||
|
||||
- alert: IPMICurrentStateWarning
|
||||
expr: ipmi_current_state{job=~"ipmi.*|.*ipmi.*"} == 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Current sensor warning"
|
||||
description: "{{ $labels.instance }} current sensor {{ $labels.name }} reports WARNING state."
|
||||
|
||||
- alert: IPMICurrentStateCritical
|
||||
expr: ipmi_current_state{job=~"ipmi.*|.*ipmi.*"} >= 2
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Current sensor critical"
|
||||
description: "{{ $labels.instance }} current sensor {{ $labels.name }} reports CRITICAL state."
|
||||
|
||||
- alert: IPMIVoltageStateWarning
|
||||
expr: ipmi_voltage_state{job=~"ipmi.*|.*ipmi.*"} == 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Voltage sensor warning"
|
||||
description: "{{ $labels.instance }} voltage sensor {{ $labels.name }} reports WARNING state."
|
||||
|
||||
- alert: IPMIVoltageStateCritical
|
||||
expr: ipmi_voltage_state{job=~"ipmi.*|.*ipmi.*"} >= 2
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Voltage sensor critical"
|
||||
description: "{{ $labels.instance }} voltage sensor {{ $labels.name }} reports CRITICAL state."
|
||||
|
||||
- name: ipmi-generic-thresholds
|
||||
rules:
|
||||
- alert: IPMISensorVoltageVeryLow
|
||||
expr: ipmi_sensor_value{type=~"Voltage|voltage",job=~"ipmi.*|.*ipmi.*"} > 0 and ipmi_sensor_value{type=~"Voltage|voltage",job=~"ipmi.*|.*ipmi.*"} < 0.8
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Voltage sensor very low"
|
||||
description: "{{ $labels.instance }} voltage sensor {{ $labels.name }} is very low."
|
||||
|
||||
- alert: IPMISensorVoltageVeryHigh
|
||||
expr: ipmi_sensor_value{type=~"Voltage|voltage",job=~"ipmi.*|.*ipmi.*"} > 14
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Voltage sensor very high"
|
||||
description: "{{ $labels.instance }} voltage sensor {{ $labels.name }} is very high."
|
||||
|
||||
- alert: IPMISensorCurrentHigh
|
||||
expr: ipmi_current_amperes{job=~"ipmi.*|.*ipmi.*"} > 20
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High current"
|
||||
description: "{{ $labels.instance }} current sensor {{ $labels.name }} is above 20 A."
|
||||
|
||||
- name: ipmi-bmc-info
|
||||
rules:
|
||||
- alert: IPMIBMCInfoMissing
|
||||
expr: absent(ipmi_bmc_info{job=~"ipmi.*|.*ipmi.*"})
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "IPMI BMC info missing"
|
||||
description: "No ipmi_bmc_info metrics are present."
|
||||
|
||||
- alert: IPMIBMCFirmwareUnknown
|
||||
expr: ipmi_bmc_info{firmware_revision="",job=~"ipmi.*|.*ipmi.*"} == 1
|
||||
for: 10m
|
||||
labels:
|
||||
severity: info
|
||||
annotations:
|
||||
summary: "BMC firmware revision unknown"
|
||||
description: "{{ $labels.instance }} does not report BMC firmware revision."
|
||||
@@ -0,0 +1,470 @@
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: node-exporter-alerts
|
||||
namespace: monitoring
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
|
||||
spec:
|
||||
groups:
|
||||
- name: node-exporter-availability
|
||||
rules:
|
||||
- alert: NodeDown
|
||||
expr: up{job=~".*node.*|external-node-exporters"} == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Node down"
|
||||
description: "{{ $labels.instance }} is down."
|
||||
|
||||
- alert: NodeExporterMissing
|
||||
expr: absent(up{job=~".*node.*|external-node-exporters"})
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Node exporter missing"
|
||||
description: "No node-exporter targets are currently being scraped."
|
||||
|
||||
- name: node-exporter-cpu-load
|
||||
rules:
|
||||
- alert: NodeHighCPU
|
||||
expr: 100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 85
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High CPU usage"
|
||||
description: "{{ $labels.instance }} CPU usage is above 85% for 10 minutes."
|
||||
|
||||
- alert: NodeCriticalCPU
|
||||
expr: 100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 95
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Critical CPU usage"
|
||||
description: "{{ $labels.instance }} CPU usage is above 95%."
|
||||
|
||||
- alert: NodeHighLoad1
|
||||
expr: node_load1 / count by(instance) (node_cpu_seconds_total{mode="idle"}) > 2
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High 1m load"
|
||||
description: "{{ $labels.instance }} has high 1-minute load."
|
||||
|
||||
- alert: NodeHighLoad5
|
||||
expr: node_load5 / count by(instance) (node_cpu_seconds_total{mode="idle"}) > 1.75
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High 5m load"
|
||||
description: "{{ $labels.instance }} has high 5-minute load."
|
||||
|
||||
- alert: NodeHighLoad15
|
||||
expr: node_load15 / count by(instance) (node_cpu_seconds_total{mode="idle"}) > 1.5
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High 15m load"
|
||||
description: "{{ $labels.instance }} has high 15-minute load."
|
||||
|
||||
- alert: NodeCriticalLoad15
|
||||
expr: node_load15 / count by(instance) (node_cpu_seconds_total{mode="idle"}) > 3
|
||||
for: 10m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Critical 15m load"
|
||||
description: "{{ $labels.instance }} has critically high 15-minute load."
|
||||
|
||||
- name: node-exporter-memory-swap
|
||||
rules:
|
||||
- alert: NodeMemoryLow
|
||||
expr: (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) < 0.10
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Low available memory"
|
||||
description: "{{ $labels.instance }} has less than 10% memory available."
|
||||
|
||||
- alert: NodeMemoryCritical
|
||||
expr: (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) < 0.05
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Critical available memory"
|
||||
description: "{{ $labels.instance }} has less than 5% memory available."
|
||||
|
||||
- alert: NodeSwapHigh
|
||||
expr: (node_memory_SwapTotal_bytes > 0) and ((node_memory_SwapTotal_bytes - node_memory_SwapFree_bytes) / node_memory_SwapTotal_bytes > 0.50)
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High swap usage"
|
||||
description: "{{ $labels.instance }} swap usage is above 50%."
|
||||
|
||||
- alert: NodeSwapCritical
|
||||
expr: (node_memory_SwapTotal_bytes > 0) and ((node_memory_SwapTotal_bytes - node_memory_SwapFree_bytes) / node_memory_SwapTotal_bytes > 0.80)
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Critical swap usage"
|
||||
description: "{{ $labels.instance }} swap usage is above 80%."
|
||||
|
||||
- alert: NodeOOMKills
|
||||
expr: increase(node_vmstat_oom_kill[10m]) > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "OOM kill detected"
|
||||
description: "{{ $labels.instance }} had OOM kill event in last 10 minutes."
|
||||
|
||||
- name: node-exporter-filesystems
|
||||
rules:
|
||||
- alert: NodeRootFilesystemLow
|
||||
expr: (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs|nsfs|ramfs",mountpoint="/"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay|squashfs|nsfs|ramfs",mountpoint="/"}) < 0.15
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Root filesystem low"
|
||||
description: "{{ $labels.instance }} root filesystem has less than 15% free space."
|
||||
|
||||
- alert: NodeRootFilesystemCritical
|
||||
expr: (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs|nsfs|ramfs",mountpoint="/"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay|squashfs|nsfs|ramfs",mountpoint="/"}) < 0.05
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Root filesystem critical"
|
||||
description: "{{ $labels.instance }} root filesystem has less than 5% free space."
|
||||
|
||||
- alert: NodeFilesystemLow
|
||||
expr: (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs|nsfs|ramfs"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay|squashfs|nsfs|ramfs"}) < 0.15
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Filesystem low"
|
||||
description: "{{ $labels.instance }} {{ $labels.mountpoint }} has less than 15% free space."
|
||||
|
||||
- alert: NodeFilesystemCritical
|
||||
expr: (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs|nsfs|ramfs"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay|squashfs|nsfs|ramfs"}) < 0.05
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Filesystem critical"
|
||||
description: "{{ $labels.instance }} {{ $labels.mountpoint }} has less than 5% free space."
|
||||
|
||||
- alert: NodeFilesystemWillFillSoon
|
||||
expr: predict_linear(node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs|nsfs|ramfs"}[6h], 24 * 3600) < 0
|
||||
for: 30m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Filesystem will fill soon"
|
||||
description: "{{ $labels.instance }} {{ $labels.mountpoint }} is predicted to fill within 24 hours."
|
||||
|
||||
- alert: NodeFilesystemWillFillVerySoon
|
||||
expr: predict_linear(node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs|nsfs|ramfs"}[2h], 6 * 3600) < 0
|
||||
for: 15m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Filesystem will fill very soon"
|
||||
description: "{{ $labels.instance }} {{ $labels.mountpoint }} is predicted to fill within 6 hours."
|
||||
|
||||
- alert: NodeInodesLow
|
||||
expr: (node_filesystem_files_free{fstype!~"tmpfs|overlay|squashfs|nsfs|ramfs"} / node_filesystem_files{fstype!~"tmpfs|overlay|squashfs|nsfs|ramfs"}) < 0.10
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Filesystem inodes low"
|
||||
description: "{{ $labels.instance }} {{ $labels.mountpoint }} has less than 10% free inodes."
|
||||
|
||||
- alert: NodeInodesCritical
|
||||
expr: (node_filesystem_files_free{fstype!~"tmpfs|overlay|squashfs|nsfs|ramfs"} / node_filesystem_files{fstype!~"tmpfs|overlay|squashfs|nsfs|ramfs"}) < 0.05
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Filesystem inodes critical"
|
||||
description: "{{ $labels.instance }} {{ $labels.mountpoint }} has less than 5% free inodes."
|
||||
|
||||
- alert: NodeFilesystemReadOnly
|
||||
expr: node_filesystem_readonly{fstype!~"tmpfs|overlay|squashfs|nsfs|ramfs"} == 1
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Filesystem read-only"
|
||||
description: "{{ $labels.instance }} {{ $labels.mountpoint }} is read-only."
|
||||
|
||||
- name: node-exporter-disk-io
|
||||
rules:
|
||||
- alert: NodeDiskIOHigh
|
||||
expr: rate(node_disk_io_time_seconds_total{device!~"loop.*|ram.*|dm-.*"}[5m]) > 0.90
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High disk I/O utilization"
|
||||
description: "{{ $labels.instance }} disk {{ $labels.device }} I/O utilization is high."
|
||||
|
||||
- alert: NodeDiskReadLatencyHigh
|
||||
expr: rate(node_disk_read_time_seconds_total{device!~"loop.*|ram.*"}[5m]) / rate(node_disk_reads_completed_total{device!~"loop.*|ram.*"}[5m]) > 0.1
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High disk read latency"
|
||||
description: "{{ $labels.instance }} disk {{ $labels.device }} read latency is high."
|
||||
|
||||
- alert: NodeDiskWriteLatencyHigh
|
||||
expr: rate(node_disk_write_time_seconds_total{device!~"loop.*|ram.*"}[5m]) / rate(node_disk_writes_completed_total{device!~"loop.*|ram.*"}[5m]) > 0.1
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High disk write latency"
|
||||
description: "{{ $labels.instance }} disk {{ $labels.device }} write latency is high."
|
||||
|
||||
- alert: NodeDiskReadErrorsOrWeirdness
|
||||
expr: increase(node_disk_reads_completed_total{device!~"loop.*|ram.*"}[10m]) == 0 and rate(node_disk_read_time_seconds_total{device!~"loop.*|ram.*"}[10m]) > 0
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Possible disk read issue"
|
||||
description: "{{ $labels.instance }} disk {{ $labels.device }} reports read time without completed reads."
|
||||
|
||||
- name: node-exporter-network
|
||||
rules:
|
||||
- alert: NodeNetworkReceiveErrors
|
||||
expr: increase(node_network_receive_errs_total{device!~"lo|veth.*|docker.*|br-.*|cni.*|flannel.*|calico.*"}[10m]) > 10
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Network RX errors"
|
||||
description: "{{ $labels.instance }} interface {{ $labels.device }} has RX errors."
|
||||
|
||||
- alert: NodeNetworkTransmitErrors
|
||||
expr: increase(node_network_transmit_errs_total{device!~"lo|veth.*|docker.*|br-.*|cni.*|flannel.*|calico.*"}[10m]) > 10
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Network TX errors"
|
||||
description: "{{ $labels.instance }} interface {{ $labels.device }} has TX errors."
|
||||
|
||||
- alert: NodeNetworkReceiveDrops
|
||||
expr: increase(node_network_receive_drop_total{device!~"lo|veth.*|docker.*|br-.*|cni.*|flannel.*|calico.*"}[10m]) > 50
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Network RX drops"
|
||||
description: "{{ $labels.instance }} interface {{ $labels.device }} has RX drops."
|
||||
|
||||
- alert: NodeNetworkTransmitDrops
|
||||
expr: increase(node_network_transmit_drop_total{device!~"lo|veth.*|docker.*|br-.*|cni.*|flannel.*|calico.*"}[10m]) > 50
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Network TX drops"
|
||||
description: "{{ $labels.instance }} interface {{ $labels.device }} has TX drops."
|
||||
|
||||
- alert: NodeNetworkInterfaceFlapping
|
||||
expr: changes(node_network_up{device!~"lo|veth.*|docker.*|br-.*|cni.*|flannel.*|calico.*"}[10m]) > 2
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Network interface flapping"
|
||||
description: "{{ $labels.instance }} interface {{ $labels.device }} changed link state more than twice in 10 minutes."
|
||||
|
||||
- alert: NodeNetworkInterfaceDown
|
||||
expr: node_network_up{device!~"lo|veth.*|docker.*|br-.*|cni.*|flannel.*|calico.*"} == 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Network interface down"
|
||||
description: "{{ $labels.instance }} interface {{ $labels.device }} is down."
|
||||
|
||||
- name: node-exporter-system-health
|
||||
rules:
|
||||
- alert: NodeRebooted
|
||||
expr: time() - node_boot_time_seconds < 600
|
||||
for: 1m
|
||||
labels:
|
||||
severity: info
|
||||
annotations:
|
||||
summary: "Node recently rebooted"
|
||||
description: "{{ $labels.instance }} rebooted less than 10 minutes ago."
|
||||
|
||||
- alert: NodeTimeDrift
|
||||
expr: abs(node_timex_offset_seconds) > 0.1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Node time drift"
|
||||
description: "{{ $labels.instance }} has time drift greater than 100 ms."
|
||||
|
||||
- alert: NodeTimeDriftCritical
|
||||
expr: abs(node_timex_offset_seconds) > 1
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Critical node time drift"
|
||||
description: "{{ $labels.instance }} has time drift greater than 1 second."
|
||||
|
||||
- alert: NodeClockNotSynchronized
|
||||
expr: node_timex_sync_status == 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Clock not synchronized"
|
||||
description: "{{ $labels.instance }} clock is not synchronized."
|
||||
|
||||
- alert: NodeEntropyLow
|
||||
expr: node_entropy_available_bits < 256
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Low entropy"
|
||||
description: "{{ $labels.instance }} has low available entropy."
|
||||
|
||||
- alert: NodeFileDescriptorsHigh
|
||||
expr: node_filefd_allocated / node_filefd_maximum > 0.80
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High file descriptor usage"
|
||||
description: "{{ $labels.instance }} file descriptor usage is above 80%."
|
||||
|
||||
- alert: NodeFileDescriptorsCritical
|
||||
expr: node_filefd_allocated / node_filefd_maximum > 0.95
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Critical file descriptor usage"
|
||||
description: "{{ $labels.instance }} file descriptor usage is above 95%."
|
||||
|
||||
- alert: NodeProcessesHigh
|
||||
expr: node_procs_running > 300
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High running process count"
|
||||
description: "{{ $labels.instance }} has more than 300 running processes."
|
||||
|
||||
- alert: NodeProcessesBlocked
|
||||
expr: node_procs_blocked > 5
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Blocked processes"
|
||||
description: "{{ $labels.instance }} has more than 5 blocked processes."
|
||||
|
||||
- name: node-exporter-systemd
|
||||
rules:
|
||||
- alert: NodeSystemdUnitFailed
|
||||
expr: node_systemd_unit_state{state="failed"} == 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Systemd unit failed"
|
||||
description: "{{ $labels.instance }} systemd unit {{ $labels.name }} is failed."
|
||||
|
||||
- alert: NodeSystemdServiceRestartingTooOften
|
||||
expr: increase(node_systemd_service_restart_total[15m]) > 3
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Systemd service restarting too often"
|
||||
description: "{{ $labels.instance }} systemd service {{ $labels.name }} restarted more than 3 times in 15 minutes."
|
||||
|
||||
- name: node-exporter-hardware
|
||||
rules:
|
||||
- alert: NodeTemperatureHigh
|
||||
expr: node_hwmon_temp_celsius > 80
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High hardware temperature"
|
||||
description: "{{ $labels.instance }} sensor {{ $labels.chip }} {{ $labels.sensor }} is above 80°C."
|
||||
|
||||
- alert: NodeTemperatureCritical
|
||||
expr: node_hwmon_temp_celsius > 90
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Critical hardware temperature"
|
||||
description: "{{ $labels.instance }} sensor {{ $labels.chip }} {{ $labels.sensor }} is above 90°C."
|
||||
|
||||
- alert: NodeThermalZoneHigh
|
||||
expr: node_thermal_zone_temp > 80000
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High thermal zone temperature"
|
||||
description: "{{ $labels.instance }} thermal zone {{ $labels.zone }} is above 80°C."
|
||||
|
||||
- alert: NodeThermalZoneCritical
|
||||
expr: node_thermal_zone_temp > 90000
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Critical thermal zone temperature"
|
||||
description: "{{ $labels.instance }} thermal zone {{ $labels.zone }} is above 90°C."
|
||||
|
||||
- name: node-exporter-prometheus-scrape
|
||||
rules:
|
||||
- alert: NodeExporterScrapeSlow
|
||||
expr: scrape_duration_seconds{job=~".*node.*|external-node-exporters"} > 10
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Node exporter scrape slow"
|
||||
description: "Scrape of {{ $labels.instance }} takes more than 10 seconds."
|
||||
|
||||
- alert: NodeExporterLowSamples
|
||||
expr: scrape_samples_scraped{job=~".*node.*|external-node-exporters"} < 100
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Node exporter low sample count"
|
||||
description: "{{ $labels.instance }} returns fewer than 100 samples."
|
||||
@@ -0,0 +1,423 @@
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: proxmox-alerts
|
||||
namespace: monitoring
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
|
||||
spec:
|
||||
groups:
|
||||
- name: proxmox-targets
|
||||
rules:
|
||||
- alert: ProxmoxTargetDown
|
||||
expr: up{job=~"proxmox.*|.*pve.*"} == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Proxmox exporter target down"
|
||||
description: "{{ $labels.instance }} Proxmox/PVE scrape is down."
|
||||
|
||||
- alert: ProxmoxScrapeSlow
|
||||
expr: scrape_duration_seconds{job=~"proxmox.*|.*pve.*"} > 20
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Proxmox scrape slow"
|
||||
description: "{{ $labels.instance }} Proxmox scrape takes more than 20 seconds."
|
||||
|
||||
- alert: ProxmoxScrapeVerySlow
|
||||
expr: scrape_duration_seconds{job=~"proxmox.*|.*pve.*"} > 60
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Proxmox scrape very slow"
|
||||
description: "{{ $labels.instance }} Proxmox scrape takes more than 60 seconds."
|
||||
|
||||
- alert: ProxmoxLowSamples
|
||||
expr: scrape_samples_scraped{job=~"proxmox.*|.*pve.*"} < 20
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Proxmox low sample count"
|
||||
description: "{{ $labels.instance }} Proxmox scrape returns fewer than 20 samples."
|
||||
|
||||
- name: proxmox-nodes
|
||||
rules:
|
||||
- alert: ProxmoxNodeDown
|
||||
expr: pve_node_status == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Proxmox node down"
|
||||
description: "Proxmox node {{ $labels.node }} is down."
|
||||
|
||||
- alert: ProxmoxNodeHighCPU
|
||||
expr: pve_node_cpu_usage_ratio > 0.85
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Proxmox node high CPU"
|
||||
description: "Node {{ $labels.node }} CPU usage is above 85%."
|
||||
|
||||
- alert: ProxmoxNodeCriticalCPU
|
||||
expr: pve_node_cpu_usage_ratio > 0.95
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Proxmox node critical CPU"
|
||||
description: "Node {{ $labels.node }} CPU usage is above 95%."
|
||||
|
||||
- alert: ProxmoxNodeHighMemory
|
||||
expr: pve_node_memory_usage_bytes / pve_node_memory_size_bytes > 0.90
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Proxmox node high memory"
|
||||
description: "Node {{ $labels.node }} memory usage is above 90%."
|
||||
|
||||
- alert: ProxmoxNodeCriticalMemory
|
||||
expr: pve_node_memory_usage_bytes / pve_node_memory_size_bytes > 0.97
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Proxmox node critical memory"
|
||||
description: "Node {{ $labels.node }} memory usage is above 97%."
|
||||
|
||||
- alert: ProxmoxNodeLowFreeMemory
|
||||
expr: pve_node_memory_free_bytes / pve_node_memory_size_bytes < 0.10
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Proxmox node low free memory"
|
||||
description: "Node {{ $labels.node }} has less than 10% free memory."
|
||||
|
||||
- alert: ProxmoxNodeVeryLowFreeMemory
|
||||
expr: pve_node_memory_free_bytes / pve_node_memory_size_bytes < 0.05
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Proxmox node very low free memory"
|
||||
description: "Node {{ $labels.node }} has less than 5% free memory."
|
||||
|
||||
- alert: ProxmoxNodeHighLoad
|
||||
expr: pve_node_loadavg_15 / pve_node_cpu_count > 1.5
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Proxmox node high load"
|
||||
description: "Node {{ $labels.node }} has high 15m load."
|
||||
|
||||
- alert: ProxmoxNodeCriticalLoad
|
||||
expr: pve_node_loadavg_15 / pve_node_cpu_count > 3
|
||||
for: 10m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Proxmox node critical load"
|
||||
description: "Node {{ $labels.node }} has critically high 15m load."
|
||||
|
||||
- alert: ProxmoxNodeUptimeTooLow
|
||||
expr: pve_node_uptime_seconds < 600
|
||||
for: 1m
|
||||
labels:
|
||||
severity: info
|
||||
annotations:
|
||||
summary: "Proxmox node recently rebooted"
|
||||
description: "Node {{ $labels.node }} uptime is less than 10 minutes."
|
||||
|
||||
- name: proxmox-guests
|
||||
rules:
|
||||
- alert: ProxmoxGuestDown
|
||||
expr: pve_guest_status == 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Proxmox guest down"
|
||||
description: "Guest {{ $labels.id }} {{ $labels.name }} on node {{ $labels.node }} is down."
|
||||
|
||||
- alert: ProxmoxGuestHighCPU
|
||||
expr: pve_guest_cpu_usage_ratio > 0.90
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Proxmox guest high CPU"
|
||||
description: "Guest {{ $labels.id }} {{ $labels.name }} CPU usage is above 90%."
|
||||
|
||||
- alert: ProxmoxGuestCriticalCPU
|
||||
expr: pve_guest_cpu_usage_ratio > 0.98
|
||||
for: 10m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Proxmox guest critical CPU"
|
||||
description: "Guest {{ $labels.id }} {{ $labels.name }} CPU usage is above 98%."
|
||||
|
||||
- alert: ProxmoxGuestHighMemory
|
||||
expr: pve_guest_memory_usage_bytes / pve_guest_memory_size_bytes > 0.90
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Proxmox guest high memory"
|
||||
description: "Guest {{ $labels.id }} {{ $labels.name }} memory usage is above 90%."
|
||||
|
||||
- alert: ProxmoxGuestCriticalMemory
|
||||
expr: pve_guest_memory_usage_bytes / pve_guest_memory_size_bytes > 0.97
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Proxmox guest critical memory"
|
||||
description: "Guest {{ $labels.id }} {{ $labels.name }} memory usage is above 97%."
|
||||
|
||||
- alert: ProxmoxGuestLowFreeMemory
|
||||
expr: pve_guest_memory_free_bytes / pve_guest_memory_size_bytes < 0.10
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Proxmox guest low free memory"
|
||||
description: "Guest {{ $labels.id }} {{ $labels.name }} has less than 10% free memory."
|
||||
|
||||
- alert: ProxmoxGuestVeryLowFreeMemory
|
||||
expr: pve_guest_memory_free_bytes / pve_guest_memory_size_bytes < 0.05
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Proxmox guest very low free memory"
|
||||
description: "Guest {{ $labels.id }} {{ $labels.name }} has less than 5% free memory."
|
||||
|
||||
- alert: ProxmoxGuestHighDiskUsage
|
||||
expr: pve_guest_disk_usage_bytes / pve_guest_disk_size_bytes > 0.85
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Proxmox guest disk high usage"
|
||||
description: "Guest {{ $labels.id }} {{ $labels.name }} disk usage is above 85%."
|
||||
|
||||
- alert: ProxmoxGuestCriticalDiskUsage
|
||||
expr: pve_guest_disk_usage_bytes / pve_guest_disk_size_bytes > 0.95
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Proxmox guest disk critical usage"
|
||||
description: "Guest {{ $labels.id }} {{ $labels.name }} disk usage is above 95%."
|
||||
|
||||
- alert: ProxmoxGuestUnexpectedlyRunning
|
||||
expr: pve_guest_status == 1 and on(id) pve_guest_template == 1
|
||||
for: 10m
|
||||
labels:
|
||||
severity: info
|
||||
annotations:
|
||||
summary: "Proxmox template seems running"
|
||||
description: "Template guest {{ $labels.id }} {{ $labels.name }} appears to be running."
|
||||
|
||||
- alert: ProxmoxGuestRecentlyRebooted
|
||||
expr: pve_guest_uptime_seconds > 0 and pve_guest_uptime_seconds < 600
|
||||
for: 1m
|
||||
labels:
|
||||
severity: info
|
||||
annotations:
|
||||
summary: "Proxmox guest recently rebooted"
|
||||
description: "Guest {{ $labels.id }} {{ $labels.name }} uptime is less than 10 minutes."
|
||||
|
||||
- name: proxmox-storage
|
||||
rules:
|
||||
- alert: ProxmoxStorageHighUsage
|
||||
expr: pve_storage_usage_bytes / pve_storage_size_bytes > 0.85
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Proxmox storage high usage"
|
||||
description: "Storage {{ $labels.storage }} on node {{ $labels.node }} is above 85%."
|
||||
|
||||
- alert: ProxmoxStorageCriticalUsage
|
||||
expr: pve_storage_usage_bytes / pve_storage_size_bytes > 0.95
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Proxmox storage critical usage"
|
||||
description: "Storage {{ $labels.storage }} on node {{ $labels.node }} is above 95%."
|
||||
|
||||
- alert: ProxmoxStorageLowFree
|
||||
expr: pve_storage_free_bytes / pve_storage_size_bytes < 0.15
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Proxmox storage low free"
|
||||
description: "Storage {{ $labels.storage }} on node {{ $labels.node }} has less than 15% free space."
|
||||
|
||||
- alert: ProxmoxStorageVeryLowFree
|
||||
expr: pve_storage_free_bytes / pve_storage_size_bytes < 0.05
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Proxmox storage very low free"
|
||||
description: "Storage {{ $labels.storage }} on node {{ $labels.node }} has less than 5% free space."
|
||||
|
||||
- alert: ProxmoxStorageDisabled
|
||||
expr: pve_storage_status == 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Proxmox storage disabled/unavailable"
|
||||
description: "Storage {{ $labels.storage }} on node {{ $labels.node }} is disabled or unavailable."
|
||||
|
||||
- name: proxmox-cluster
|
||||
rules:
|
||||
- alert: ProxmoxClusterQuorumLost
|
||||
expr: pve_cluster_quorate == 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Proxmox cluster quorum lost"
|
||||
description: "Proxmox cluster has lost quorum."
|
||||
|
||||
- alert: ProxmoxClusterNodeOffline
|
||||
expr: pve_cluster_node_online == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Proxmox cluster node offline"
|
||||
description: "Cluster node {{ $labels.node }} is offline."
|
||||
|
||||
- alert: ProxmoxClusterExpectedVotesLow
|
||||
expr: pve_cluster_expected_votes < 2
|
||||
for: 10m
|
||||
labels:
|
||||
severity: info
|
||||
annotations:
|
||||
summary: "Proxmox cluster expected votes low"
|
||||
description: "Proxmox cluster expected votes is below 2."
|
||||
|
||||
- name: proxmox-backups
|
||||
rules:
|
||||
- alert: ProxmoxBackupFailed
|
||||
expr: pve_guest_backup_last_status == 0
|
||||
for: 10m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Proxmox backup failed"
|
||||
description: "Backup for guest {{ $labels.id }} {{ $labels.name }} failed."
|
||||
|
||||
- alert: ProxmoxBackupTooOld
|
||||
expr: time() - pve_guest_backup_last_timestamp_seconds > 7 * 24 * 3600
|
||||
for: 1h
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Proxmox backup too old"
|
||||
description: "Backup for guest {{ $labels.id }} {{ $labels.name }} is older than 7 days."
|
||||
|
||||
- alert: ProxmoxNoRecentBackup
|
||||
expr: absent(pve_guest_backup_last_timestamp_seconds)
|
||||
for: 1h
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "No Proxmox backup metric found"
|
||||
description: "No Proxmox backup timestamp metrics are present."
|
||||
|
||||
- name: proxmox-replication-ha
|
||||
rules:
|
||||
- alert: ProxmoxReplicationFailed
|
||||
expr: pve_replication_status == 0
|
||||
for: 10m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Proxmox replication failed"
|
||||
description: "Replication job {{ $labels.job }} on node {{ $labels.node }} failed."
|
||||
|
||||
- alert: ProxmoxReplicationTooOld
|
||||
expr: time() - pve_replication_last_sync_timestamp_seconds > 6 * 3600
|
||||
for: 30m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Proxmox replication too old"
|
||||
description: "Replication job {{ $labels.job }} last sync is older than 6 hours."
|
||||
|
||||
- alert: ProxmoxHAServiceError
|
||||
expr: pve_ha_service_status == 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Proxmox HA service error"
|
||||
description: "HA service {{ $labels.sid }} is not healthy."
|
||||
|
||||
- name: proxmox-tasks
|
||||
rules:
|
||||
- alert: ProxmoxTaskFailed
|
||||
expr: increase(pve_tasks_failed_total[30m]) > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Proxmox task failed"
|
||||
description: "A Proxmox task failed on {{ $labels.instance }}."
|
||||
|
||||
- alert: ProxmoxTasksFailingOften
|
||||
expr: increase(pve_tasks_failed_total[6h]) > 5
|
||||
for: 10m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Proxmox tasks failing often"
|
||||
description: "More than 5 Proxmox tasks failed in 6 hours on {{ $labels.instance }}."
|
||||
|
||||
- name: proxmox-exporter-health
|
||||
rules:
|
||||
- alert: ProxmoxExporterNoNodeMetrics
|
||||
expr: absent(pve_node_status)
|
||||
for: 10m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "No Proxmox node metrics"
|
||||
description: "PVE exporter is not returning node metrics."
|
||||
|
||||
- alert: ProxmoxExporterNoGuestMetrics
|
||||
expr: absent(pve_guest_status)
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "No Proxmox guest metrics"
|
||||
description: "PVE exporter is not returning guest metrics."
|
||||
|
||||
- alert: ProxmoxExporterNoStorageMetrics
|
||||
expr: absent(pve_storage_size_bytes)
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "No Proxmox storage metrics"
|
||||
description: "PVE exporter is not returning storage metrics."
|
||||
@@ -0,0 +1,211 @@
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: snmp-alerts
|
||||
namespace: monitoring
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
|
||||
spec:
|
||||
groups:
|
||||
- name: snmp-targets
|
||||
rules:
|
||||
- alert: SNMPTargetDown
|
||||
expr: up{job=~"snmp.*"} == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "SNMP target down"
|
||||
description: "SNMP target {{ $labels.instance }} in job {{ $labels.job }} is down."
|
||||
|
||||
- alert: SNMPScrapeSlow
|
||||
expr: scrape_duration_seconds{job=~"snmp.*"} > 20
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "SNMP scrape is slow"
|
||||
description: "SNMP scrape for {{ $labels.instance }} in job {{ $labels.job }} takes more than 20 seconds."
|
||||
|
||||
- alert: SNMPScrapeVerySlow
|
||||
expr: scrape_duration_seconds{job=~"snmp.*"} > 60
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "SNMP scrape is very slow"
|
||||
description: "SNMP scrape for {{ $labels.instance }} in job {{ $labels.job }} takes more than 60 seconds."
|
||||
|
||||
- alert: SNMPLowSamples
|
||||
expr: scrape_samples_scraped{job=~"snmp.*"} < 5
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "SNMP scrape returns very few samples"
|
||||
description: "SNMP target {{ $labels.instance }} in job {{ $labels.job }} returns fewer than 5 samples."
|
||||
|
||||
- name: snmp-interfaces
|
||||
rules:
|
||||
- alert: InterfaceAdminUpButDown
|
||||
expr: ifAdminStatus{job=~"snmp.*"} == 1 and ifOperStatus{job=~"snmp.*"} == 2
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Interface admin up but operationally down"
|
||||
description: "{{ $labels.instance }} {{ $labels.ifName }} is enabled but operationally down."
|
||||
|
||||
- alert: InterfaceDown
|
||||
expr: ifOperStatus{job=~"snmp.*"} == 2
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Interface down"
|
||||
description: "{{ $labels.instance }} {{ $labels.ifName }} is down."
|
||||
|
||||
- alert: InterfaceHighRXUtilization
|
||||
expr: ((rate(ifHCInOctets{job=~"snmp.*"}[5m]) * 8) / (ifHighSpeed{job=~"snmp.*"} * 1000000)) > 0.80
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High RX interface utilization"
|
||||
description: "{{ $labels.instance }} {{ $labels.ifName }} RX utilization is above 80%."
|
||||
|
||||
- alert: InterfaceHighTXUtilization
|
||||
expr: ((rate(ifHCOutOctets{job=~"snmp.*"}[5m]) * 8) / (ifHighSpeed{job=~"snmp.*"} * 1000000)) > 0.80
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High TX interface utilization"
|
||||
description: "{{ $labels.instance }} {{ $labels.ifName }} TX utilization is above 80%."
|
||||
|
||||
- alert: InterfaceCriticalRXUtilization
|
||||
expr: ((rate(ifHCInOctets{job=~"snmp.*"}[5m]) * 8) / (ifHighSpeed{job=~"snmp.*"} * 1000000)) > 0.95
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Critical RX interface utilization"
|
||||
description: "{{ $labels.instance }} {{ $labels.ifName }} RX utilization is above 95%."
|
||||
|
||||
- alert: InterfaceCriticalTXUtilization
|
||||
expr: ((rate(ifHCOutOctets{job=~"snmp.*"}[5m]) * 8) / (ifHighSpeed{job=~"snmp.*"} * 1000000)) > 0.95
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Critical TX interface utilization"
|
||||
description: "{{ $labels.instance }} {{ $labels.ifName }} TX utilization is above 95%."
|
||||
|
||||
- alert: InterfaceHighRXFixedThreshold
|
||||
expr: (rate(ifHCInOctets{job=~"snmp.*"}[5m]) * 8) > 800000000
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High RX traffic"
|
||||
description: "{{ $labels.instance }} {{ $labels.ifName }} RX traffic is above 800 Mbit/s."
|
||||
|
||||
- alert: InterfaceHighTXFixedThreshold
|
||||
expr: (rate(ifHCOutOctets{job=~"snmp.*"}[5m]) * 8) > 800000000
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High TX traffic"
|
||||
description: "{{ $labels.instance }} {{ $labels.ifName }} TX traffic is above 800 Mbit/s."
|
||||
|
||||
- alert: InterfaceRXErrors
|
||||
expr: increase(ifInErrors{job=~"snmp.*"}[10m]) > 10
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Interface RX errors"
|
||||
description: "{{ $labels.instance }} {{ $labels.ifName }} has more than 10 RX errors in 10 minutes."
|
||||
|
||||
- alert: InterfaceTXErrors
|
||||
expr: increase(ifOutErrors{job=~"snmp.*"}[10m]) > 10
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Interface TX errors"
|
||||
description: "{{ $labels.instance }} {{ $labels.ifName }} has more than 10 TX errors in 10 minutes."
|
||||
|
||||
- alert: InterfaceRXDiscards
|
||||
expr: increase(ifInDiscards{job=~"snmp.*"}[10m]) > 50
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Interface RX discards"
|
||||
description: "{{ $labels.instance }} {{ $labels.ifName }} has more than 50 RX discards in 10 minutes."
|
||||
|
||||
- alert: InterfaceTXDiscards
|
||||
expr: increase(ifOutDiscards{job=~"snmp.*"}[10m]) > 50
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Interface TX discards"
|
||||
description: "{{ $labels.instance }} {{ $labels.ifName }} has more than 50 TX discards in 10 minutes."
|
||||
|
||||
- alert: InterfaceBroadcastStorm
|
||||
expr: increase(ifHCInBroadcastPkts{job=~"snmp.*"}[5m]) > 10000
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Possible broadcast storm"
|
||||
description: "{{ $labels.instance }} {{ $labels.ifName }} received more than 10000 broadcast packets in 5 minutes."
|
||||
|
||||
- alert: InterfaceMulticastStorm
|
||||
expr: increase(ifHCInMulticastPkts{job=~"snmp.*"}[5m]) > 50000
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Possible multicast storm"
|
||||
description: "{{ $labels.instance }} {{ $labels.ifName }} received more than 50000 multicast packets in 5 minutes."
|
||||
|
||||
- alert: InterfaceNoTraffic
|
||||
expr: rate(ifHCInOctets{job=~"snmp.*"}[30m]) == 0 and rate(ifHCOutOctets{job=~"snmp.*"}[30m]) == 0 and ifOperStatus{job=~"snmp.*"} == 1
|
||||
for: 30m
|
||||
labels:
|
||||
severity: info
|
||||
annotations:
|
||||
summary: "Interface has no traffic"
|
||||
description: "{{ $labels.instance }} {{ $labels.ifName }} is up but has no RX/TX traffic for 30 minutes."
|
||||
|
||||
- alert: InterfaceCounterReset
|
||||
expr: resets(ifHCInOctets{job=~"snmp.*"}[15m]) > 0 or resets(ifHCOutOctets{job=~"snmp.*"}[15m]) > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: info
|
||||
annotations:
|
||||
summary: "Interface counter reset"
|
||||
description: "{{ $labels.instance }} {{ $labels.ifName }} interface counter reset detected."
|
||||
|
||||
- alert: InterfaceSpeedUnknown
|
||||
expr: ifHighSpeed{job=~"snmp.*"} == 0
|
||||
for: 10m
|
||||
labels:
|
||||
severity: info
|
||||
annotations:
|
||||
summary: "Interface speed unknown"
|
||||
description: "{{ $labels.instance }} {{ $labels.ifName }} reports unknown interface speed."
|
||||
|
||||
- alert: InterfacePossibleFlapping
|
||||
expr: changes(ifOperStatus{job=~"snmp.*"}[10m]) > 2
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Possible interface flapping"
|
||||
description: "{{ $labels.instance }} {{ $labels.ifName }} changed operational state more than twice in 10 minutes."
|
||||
@@ -0,0 +1,12 @@
|
||||
config:
|
||||
modules:
|
||||
redfish:
|
||||
prober: http
|
||||
timeout: 10s
|
||||
http:
|
||||
method: GET
|
||||
|
||||
tls_config:
|
||||
insecure_skip_verify: true
|
||||
|
||||
preferred_ip_protocol: ip4
|
||||
@@ -0,0 +1,13 @@
|
||||
config:
|
||||
modules:
|
||||
icmp:
|
||||
prober: icmp
|
||||
timeout: 5s
|
||||
icmp:
|
||||
preferred_ip_protocol: ip4
|
||||
|
||||
http_2xx:
|
||||
prober: http
|
||||
timeout: 5s
|
||||
http:
|
||||
preferred_ip_protocol: ip4
|
||||
@@ -0,0 +1,36 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: pve-exporter
|
||||
namespace: monitoring
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: pve-exporter
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: pve-exporter
|
||||
spec:
|
||||
containers:
|
||||
- name: pve-exporter
|
||||
image: prompve/prometheus-pve-exporter:latest
|
||||
ports:
|
||||
- containerPort: 9221
|
||||
envFrom:
|
||||
- secretRef:
|
||||
name: pve-exporter-secret
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: pve-exporter
|
||||
namespace: monitoring
|
||||
spec:
|
||||
selector:
|
||||
app: pve-exporter
|
||||
ports:
|
||||
- name: http
|
||||
port: 9221
|
||||
targetPort: 9221
|
||||
@@ -0,0 +1,44 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: snmp-exporter-manual
|
||||
namespace: monitoring
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: snmp-exporter-manual
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: snmp-exporter-manual
|
||||
spec:
|
||||
containers:
|
||||
- name: snmp-exporter
|
||||
image: quay.io/prometheus/snmp-exporter:v0.30.1
|
||||
args:
|
||||
- --config.file=/etc/snmp_exporter/snmp.yml
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 9116
|
||||
volumeMounts:
|
||||
- name: snmp-config
|
||||
mountPath: /etc/snmp_exporter/snmp.yml
|
||||
subPath: snmp.yml
|
||||
volumes:
|
||||
- name: snmp-config
|
||||
configMap:
|
||||
name: snmp-exporter-config
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: snmp-exporter-manual
|
||||
namespace: monitoring
|
||||
spec:
|
||||
selector:
|
||||
app: snmp-exporter-manual
|
||||
ports:
|
||||
- name: http
|
||||
port: 9116
|
||||
targetPort: 9116
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,25 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: external-node-exporters
|
||||
namespace: monitoring
|
||||
|
||||
data:
|
||||
prometheus-additional.yaml: |
|
||||
- job_name: 'external-node-exporters'
|
||||
|
||||
static_configs:
|
||||
- targets:
|
||||
- 10.0.1.2:9100
|
||||
- 10.0.1.3:9100
|
||||
- 10.0.1.4:9100
|
||||
- 10.0.1.6:9100
|
||||
- 10.0.1.7:9100
|
||||
- 10.0.1.8:9100
|
||||
- 10.0.1.9:9100
|
||||
- 10.0.1.10:9100
|
||||
- 10.0.1.11:9100
|
||||
- 10.0.1.12:9100
|
||||
- 10.0.1.20:9100
|
||||
- 10.0.2.5:9100
|
||||
- 10.0.3.2:9100
|
||||
@@ -0,0 +1,12 @@
|
||||
config:
|
||||
modules:
|
||||
default:
|
||||
user: "Admin"
|
||||
pass: "Richa1000"
|
||||
privilege: "administrator"
|
||||
driver: "LAN_2_0"
|
||||
|
||||
collectors:
|
||||
- ipmi
|
||||
- chassis
|
||||
- bmc
|
||||
@@ -0,0 +1,415 @@
|
||||
prometheus:
|
||||
prometheusSpec:
|
||||
retention: 7d
|
||||
resources:
|
||||
requests:
|
||||
memory: 512Mi
|
||||
limits:
|
||||
memory: 1Gi
|
||||
additionalScrapeConfigs:
|
||||
|
||||
- job_name: 'blackbox-icmp'
|
||||
|
||||
metrics_path: /probe
|
||||
|
||||
params:
|
||||
module: [icmp]
|
||||
|
||||
static_configs:
|
||||
- targets: ["10.0.0.1"]
|
||||
labels:
|
||||
hostname: router
|
||||
device_type: router
|
||||
|
||||
- targets: ["10.0.0.123"]
|
||||
labels:
|
||||
hostname: switch
|
||||
device_type: switch
|
||||
|
||||
- targets: ["10.0.0.232"]
|
||||
labels:
|
||||
hostname: ap-loznice
|
||||
device_type: ap
|
||||
|
||||
- targets: ["10.0.0.233"]
|
||||
labels:
|
||||
hostname: ap-det-pokoj
|
||||
device_type: ap
|
||||
|
||||
- targets: ["10.0.0.13"]
|
||||
labels:
|
||||
hostname: konica-minolta-bizhub-227
|
||||
device_type: printer
|
||||
|
||||
- targets: ["10.0.0.152"]
|
||||
labels:
|
||||
hostname: hp-officejet-pro-6830
|
||||
device_type: printer
|
||||
|
||||
- targets: ["10.0.2.2"]
|
||||
labels:
|
||||
hostname: main-nas
|
||||
device_type: nas
|
||||
|
||||
- targets: ["10.0.2.12"]
|
||||
labels:
|
||||
hostname: backup-nas
|
||||
device_type: nas
|
||||
|
||||
- targets: ["10.0.2.6"]
|
||||
labels:
|
||||
hostname: nas-raid-card
|
||||
device_type: storage-controller
|
||||
|
||||
- targets: ["10.0.2.4"]
|
||||
labels:
|
||||
hostname: mac-mini
|
||||
device_type: mac
|
||||
|
||||
- targets: ["10.0.0.139"]
|
||||
labels:
|
||||
hostname: klipper-rpi4b
|
||||
device_type: raspberry-pi
|
||||
|
||||
- targets: ["10.0.2.130"]
|
||||
labels:
|
||||
hostname: r720-overlord-idrac
|
||||
device_type: idrac
|
||||
|
||||
- targets: ["10.0.2.131"]
|
||||
labels:
|
||||
hostname: r620-idrac
|
||||
device_type: idrac
|
||||
|
||||
- targets: ["10.0.2.132"]
|
||||
labels:
|
||||
hostname: r720-proxmox-idrac
|
||||
device_type: idrac
|
||||
|
||||
- targets: ["10.0.2.133"]
|
||||
labels:
|
||||
hostname: hp-dl360p-g8-ilo
|
||||
device_type: ilo
|
||||
|
||||
- targets: ["10.0.2.134"]
|
||||
labels:
|
||||
hostname: dell-precision-rack-7910-idrac
|
||||
device_type: idrac
|
||||
|
||||
- targets: ["10.0.2.136"]
|
||||
labels:
|
||||
hostname: r420-idrac
|
||||
device_type: idrac
|
||||
|
||||
- targets: ["10.0.2.137"]
|
||||
labels:
|
||||
hostname: t620-idrac
|
||||
device_type: idrac
|
||||
|
||||
- targets: ["10.0.1.2"]
|
||||
labels:
|
||||
hostname: plexisklo
|
||||
device_type: lxc
|
||||
|
||||
- targets: ["10.0.1.3"]
|
||||
labels:
|
||||
hostname: synch
|
||||
device_type: lxc
|
||||
|
||||
- targets: ["10.0.1.4"]
|
||||
labels:
|
||||
hostname: graylog
|
||||
device_type: vm
|
||||
|
||||
- targets: ["10.0.1.6"]
|
||||
labels:
|
||||
hostname: kozel-tajne
|
||||
device_type: lxc
|
||||
|
||||
- targets: ["10.0.1.7"]
|
||||
labels:
|
||||
hostname: kozel-web
|
||||
device_type: lxc
|
||||
|
||||
- targets: ["10.0.1.8"]
|
||||
labels:
|
||||
hostname: qbittorrent
|
||||
device_type: lxc
|
||||
|
||||
- targets: ["10.0.1.9"]
|
||||
labels:
|
||||
hostname: docker
|
||||
device_type: lxc
|
||||
|
||||
- targets: ["10.0.1.10"]
|
||||
labels:
|
||||
hostname: pihole
|
||||
device_type: lxc
|
||||
|
||||
- targets: ["10.0.1.11"]
|
||||
labels:
|
||||
hostname: gitea
|
||||
device_type: lxc
|
||||
|
||||
- targets: ["10.0.1.12"]
|
||||
labels:
|
||||
hostname: mariadb
|
||||
device_type: lxc
|
||||
|
||||
- targets: ["10.0.1.20"]
|
||||
labels:
|
||||
hostname: zabbix
|
||||
device_type: vm
|
||||
|
||||
- targets: ["10.0.2.5"]
|
||||
labels:
|
||||
hostname: proxmox
|
||||
device_type: server
|
||||
|
||||
- targets: ["10.0.3.2"]
|
||||
labels:
|
||||
hostname: mc-servers
|
||||
device_type: server
|
||||
|
||||
relabel_configs:
|
||||
|
||||
- source_labels: [__address__]
|
||||
target_label: __param_target
|
||||
|
||||
- source_labels: [__param_target]
|
||||
target_label: instance
|
||||
|
||||
- target_label: __address__
|
||||
replacement: blackbox-exporter-prometheus-blackbox-exporter.monitoring.svc.cluster.local:9115
|
||||
|
||||
- job_name: external-node-exporters
|
||||
|
||||
static_configs:
|
||||
- targets:
|
||||
- 10.0.1.2:9100
|
||||
labels:
|
||||
hostname: plexisklo
|
||||
|
||||
- targets:
|
||||
- 10.0.1.3:9100
|
||||
labels:
|
||||
hostname: synch
|
||||
|
||||
- targets:
|
||||
- 10.0.1.4:9100
|
||||
labels:
|
||||
hostname: graylog
|
||||
|
||||
- targets:
|
||||
- 10.0.1.6:9100
|
||||
labels:
|
||||
hostname: kozel-tajne
|
||||
|
||||
- targets:
|
||||
- 10.0.1.7:9100
|
||||
labels:
|
||||
hostname: kozel-web
|
||||
|
||||
- targets:
|
||||
- 10.0.1.8:9100
|
||||
labels:
|
||||
hostname: qbittorrent
|
||||
|
||||
- targets:
|
||||
- 10.0.1.9:9100
|
||||
labels:
|
||||
hostname: docker
|
||||
|
||||
- targets:
|
||||
- 10.0.1.10:9100
|
||||
labels:
|
||||
hostname: pihloe
|
||||
|
||||
- targets:
|
||||
- 10.0.1.11:9100
|
||||
labels:
|
||||
hostname: gitea
|
||||
|
||||
- targets:
|
||||
- 10.0.1.12:9100
|
||||
labels:
|
||||
hostname: mariadb
|
||||
|
||||
- targets:
|
||||
- 10.0.1.20:9100
|
||||
labels:
|
||||
hostname: zabbix
|
||||
|
||||
- targets:
|
||||
- 10.0.2.5:9100
|
||||
labels:
|
||||
hostname: proxmox
|
||||
|
||||
- targets:
|
||||
- 10.0.3.2:9100
|
||||
labels:
|
||||
hostname: mc-servers
|
||||
|
||||
- job_name: 'snmp-if-mib'
|
||||
scrape_interval: 4m
|
||||
scrape_timeout: 180s
|
||||
metrics_path: /snmp
|
||||
params:
|
||||
module: [if_mib]
|
||||
auth: [public_v2]
|
||||
static_configs:
|
||||
- targets:
|
||||
- 10.0.0.1 # Router
|
||||
- 10.0.0.123 # Switch
|
||||
- 10.0.0.233 # AP-Dět.Pkoj
|
||||
- 10.0.0.232 # AP-Ložnice
|
||||
- 10.0.2.12 # Backup-NAS
|
||||
- 10.0.2.2 # Main NAS
|
||||
- 10.0.0.13 # Konica Minolta
|
||||
relabel_configs:
|
||||
- source_labels: [__address__]
|
||||
target_label: __param_target
|
||||
- source_labels: [__param_target]
|
||||
target_label: instance
|
||||
- target_label: __address__
|
||||
replacement: snmp-exporter-manual.monitoring.svc.cluster.local:9116
|
||||
|
||||
# - job_name: 'snmp-mib'
|
||||
# scrape_interval: 4m
|
||||
# scrape_timeout: 180s
|
||||
# metrics_path: /snmp
|
||||
# params:
|
||||
# module: [mikrotik]
|
||||
# auth: [public_v2]
|
||||
# static_configs:
|
||||
# - targets:
|
||||
# - 10.0.0.1 # Router
|
||||
# - 10.0.0.123 # Switch
|
||||
# - 10.0.0.233 # AP-Dět.Pkoj
|
||||
# - 10.0.0.232 # AP-Ložnice
|
||||
# relabel_configs:
|
||||
# - source_labels: [__address__]
|
||||
# target_label: __param_target
|
||||
# - source_labels: [__param_target]
|
||||
# target_label: instance
|
||||
# - target_label: __address__
|
||||
# replacement: snmp-exporter-manual.monitoring.svc.cluster.local:9116
|
||||
|
||||
# - job_name: 'snmp-host-resources'
|
||||
# scrape_interval: 4m
|
||||
# scrape_timeout: 180s
|
||||
# metrics_path: /snmp
|
||||
# params:
|
||||
# module: [resources_hst]
|
||||
# auth: [public_v2]
|
||||
# static_configs:
|
||||
# - targets:
|
||||
# - 10.0.0.1 # Router
|
||||
# - 10.0.0.123 # Switch
|
||||
# - 10.0.0.233 # AP-Dět.Pkoj
|
||||
# - 10.0.0.232 # AP-Ložnice
|
||||
# - 10.0.2.12 # Backup-NAS
|
||||
# - 10.0.2.2 # Main NAS
|
||||
# - 10.0.2.6 # NAS RAID karta
|
||||
# - 10.0.0.13 # Konica Minolta
|
||||
# relabel_configs:
|
||||
# - source_labels: [__address__]
|
||||
# target_label: __param_target
|
||||
# - source_labels: [__param_target]
|
||||
# target_label: instance
|
||||
# - target_label: __address__
|
||||
# replacement: snmp-exporter-manual.monitoring.svc.cluster.local:9116
|
||||
|
||||
# - job_name: 'snmp-dell-idrac'
|
||||
# scrape_interval: 4m
|
||||
# scrape_timeout: 180s
|
||||
# metrics_path: /snmp
|
||||
# params:
|
||||
# module: [dell_rac]
|
||||
# auth: [public_v2]
|
||||
# static_configs:
|
||||
# - targets:
|
||||
# - 10.0.2.130
|
||||
# - 10.0.2.131
|
||||
# - 10.0.2.132
|
||||
# - 10.0.2.134
|
||||
# - 10.0.2.136
|
||||
# - 10.0.2.137
|
||||
# relabel_configs:
|
||||
# - source_labels: [__address__]
|
||||
# target_label: __param_target
|
||||
# - source_labels: [__param_target]
|
||||
# target_label: instance
|
||||
# - target_label: __address__
|
||||
# replacement: snmp-exporter-manual.monitoring.svc.cluster.local:9116
|
||||
|
||||
# - job_name: 'snmp-hpe-ilo'
|
||||
# scrape_interval: 4m
|
||||
# scrape_timeout: 180s
|
||||
# metrics_path: /snmp
|
||||
# params:
|
||||
# module: [hpe]
|
||||
# auth: [public_v2]
|
||||
# static_configs:
|
||||
# - targets:
|
||||
# - 10.0.2.133
|
||||
# relabel_configs:
|
||||
# - source_labels: [__address__]
|
||||
# target_label: __param_target
|
||||
# - source_labels: [__param_target]
|
||||
# target_label: instance
|
||||
# - target_label: __address__
|
||||
# replacement: snmp-exporter-manual.monitoring.svc.cluster.local:9116
|
||||
|
||||
- job_name: 'proxmox'
|
||||
metrics_path: /pve
|
||||
params:
|
||||
module: [default]
|
||||
cluster: ['1']
|
||||
node: ['1']
|
||||
static_configs:
|
||||
- targets:
|
||||
- 10.0.2.5
|
||||
relabel_configs:
|
||||
- source_labels: [__address__]
|
||||
target_label: __param_target
|
||||
- source_labels: [__param_target]
|
||||
target_label: instance
|
||||
- target_label: __address__
|
||||
replacement: pve-exporter.monitoring.svc.cluster.local:9221
|
||||
|
||||
- job_name: 'ipmi-dell-ilo'
|
||||
metrics_path: /ipmi
|
||||
params:
|
||||
module: [default]
|
||||
static_configs:
|
||||
- targets:
|
||||
- 10.0.2.130
|
||||
- 10.0.2.131
|
||||
- 10.0.2.132
|
||||
- 10.0.2.133
|
||||
- 10.0.2.134
|
||||
- 10.0.2.136
|
||||
- 10.0.2.137
|
||||
relabel_configs:
|
||||
- source_labels: [__address__]
|
||||
target_label: __param_target
|
||||
- source_labels: [__param_target]
|
||||
target_label: instance
|
||||
- target_label: __address__
|
||||
replacement: ipmi-exporter-prometheus-ipmi-exporter.monitoring.svc.cluster.local:9290
|
||||
|
||||
grafana:
|
||||
resources:
|
||||
requests:
|
||||
memory: 128Mi
|
||||
limits:
|
||||
memory: 256Mi
|
||||
|
||||
alertmanager:
|
||||
alertmanagerSpec:
|
||||
resources:
|
||||
requests:
|
||||
memory: 128Mi
|
||||
limits:
|
||||
memory: 256Mi
|
||||
Reference in New Issue
Block a user