Added a BUNCH of configs.
This commit is contained in:
@@ -0,0 +1,470 @@
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: node-exporter-alerts
|
||||
namespace: monitoring
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
|
||||
spec:
|
||||
groups:
|
||||
- name: node-exporter-availability
|
||||
rules:
|
||||
- alert: NodeDown
|
||||
expr: up{job=~".*node.*|external-node-exporters"} == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Node down"
|
||||
description: "{{ $labels.instance }} is down."
|
||||
|
||||
- alert: NodeExporterMissing
|
||||
expr: absent(up{job=~".*node.*|external-node-exporters"})
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Node exporter missing"
|
||||
description: "No node-exporter targets are currently being scraped."
|
||||
|
||||
- name: node-exporter-cpu-load
|
||||
rules:
|
||||
- alert: NodeHighCPU
|
||||
expr: 100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 85
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High CPU usage"
|
||||
description: "{{ $labels.instance }} CPU usage is above 85% for 10 minutes."
|
||||
|
||||
- alert: NodeCriticalCPU
|
||||
expr: 100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 95
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Critical CPU usage"
|
||||
description: "{{ $labels.instance }} CPU usage is above 95%."
|
||||
|
||||
- alert: NodeHighLoad1
|
||||
expr: node_load1 / count by(instance) (node_cpu_seconds_total{mode="idle"}) > 2
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High 1m load"
|
||||
description: "{{ $labels.instance }} has high 1-minute load."
|
||||
|
||||
- alert: NodeHighLoad5
|
||||
expr: node_load5 / count by(instance) (node_cpu_seconds_total{mode="idle"}) > 1.75
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High 5m load"
|
||||
description: "{{ $labels.instance }} has high 5-minute load."
|
||||
|
||||
- alert: NodeHighLoad15
|
||||
expr: node_load15 / count by(instance) (node_cpu_seconds_total{mode="idle"}) > 1.5
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High 15m load"
|
||||
description: "{{ $labels.instance }} has high 15-minute load."
|
||||
|
||||
- alert: NodeCriticalLoad15
|
||||
expr: node_load15 / count by(instance) (node_cpu_seconds_total{mode="idle"}) > 3
|
||||
for: 10m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Critical 15m load"
|
||||
description: "{{ $labels.instance }} has critically high 15-minute load."
|
||||
|
||||
- name: node-exporter-memory-swap
|
||||
rules:
|
||||
- alert: NodeMemoryLow
|
||||
expr: (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) < 0.10
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Low available memory"
|
||||
description: "{{ $labels.instance }} has less than 10% memory available."
|
||||
|
||||
- alert: NodeMemoryCritical
|
||||
expr: (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) < 0.05
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Critical available memory"
|
||||
description: "{{ $labels.instance }} has less than 5% memory available."
|
||||
|
||||
- alert: NodeSwapHigh
|
||||
expr: (node_memory_SwapTotal_bytes > 0) and ((node_memory_SwapTotal_bytes - node_memory_SwapFree_bytes) / node_memory_SwapTotal_bytes > 0.50)
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High swap usage"
|
||||
description: "{{ $labels.instance }} swap usage is above 50%."
|
||||
|
||||
- alert: NodeSwapCritical
|
||||
expr: (node_memory_SwapTotal_bytes > 0) and ((node_memory_SwapTotal_bytes - node_memory_SwapFree_bytes) / node_memory_SwapTotal_bytes > 0.80)
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Critical swap usage"
|
||||
description: "{{ $labels.instance }} swap usage is above 80%."
|
||||
|
||||
- alert: NodeOOMKills
|
||||
expr: increase(node_vmstat_oom_kill[10m]) > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "OOM kill detected"
|
||||
description: "{{ $labels.instance }} had OOM kill event in last 10 minutes."
|
||||
|
||||
- name: node-exporter-filesystems
|
||||
rules:
|
||||
- alert: NodeRootFilesystemLow
|
||||
expr: (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs|nsfs|ramfs",mountpoint="/"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay|squashfs|nsfs|ramfs",mountpoint="/"}) < 0.15
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Root filesystem low"
|
||||
description: "{{ $labels.instance }} root filesystem has less than 15% free space."
|
||||
|
||||
- alert: NodeRootFilesystemCritical
|
||||
expr: (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs|nsfs|ramfs",mountpoint="/"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay|squashfs|nsfs|ramfs",mountpoint="/"}) < 0.05
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Root filesystem critical"
|
||||
description: "{{ $labels.instance }} root filesystem has less than 5% free space."
|
||||
|
||||
- alert: NodeFilesystemLow
|
||||
expr: (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs|nsfs|ramfs"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay|squashfs|nsfs|ramfs"}) < 0.15
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Filesystem low"
|
||||
description: "{{ $labels.instance }} {{ $labels.mountpoint }} has less than 15% free space."
|
||||
|
||||
- alert: NodeFilesystemCritical
|
||||
expr: (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs|nsfs|ramfs"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay|squashfs|nsfs|ramfs"}) < 0.05
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Filesystem critical"
|
||||
description: "{{ $labels.instance }} {{ $labels.mountpoint }} has less than 5% free space."
|
||||
|
||||
- alert: NodeFilesystemWillFillSoon
|
||||
expr: predict_linear(node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs|nsfs|ramfs"}[6h], 24 * 3600) < 0
|
||||
for: 30m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Filesystem will fill soon"
|
||||
description: "{{ $labels.instance }} {{ $labels.mountpoint }} is predicted to fill within 24 hours."
|
||||
|
||||
- alert: NodeFilesystemWillFillVerySoon
|
||||
expr: predict_linear(node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs|nsfs|ramfs"}[2h], 6 * 3600) < 0
|
||||
for: 15m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Filesystem will fill very soon"
|
||||
description: "{{ $labels.instance }} {{ $labels.mountpoint }} is predicted to fill within 6 hours."
|
||||
|
||||
- alert: NodeInodesLow
|
||||
expr: (node_filesystem_files_free{fstype!~"tmpfs|overlay|squashfs|nsfs|ramfs"} / node_filesystem_files{fstype!~"tmpfs|overlay|squashfs|nsfs|ramfs"}) < 0.10
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Filesystem inodes low"
|
||||
description: "{{ $labels.instance }} {{ $labels.mountpoint }} has less than 10% free inodes."
|
||||
|
||||
- alert: NodeInodesCritical
|
||||
expr: (node_filesystem_files_free{fstype!~"tmpfs|overlay|squashfs|nsfs|ramfs"} / node_filesystem_files{fstype!~"tmpfs|overlay|squashfs|nsfs|ramfs"}) < 0.05
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Filesystem inodes critical"
|
||||
description: "{{ $labels.instance }} {{ $labels.mountpoint }} has less than 5% free inodes."
|
||||
|
||||
- alert: NodeFilesystemReadOnly
|
||||
expr: node_filesystem_readonly{fstype!~"tmpfs|overlay|squashfs|nsfs|ramfs"} == 1
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Filesystem read-only"
|
||||
description: "{{ $labels.instance }} {{ $labels.mountpoint }} is read-only."
|
||||
|
||||
- name: node-exporter-disk-io
|
||||
rules:
|
||||
- alert: NodeDiskIOHigh
|
||||
expr: rate(node_disk_io_time_seconds_total{device!~"loop.*|ram.*|dm-.*"}[5m]) > 0.90
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High disk I/O utilization"
|
||||
description: "{{ $labels.instance }} disk {{ $labels.device }} I/O utilization is high."
|
||||
|
||||
- alert: NodeDiskReadLatencyHigh
|
||||
expr: rate(node_disk_read_time_seconds_total{device!~"loop.*|ram.*"}[5m]) / rate(node_disk_reads_completed_total{device!~"loop.*|ram.*"}[5m]) > 0.1
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High disk read latency"
|
||||
description: "{{ $labels.instance }} disk {{ $labels.device }} read latency is high."
|
||||
|
||||
- alert: NodeDiskWriteLatencyHigh
|
||||
expr: rate(node_disk_write_time_seconds_total{device!~"loop.*|ram.*"}[5m]) / rate(node_disk_writes_completed_total{device!~"loop.*|ram.*"}[5m]) > 0.1
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High disk write latency"
|
||||
description: "{{ $labels.instance }} disk {{ $labels.device }} write latency is high."
|
||||
|
||||
- alert: NodeDiskReadErrorsOrWeirdness
|
||||
expr: increase(node_disk_reads_completed_total{device!~"loop.*|ram.*"}[10m]) == 0 and rate(node_disk_read_time_seconds_total{device!~"loop.*|ram.*"}[10m]) > 0
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Possible disk read issue"
|
||||
description: "{{ $labels.instance }} disk {{ $labels.device }} reports read time without completed reads."
|
||||
|
||||
- name: node-exporter-network
|
||||
rules:
|
||||
- alert: NodeNetworkReceiveErrors
|
||||
expr: increase(node_network_receive_errs_total{device!~"lo|veth.*|docker.*|br-.*|cni.*|flannel.*|calico.*"}[10m]) > 10
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Network RX errors"
|
||||
description: "{{ $labels.instance }} interface {{ $labels.device }} has RX errors."
|
||||
|
||||
- alert: NodeNetworkTransmitErrors
|
||||
expr: increase(node_network_transmit_errs_total{device!~"lo|veth.*|docker.*|br-.*|cni.*|flannel.*|calico.*"}[10m]) > 10
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Network TX errors"
|
||||
description: "{{ $labels.instance }} interface {{ $labels.device }} has TX errors."
|
||||
|
||||
- alert: NodeNetworkReceiveDrops
|
||||
expr: increase(node_network_receive_drop_total{device!~"lo|veth.*|docker.*|br-.*|cni.*|flannel.*|calico.*"}[10m]) > 50
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Network RX drops"
|
||||
description: "{{ $labels.instance }} interface {{ $labels.device }} has RX drops."
|
||||
|
||||
- alert: NodeNetworkTransmitDrops
|
||||
expr: increase(node_network_transmit_drop_total{device!~"lo|veth.*|docker.*|br-.*|cni.*|flannel.*|calico.*"}[10m]) > 50
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Network TX drops"
|
||||
description: "{{ $labels.instance }} interface {{ $labels.device }} has TX drops."
|
||||
|
||||
- alert: NodeNetworkInterfaceFlapping
|
||||
expr: changes(node_network_up{device!~"lo|veth.*|docker.*|br-.*|cni.*|flannel.*|calico.*"}[10m]) > 2
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Network interface flapping"
|
||||
description: "{{ $labels.instance }} interface {{ $labels.device }} changed link state more than twice in 10 minutes."
|
||||
|
||||
- alert: NodeNetworkInterfaceDown
|
||||
expr: node_network_up{device!~"lo|veth.*|docker.*|br-.*|cni.*|flannel.*|calico.*"} == 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Network interface down"
|
||||
description: "{{ $labels.instance }} interface {{ $labels.device }} is down."
|
||||
|
||||
- name: node-exporter-system-health
|
||||
rules:
|
||||
- alert: NodeRebooted
|
||||
expr: time() - node_boot_time_seconds < 600
|
||||
for: 1m
|
||||
labels:
|
||||
severity: info
|
||||
annotations:
|
||||
summary: "Node recently rebooted"
|
||||
description: "{{ $labels.instance }} rebooted less than 10 minutes ago."
|
||||
|
||||
- alert: NodeTimeDrift
|
||||
expr: abs(node_timex_offset_seconds) > 0.1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Node time drift"
|
||||
description: "{{ $labels.instance }} has time drift greater than 100 ms."
|
||||
|
||||
- alert: NodeTimeDriftCritical
|
||||
expr: abs(node_timex_offset_seconds) > 1
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Critical node time drift"
|
||||
description: "{{ $labels.instance }} has time drift greater than 1 second."
|
||||
|
||||
- alert: NodeClockNotSynchronized
|
||||
expr: node_timex_sync_status == 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Clock not synchronized"
|
||||
description: "{{ $labels.instance }} clock is not synchronized."
|
||||
|
||||
- alert: NodeEntropyLow
|
||||
expr: node_entropy_available_bits < 256
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Low entropy"
|
||||
description: "{{ $labels.instance }} has low available entropy."
|
||||
|
||||
- alert: NodeFileDescriptorsHigh
|
||||
expr: node_filefd_allocated / node_filefd_maximum > 0.80
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High file descriptor usage"
|
||||
description: "{{ $labels.instance }} file descriptor usage is above 80%."
|
||||
|
||||
- alert: NodeFileDescriptorsCritical
|
||||
expr: node_filefd_allocated / node_filefd_maximum > 0.95
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Critical file descriptor usage"
|
||||
description: "{{ $labels.instance }} file descriptor usage is above 95%."
|
||||
|
||||
- alert: NodeProcessesHigh
|
||||
expr: node_procs_running > 300
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High running process count"
|
||||
description: "{{ $labels.instance }} has more than 300 running processes."
|
||||
|
||||
- alert: NodeProcessesBlocked
|
||||
expr: node_procs_blocked > 5
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Blocked processes"
|
||||
description: "{{ $labels.instance }} has more than 5 blocked processes."
|
||||
|
||||
- name: node-exporter-systemd
|
||||
rules:
|
||||
- alert: NodeSystemdUnitFailed
|
||||
expr: node_systemd_unit_state{state="failed"} == 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Systemd unit failed"
|
||||
description: "{{ $labels.instance }} systemd unit {{ $labels.name }} is failed."
|
||||
|
||||
- alert: NodeSystemdServiceRestartingTooOften
|
||||
expr: increase(node_systemd_service_restart_total[15m]) > 3
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Systemd service restarting too often"
|
||||
description: "{{ $labels.instance }} systemd service {{ $labels.name }} restarted more than 3 times in 15 minutes."
|
||||
|
||||
- name: node-exporter-hardware
|
||||
rules:
|
||||
- alert: NodeTemperatureHigh
|
||||
expr: node_hwmon_temp_celsius > 80
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High hardware temperature"
|
||||
description: "{{ $labels.instance }} sensor {{ $labels.chip }} {{ $labels.sensor }} is above 80°C."
|
||||
|
||||
- alert: NodeTemperatureCritical
|
||||
expr: node_hwmon_temp_celsius > 90
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Critical hardware temperature"
|
||||
description: "{{ $labels.instance }} sensor {{ $labels.chip }} {{ $labels.sensor }} is above 90°C."
|
||||
|
||||
- alert: NodeThermalZoneHigh
|
||||
expr: node_thermal_zone_temp > 80000
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High thermal zone temperature"
|
||||
description: "{{ $labels.instance }} thermal zone {{ $labels.zone }} is above 80°C."
|
||||
|
||||
- alert: NodeThermalZoneCritical
|
||||
expr: node_thermal_zone_temp > 90000
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Critical thermal zone temperature"
|
||||
description: "{{ $labels.instance }} thermal zone {{ $labels.zone }} is above 90°C."
|
||||
|
||||
- name: node-exporter-prometheus-scrape
|
||||
rules:
|
||||
- alert: NodeExporterScrapeSlow
|
||||
expr: scrape_duration_seconds{job=~".*node.*|external-node-exporters"} > 10
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Node exporter scrape slow"
|
||||
description: "Scrape of {{ $labels.instance }} takes more than 10 seconds."
|
||||
|
||||
- alert: NodeExporterLowSamples
|
||||
expr: scrape_samples_scraped{job=~".*node.*|external-node-exporters"} < 100
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Node exporter low sample count"
|
||||
description: "{{ $labels.instance }} returns fewer than 100 samples."
|
||||
Reference in New Issue
Block a user