424 lines
15 KiB
YAML
424 lines
15 KiB
YAML
apiVersion: monitoring.coreos.com/v1
|
|
kind: PrometheusRule
|
|
metadata:
|
|
name: proxmox-alerts
|
|
namespace: monitoring
|
|
labels:
|
|
release: kube-prometheus-stack
|
|
|
|
spec:
|
|
groups:
|
|
- name: proxmox-targets
|
|
rules:
|
|
- alert: ProxmoxTargetDown
|
|
expr: up{job=~"proxmox.*|.*pve.*"} == 0
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Proxmox exporter target down"
|
|
description: "{{ $labels.instance }} Proxmox/PVE scrape is down."
|
|
|
|
- alert: ProxmoxScrapeSlow
|
|
expr: scrape_duration_seconds{job=~"proxmox.*|.*pve.*"} > 20
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Proxmox scrape slow"
|
|
description: "{{ $labels.instance }} Proxmox scrape takes more than 20 seconds."
|
|
|
|
- alert: ProxmoxScrapeVerySlow
|
|
expr: scrape_duration_seconds{job=~"proxmox.*|.*pve.*"} > 60
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Proxmox scrape very slow"
|
|
description: "{{ $labels.instance }} Proxmox scrape takes more than 60 seconds."
|
|
|
|
- alert: ProxmoxLowSamples
|
|
expr: scrape_samples_scraped{job=~"proxmox.*|.*pve.*"} < 20
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Proxmox low sample count"
|
|
description: "{{ $labels.instance }} Proxmox scrape returns fewer than 20 samples."
|
|
|
|
- name: proxmox-nodes
|
|
rules:
|
|
- alert: ProxmoxNodeDown
|
|
expr: pve_node_status == 0
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Proxmox node down"
|
|
description: "Proxmox node {{ $labels.node }} is down."
|
|
|
|
- alert: ProxmoxNodeHighCPU
|
|
expr: pve_node_cpu_usage_ratio > 0.85
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Proxmox node high CPU"
|
|
description: "Node {{ $labels.node }} CPU usage is above 85%."
|
|
|
|
- alert: ProxmoxNodeCriticalCPU
|
|
expr: pve_node_cpu_usage_ratio > 0.95
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Proxmox node critical CPU"
|
|
description: "Node {{ $labels.node }} CPU usage is above 95%."
|
|
|
|
- alert: ProxmoxNodeHighMemory
|
|
expr: pve_node_memory_usage_bytes / pve_node_memory_size_bytes > 0.90
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Proxmox node high memory"
|
|
description: "Node {{ $labels.node }} memory usage is above 90%."
|
|
|
|
- alert: ProxmoxNodeCriticalMemory
|
|
expr: pve_node_memory_usage_bytes / pve_node_memory_size_bytes > 0.97
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Proxmox node critical memory"
|
|
description: "Node {{ $labels.node }} memory usage is above 97%."
|
|
|
|
- alert: ProxmoxNodeLowFreeMemory
|
|
expr: pve_node_memory_free_bytes / pve_node_memory_size_bytes < 0.10
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Proxmox node low free memory"
|
|
description: "Node {{ $labels.node }} has less than 10% free memory."
|
|
|
|
- alert: ProxmoxNodeVeryLowFreeMemory
|
|
expr: pve_node_memory_free_bytes / pve_node_memory_size_bytes < 0.05
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Proxmox node very low free memory"
|
|
description: "Node {{ $labels.node }} has less than 5% free memory."
|
|
|
|
- alert: ProxmoxNodeHighLoad
|
|
expr: pve_node_loadavg_15 / pve_node_cpu_count > 1.5
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Proxmox node high load"
|
|
description: "Node {{ $labels.node }} has high 15m load."
|
|
|
|
- alert: ProxmoxNodeCriticalLoad
|
|
expr: pve_node_loadavg_15 / pve_node_cpu_count > 3
|
|
for: 10m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Proxmox node critical load"
|
|
description: "Node {{ $labels.node }} has critically high 15m load."
|
|
|
|
- alert: ProxmoxNodeUptimeTooLow
|
|
expr: pve_node_uptime_seconds < 600
|
|
for: 1m
|
|
labels:
|
|
severity: info
|
|
annotations:
|
|
summary: "Proxmox node recently rebooted"
|
|
description: "Node {{ $labels.node }} uptime is less than 10 minutes."
|
|
|
|
- name: proxmox-guests
|
|
rules:
|
|
- alert: ProxmoxGuestDown
|
|
expr: pve_guest_status == 0
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Proxmox guest down"
|
|
description: "Guest {{ $labels.id }} {{ $labels.name }} on node {{ $labels.node }} is down."
|
|
|
|
- alert: ProxmoxGuestHighCPU
|
|
expr: pve_guest_cpu_usage_ratio > 0.90
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Proxmox guest high CPU"
|
|
description: "Guest {{ $labels.id }} {{ $labels.name }} CPU usage is above 90%."
|
|
|
|
- alert: ProxmoxGuestCriticalCPU
|
|
expr: pve_guest_cpu_usage_ratio > 0.98
|
|
for: 10m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Proxmox guest critical CPU"
|
|
description: "Guest {{ $labels.id }} {{ $labels.name }} CPU usage is above 98%."
|
|
|
|
- alert: ProxmoxGuestHighMemory
|
|
expr: pve_guest_memory_usage_bytes / pve_guest_memory_size_bytes > 0.90
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Proxmox guest high memory"
|
|
description: "Guest {{ $labels.id }} {{ $labels.name }} memory usage is above 90%."
|
|
|
|
- alert: ProxmoxGuestCriticalMemory
|
|
expr: pve_guest_memory_usage_bytes / pve_guest_memory_size_bytes > 0.97
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Proxmox guest critical memory"
|
|
description: "Guest {{ $labels.id }} {{ $labels.name }} memory usage is above 97%."
|
|
|
|
- alert: ProxmoxGuestLowFreeMemory
|
|
expr: pve_guest_memory_free_bytes / pve_guest_memory_size_bytes < 0.10
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Proxmox guest low free memory"
|
|
description: "Guest {{ $labels.id }} {{ $labels.name }} has less than 10% free memory."
|
|
|
|
- alert: ProxmoxGuestVeryLowFreeMemory
|
|
expr: pve_guest_memory_free_bytes / pve_guest_memory_size_bytes < 0.05
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Proxmox guest very low free memory"
|
|
description: "Guest {{ $labels.id }} {{ $labels.name }} has less than 5% free memory."
|
|
|
|
- alert: ProxmoxGuestHighDiskUsage
|
|
expr: pve_guest_disk_usage_bytes / pve_guest_disk_size_bytes > 0.85
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Proxmox guest disk high usage"
|
|
description: "Guest {{ $labels.id }} {{ $labels.name }} disk usage is above 85%."
|
|
|
|
- alert: ProxmoxGuestCriticalDiskUsage
|
|
expr: pve_guest_disk_usage_bytes / pve_guest_disk_size_bytes > 0.95
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Proxmox guest disk critical usage"
|
|
description: "Guest {{ $labels.id }} {{ $labels.name }} disk usage is above 95%."
|
|
|
|
- alert: ProxmoxGuestUnexpectedlyRunning
|
|
expr: pve_guest_status == 1 and on(id) pve_guest_template == 1
|
|
for: 10m
|
|
labels:
|
|
severity: info
|
|
annotations:
|
|
summary: "Proxmox template seems running"
|
|
description: "Template guest {{ $labels.id }} {{ $labels.name }} appears to be running."
|
|
|
|
- alert: ProxmoxGuestRecentlyRebooted
|
|
expr: pve_guest_uptime_seconds > 0 and pve_guest_uptime_seconds < 600
|
|
for: 1m
|
|
labels:
|
|
severity: info
|
|
annotations:
|
|
summary: "Proxmox guest recently rebooted"
|
|
description: "Guest {{ $labels.id }} {{ $labels.name }} uptime is less than 10 minutes."
|
|
|
|
- name: proxmox-storage
|
|
rules:
|
|
- alert: ProxmoxStorageHighUsage
|
|
expr: pve_storage_usage_bytes / pve_storage_size_bytes > 0.85
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Proxmox storage high usage"
|
|
description: "Storage {{ $labels.storage }} on node {{ $labels.node }} is above 85%."
|
|
|
|
- alert: ProxmoxStorageCriticalUsage
|
|
expr: pve_storage_usage_bytes / pve_storage_size_bytes > 0.95
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Proxmox storage critical usage"
|
|
description: "Storage {{ $labels.storage }} on node {{ $labels.node }} is above 95%."
|
|
|
|
- alert: ProxmoxStorageLowFree
|
|
expr: pve_storage_free_bytes / pve_storage_size_bytes < 0.15
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Proxmox storage low free"
|
|
description: "Storage {{ $labels.storage }} on node {{ $labels.node }} has less than 15% free space."
|
|
|
|
- alert: ProxmoxStorageVeryLowFree
|
|
expr: pve_storage_free_bytes / pve_storage_size_bytes < 0.05
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Proxmox storage very low free"
|
|
description: "Storage {{ $labels.storage }} on node {{ $labels.node }} has less than 5% free space."
|
|
|
|
- alert: ProxmoxStorageDisabled
|
|
expr: pve_storage_status == 0
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Proxmox storage disabled/unavailable"
|
|
description: "Storage {{ $labels.storage }} on node {{ $labels.node }} is disabled or unavailable."
|
|
|
|
- name: proxmox-cluster
|
|
rules:
|
|
- alert: ProxmoxClusterQuorumLost
|
|
expr: pve_cluster_quorate == 0
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Proxmox cluster quorum lost"
|
|
description: "Proxmox cluster has lost quorum."
|
|
|
|
- alert: ProxmoxClusterNodeOffline
|
|
expr: pve_cluster_node_online == 0
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Proxmox cluster node offline"
|
|
description: "Cluster node {{ $labels.node }} is offline."
|
|
|
|
- alert: ProxmoxClusterExpectedVotesLow
|
|
expr: pve_cluster_expected_votes < 2
|
|
for: 10m
|
|
labels:
|
|
severity: info
|
|
annotations:
|
|
summary: "Proxmox cluster expected votes low"
|
|
description: "Proxmox cluster expected votes is below 2."
|
|
|
|
- name: proxmox-backups
|
|
rules:
|
|
- alert: ProxmoxBackupFailed
|
|
expr: pve_guest_backup_last_status == 0
|
|
for: 10m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Proxmox backup failed"
|
|
description: "Backup for guest {{ $labels.id }} {{ $labels.name }} failed."
|
|
|
|
- alert: ProxmoxBackupTooOld
|
|
expr: time() - pve_guest_backup_last_timestamp_seconds > 7 * 24 * 3600
|
|
for: 1h
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Proxmox backup too old"
|
|
description: "Backup for guest {{ $labels.id }} {{ $labels.name }} is older than 7 days."
|
|
|
|
- alert: ProxmoxNoRecentBackup
|
|
expr: absent(pve_guest_backup_last_timestamp_seconds)
|
|
for: 1h
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "No Proxmox backup metric found"
|
|
description: "No Proxmox backup timestamp metrics are present."
|
|
|
|
- name: proxmox-replication-ha
|
|
rules:
|
|
- alert: ProxmoxReplicationFailed
|
|
expr: pve_replication_status == 0
|
|
for: 10m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Proxmox replication failed"
|
|
description: "Replication job {{ $labels.job }} on node {{ $labels.node }} failed."
|
|
|
|
- alert: ProxmoxReplicationTooOld
|
|
expr: time() - pve_replication_last_sync_timestamp_seconds > 6 * 3600
|
|
for: 30m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Proxmox replication too old"
|
|
description: "Replication job {{ $labels.job }} last sync is older than 6 hours."
|
|
|
|
- alert: ProxmoxHAServiceError
|
|
expr: pve_ha_service_status == 0
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Proxmox HA service error"
|
|
description: "HA service {{ $labels.sid }} is not healthy."
|
|
|
|
- name: proxmox-tasks
|
|
rules:
|
|
- alert: ProxmoxTaskFailed
|
|
expr: increase(pve_tasks_failed_total[30m]) > 0
|
|
for: 1m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Proxmox task failed"
|
|
description: "A Proxmox task failed on {{ $labels.instance }}."
|
|
|
|
- alert: ProxmoxTasksFailingOften
|
|
expr: increase(pve_tasks_failed_total[6h]) > 5
|
|
for: 10m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Proxmox tasks failing often"
|
|
description: "More than 5 Proxmox tasks failed in 6 hours on {{ $labels.instance }}."
|
|
|
|
- name: proxmox-exporter-health
|
|
rules:
|
|
- alert: ProxmoxExporterNoNodeMetrics
|
|
expr: absent(pve_node_status)
|
|
for: 10m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "No Proxmox node metrics"
|
|
description: "PVE exporter is not returning node metrics."
|
|
|
|
- alert: ProxmoxExporterNoGuestMetrics
|
|
expr: absent(pve_guest_status)
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "No Proxmox guest metrics"
|
|
description: "PVE exporter is not returning guest metrics."
|
|
|
|
- alert: ProxmoxExporterNoStorageMetrics
|
|
expr: absent(pve_storage_size_bytes)
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "No Proxmox storage metrics"
|
|
description: "PVE exporter is not returning storage metrics."
|