apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: proxmox-alerts
  namespace: monitoring
  labels:
    release: kube-prometheus-stack

spec:
  groups:
    - name: proxmox-targets
      rules:
        - alert: ProxmoxTargetDown
          expr: up{job=~"proxmox.*|.*pve.*"} == 0
          for: 2m
          labels:
            severity: critical
          annotations:
            summary: "Proxmox exporter target down"
            description: "{{ $labels.instance }} Proxmox/PVE scrape is down."

        - alert: ProxmoxScrapeSlow
          expr: scrape_duration_seconds{job=~"proxmox.*|.*pve.*"} > 20
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "Proxmox scrape slow"
            description: "{{ $labels.instance }} Proxmox scrape takes more than 20 seconds."

        - alert: ProxmoxScrapeVerySlow
          expr: scrape_duration_seconds{job=~"proxmox.*|.*pve.*"} > 60
          for: 2m
          labels:
            severity: critical
          annotations:
            summary: "Proxmox scrape very slow"
            description: "{{ $labels.instance }} Proxmox scrape takes more than 60 seconds."

        - alert: ProxmoxLowSamples
          expr: scrape_samples_scraped{job=~"proxmox.*|.*pve.*"} < 20
          for: 10m
          labels:
            severity: warning
          annotations:
            summary: "Proxmox low sample count"
            description: "{{ $labels.instance }} Proxmox scrape returns fewer than 20 samples."

    - name: proxmox-nodes
      rules:
        - alert: ProxmoxNodeDown
          expr: pve_node_status == 0
          for: 2m
          labels:
            severity: critical
          annotations:
            summary: "Proxmox node down"
            description: "Proxmox node {{ $labels.node }} is down."

        - alert: ProxmoxNodeHighCPU
          expr: pve_node_cpu_usage_ratio > 0.85
          for: 10m
          labels:
            severity: warning
          annotations:
            summary: "Proxmox node high CPU"
            description: "Node {{ $labels.node }} CPU usage is above 85%."

        - alert: ProxmoxNodeCriticalCPU
          expr: pve_node_cpu_usage_ratio > 0.95
          for: 5m
          labels:
            severity: critical
          annotations:
            summary: "Proxmox node critical CPU"
            description: "Node {{ $labels.node }} CPU usage is above 95%."

        - alert: ProxmoxNodeHighMemory
          expr: pve_node_memory_usage_bytes / pve_node_memory_size_bytes > 0.90
          for: 10m
          labels:
            severity: warning
          annotations:
            summary: "Proxmox node high memory"
            description: "Node {{ $labels.node }} memory usage is above 90%."

        - alert: ProxmoxNodeCriticalMemory
          expr: pve_node_memory_usage_bytes / pve_node_memory_size_bytes > 0.97
          for: 5m
          labels:
            severity: critical
          annotations:
            summary: "Proxmox node critical memory"
            description: "Node {{ $labels.node }} memory usage is above 97%."

        - alert: ProxmoxNodeLowFreeMemory
          expr: pve_node_memory_free_bytes / pve_node_memory_size_bytes < 0.10
          for: 10m
          labels:
            severity: warning
          annotations:
            summary: "Proxmox node low free memory"
            description: "Node {{ $labels.node }} has less than 10% free memory."

        - alert: ProxmoxNodeVeryLowFreeMemory
          expr: pve_node_memory_free_bytes / pve_node_memory_size_bytes < 0.05
          for: 5m
          labels:
            severity: critical
          annotations:
            summary: "Proxmox node very low free memory"
            description: "Node {{ $labels.node }} has less than 5% free memory."

        - alert: ProxmoxNodeHighLoad
          expr: pve_node_loadavg_15 / pve_node_cpu_count > 1.5
          for: 15m
          labels:
            severity: warning
          annotations:
            summary: "Proxmox node high load"
            description: "Node {{ $labels.node }} has high 15m load."

        - alert: ProxmoxNodeCriticalLoad
          expr: pve_node_loadavg_15 / pve_node_cpu_count > 3
          for: 10m
          labels:
            severity: critical
          annotations:
            summary: "Proxmox node critical load"
            description: "Node {{ $labels.node }} has critically high 15m load."

        - alert: ProxmoxNodeUptimeTooLow
          expr: pve_node_uptime_seconds < 600
          for: 1m
          labels:
            severity: info
          annotations:
            summary: "Proxmox node recently rebooted"
            description: "Node {{ $labels.node }} uptime is less than 10 minutes."

    - name: proxmox-guests
      rules:
        - alert: ProxmoxGuestDown
          expr: pve_guest_status == 0
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "Proxmox guest down"
            description: "Guest {{ $labels.id }} {{ $labels.name }} on node {{ $labels.node }} is down."

        - alert: ProxmoxGuestHighCPU
          expr: pve_guest_cpu_usage_ratio > 0.90
          for: 15m
          labels:
            severity: warning
          annotations:
            summary: "Proxmox guest high CPU"
            description: "Guest {{ $labels.id }} {{ $labels.name }} CPU usage is above 90%."

        - alert: ProxmoxGuestCriticalCPU
          expr: pve_guest_cpu_usage_ratio > 0.98
          for: 10m
          labels:
            severity: critical
          annotations:
            summary: "Proxmox guest critical CPU"
            description: "Guest {{ $labels.id }} {{ $labels.name }} CPU usage is above 98%."

        - alert: ProxmoxGuestHighMemory
          expr: pve_guest_memory_usage_bytes / pve_guest_memory_size_bytes > 0.90
          for: 15m
          labels:
            severity: warning
          annotations:
            summary: "Proxmox guest high memory"
            description: "Guest {{ $labels.id }} {{ $labels.name }} memory usage is above 90%."

        - alert: ProxmoxGuestCriticalMemory
          expr: pve_guest_memory_usage_bytes / pve_guest_memory_size_bytes > 0.97
          for: 5m
          labels:
            severity: critical
          annotations:
            summary: "Proxmox guest critical memory"
            description: "Guest {{ $labels.id }} {{ $labels.name }} memory usage is above 97%."

        - alert: ProxmoxGuestLowFreeMemory
          expr: pve_guest_memory_free_bytes / pve_guest_memory_size_bytes < 0.10
          for: 15m
          labels:
            severity: warning
          annotations:
            summary: "Proxmox guest low free memory"
            description: "Guest {{ $labels.id }} {{ $labels.name }} has less than 10% free memory."

        - alert: ProxmoxGuestVeryLowFreeMemory
          expr: pve_guest_memory_free_bytes / pve_guest_memory_size_bytes < 0.05
          for: 5m
          labels:
            severity: critical
          annotations:
            summary: "Proxmox guest very low free memory"
            description: "Guest {{ $labels.id }} {{ $labels.name }} has less than 5% free memory."

        - alert: ProxmoxGuestHighDiskUsage
          expr: pve_guest_disk_usage_bytes / pve_guest_disk_size_bytes > 0.85
          for: 15m
          labels:
            severity: warning
          annotations:
            summary: "Proxmox guest disk high usage"
            description: "Guest {{ $labels.id }} {{ $labels.name }} disk usage is above 85%."

        - alert: ProxmoxGuestCriticalDiskUsage
          expr: pve_guest_disk_usage_bytes / pve_guest_disk_size_bytes > 0.95
          for: 5m
          labels:
            severity: critical
          annotations:
            summary: "Proxmox guest disk critical usage"
            description: "Guest {{ $labels.id }} {{ $labels.name }} disk usage is above 95%."

        - alert: ProxmoxGuestUnexpectedlyRunning
          expr: pve_guest_status == 1 and on(id) pve_guest_template == 1
          for: 10m
          labels:
            severity: info
          annotations:
            summary: "Proxmox template seems running"
            description: "Template guest {{ $labels.id }} {{ $labels.name }} appears to be running."

        - alert: ProxmoxGuestRecentlyRebooted
          expr: pve_guest_uptime_seconds > 0 and pve_guest_uptime_seconds < 600
          for: 1m
          labels:
            severity: info
          annotations:
            summary: "Proxmox guest recently rebooted"
            description: "Guest {{ $labels.id }} {{ $labels.name }} uptime is less than 10 minutes."

    - name: proxmox-storage
      rules:
        - alert: ProxmoxStorageHighUsage
          expr: pve_storage_usage_bytes / pve_storage_size_bytes > 0.85
          for: 10m
          labels:
            severity: warning
          annotations:
            summary: "Proxmox storage high usage"
            description: "Storage {{ $labels.storage }} on node {{ $labels.node }} is above 85%."

        - alert: ProxmoxStorageCriticalUsage
          expr: pve_storage_usage_bytes / pve_storage_size_bytes > 0.95
          for: 5m
          labels:
            severity: critical
          annotations:
            summary: "Proxmox storage critical usage"
            description: "Storage {{ $labels.storage }} on node {{ $labels.node }} is above 95%."

        - alert: ProxmoxStorageLowFree
          expr: pve_storage_free_bytes / pve_storage_size_bytes < 0.15
          for: 10m
          labels:
            severity: warning
          annotations:
            summary: "Proxmox storage low free"
            description: "Storage {{ $labels.storage }} on node {{ $labels.node }} has less than 15% free space."

        - alert: ProxmoxStorageVeryLowFree
          expr: pve_storage_free_bytes / pve_storage_size_bytes < 0.05
          for: 5m
          labels:
            severity: critical
          annotations:
            summary: "Proxmox storage very low free"
            description: "Storage {{ $labels.storage }} on node {{ $labels.node }} has less than 5% free space."

        - alert: ProxmoxStorageDisabled
          expr: pve_storage_status == 0
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "Proxmox storage disabled/unavailable"
            description: "Storage {{ $labels.storage }} on node {{ $labels.node }} is disabled or unavailable."

    - name: proxmox-cluster
      rules:
        - alert: ProxmoxClusterQuorumLost
          expr: pve_cluster_quorate == 0
          for: 1m
          labels:
            severity: critical
          annotations:
            summary: "Proxmox cluster quorum lost"
            description: "Proxmox cluster has lost quorum."

        - alert: ProxmoxClusterNodeOffline
          expr: pve_cluster_node_online == 0
          for: 2m
          labels:
            severity: critical
          annotations:
            summary: "Proxmox cluster node offline"
            description: "Cluster node {{ $labels.node }} is offline."

        - alert: ProxmoxClusterExpectedVotesLow
          expr: pve_cluster_expected_votes < 2
          for: 10m
          labels:
            severity: info
          annotations:
            summary: "Proxmox cluster expected votes low"
            description: "Proxmox cluster expected votes is below 2."

    - name: proxmox-backups
      rules:
        - alert: ProxmoxBackupFailed
          expr: pve_guest_backup_last_status == 0
          for: 10m
          labels:
            severity: critical
          annotations:
            summary: "Proxmox backup failed"
            description: "Backup for guest {{ $labels.id }} {{ $labels.name }} failed."

        - alert: ProxmoxBackupTooOld
          expr: time() - pve_guest_backup_last_timestamp_seconds > 7 * 24 * 3600
          for: 1h
          labels:
            severity: warning
          annotations:
            summary: "Proxmox backup too old"
            description: "Backup for guest {{ $labels.id }} {{ $labels.name }} is older than 7 days."

        - alert: ProxmoxNoRecentBackup
          expr: absent(pve_guest_backup_last_timestamp_seconds)
          for: 1h
          labels:
            severity: warning
          annotations:
            summary: "No Proxmox backup metric found"
            description: "No Proxmox backup timestamp metrics are present."

    - name: proxmox-replication-ha
      rules:
        - alert: ProxmoxReplicationFailed
          expr: pve_replication_status == 0
          for: 10m
          labels:
            severity: critical
          annotations:
            summary: "Proxmox replication failed"
            description: "Replication job {{ $labels.job }} on node {{ $labels.node }} failed."

        - alert: ProxmoxReplicationTooOld
          expr: time() - pve_replication_last_sync_timestamp_seconds > 6 * 3600
          for: 30m
          labels:
            severity: warning
          annotations:
            summary: "Proxmox replication too old"
            description: "Replication job {{ $labels.job }} last sync is older than 6 hours."

        - alert: ProxmoxHAServiceError
          expr: pve_ha_service_status == 0
          for: 5m
          labels:
            severity: critical
          annotations:
            summary: "Proxmox HA service error"
            description: "HA service {{ $labels.sid }} is not healthy."

    - name: proxmox-tasks
      rules:
        - alert: ProxmoxTaskFailed
          expr: increase(pve_tasks_failed_total[30m]) > 0
          for: 1m
          labels:
            severity: warning
          annotations:
            summary: "Proxmox task failed"
            description: "A Proxmox task failed on {{ $labels.instance }}."

        - alert: ProxmoxTasksFailingOften
          expr: increase(pve_tasks_failed_total[6h]) > 5
          for: 10m
          labels:
            severity: critical
          annotations:
            summary: "Proxmox tasks failing often"
            description: "More than 5 Proxmox tasks failed in 6 hours on {{ $labels.instance }}."

    - name: proxmox-exporter-health
      rules:
        - alert: ProxmoxExporterNoNodeMetrics
          expr: absent(pve_node_status)
          for: 10m
          labels:
            severity: critical
          annotations:
            summary: "No Proxmox node metrics"
            description: "PVE exporter is not returning node metrics."

        - alert: ProxmoxExporterNoGuestMetrics
          expr: absent(pve_guest_status)
          for: 10m
          labels:
            severity: warning
          annotations:
            summary: "No Proxmox guest metrics"
            description: "PVE exporter is not returning guest metrics."

        - alert: ProxmoxExporterNoStorageMetrics
          expr: absent(pve_storage_size_bytes)
          for: 10m
          labels:
            severity: warning
          annotations:
            summary: "No Proxmox storage metrics"
            description: "PVE exporter is not returning storage metrics."