Added a BUNCH of configs.
This commit is contained in:
@@ -19,3 +19,5 @@ talos/controlplane.yaml
|
|||||||
talos/kubeconfig
|
talos/kubeconfig
|
||||||
talos/talosconfig
|
talos/talosconfig
|
||||||
talos/worker.yaml
|
talos/worker.yaml
|
||||||
|
k8s/security/authentik-values.yaml
|
||||||
|
k8s/monitoring/alert-rules/alertmanager-discord.yml
|
||||||
|
|||||||
@@ -0,0 +1,130 @@
|
|||||||
|
# Homelab Infrastructure
|
||||||
|
|
||||||
|
Infrastructure-as-Code repository for my personal homelab and Kubernetes environment.
|
||||||
|
|
||||||
|
This repository contains configuration files, monitoring stack definitions, alerting rules, Talos Linux cluster configuration, exporters, ingress resources and various infrastructure-related manifests used across my environment.
|
||||||
|
|
||||||
|
## Main Technologies
|
||||||
|
|
||||||
|
- Kubernetes
|
||||||
|
- Talos Linux
|
||||||
|
- Prometheus
|
||||||
|
- Grafana
|
||||||
|
- Alertmanager
|
||||||
|
- Blackbox Exporter
|
||||||
|
- SNMP Exporter
|
||||||
|
- IPMI Exporter
|
||||||
|
- Proxmox VE
|
||||||
|
- MetalLB
|
||||||
|
- Ingress NGINX
|
||||||
|
- Cloudflare Tunnel
|
||||||
|
- Ansible
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# Repository Structure
|
||||||
|
|
||||||
|
## ansible/
|
||||||
|
|
||||||
|
Ansible playbooks and inventory files.
|
||||||
|
|
||||||
|
Examples:
|
||||||
|
- package installation
|
||||||
|
- node-exporter deployment
|
||||||
|
- server automation
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## k8s/
|
||||||
|
|
||||||
|
Kubernetes manifests and infrastructure configuration.
|
||||||
|
|
||||||
|
### cloudflare/
|
||||||
|
Cloudflare Tunnel manifests.
|
||||||
|
|
||||||
|
### homepage/
|
||||||
|
Homepage dashboard manifests.
|
||||||
|
|
||||||
|
### ingress-nginx/
|
||||||
|
Ingress resources for exposed services.
|
||||||
|
|
||||||
|
### longhorn/
|
||||||
|
Longhorn distributed storage configuration.
|
||||||
|
|
||||||
|
### metallb-system/
|
||||||
|
MetalLB load balancer configuration.
|
||||||
|
|
||||||
|
### monitoring/
|
||||||
|
Monitoring stack configuration.
|
||||||
|
|
||||||
|
Contains:
|
||||||
|
- Prometheus scrape configs
|
||||||
|
- exporter configs
|
||||||
|
- Prometheus alert rules
|
||||||
|
- Alertmanager Discord integration
|
||||||
|
- uptime-kuma config
|
||||||
|
|
||||||
|
### others/
|
||||||
|
Miscellaneous manifests.
|
||||||
|
|
||||||
|
### security/
|
||||||
|
Authentication and security-related configs.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## talos/
|
||||||
|
|
||||||
|
Talos Linux cluster configuration.
|
||||||
|
|
||||||
|
Contains:
|
||||||
|
- machine configs
|
||||||
|
- patches
|
||||||
|
- cluster bootstrap configs
|
||||||
|
- kubeconfig
|
||||||
|
- talosconfig
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# Monitoring Stack
|
||||||
|
|
||||||
|
The monitoring stack includes:
|
||||||
|
|
||||||
|
- Prometheus
|
||||||
|
- Grafana
|
||||||
|
- Alertmanager
|
||||||
|
- SNMP monitoring
|
||||||
|
- Blackbox ICMP monitoring
|
||||||
|
- IPMI monitoring
|
||||||
|
- Discord alerting
|
||||||
|
|
||||||
|
Alerting covers:
|
||||||
|
- host availability
|
||||||
|
- interface utilization
|
||||||
|
- packet loss
|
||||||
|
- scrape failures
|
||||||
|
- CPU/RAM pressure
|
||||||
|
- Kubernetes health
|
||||||
|
- Proxmox monitoring
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# Goals
|
||||||
|
|
||||||
|
Main goals of this homelab:
|
||||||
|
|
||||||
|
- infrastructure automation
|
||||||
|
- observability
|
||||||
|
- Kubernetes learning
|
||||||
|
- monitoring and alerting
|
||||||
|
- network engineering
|
||||||
|
- high availability experiments
|
||||||
|
- infrastructure testing
|
||||||
|
- self-hosted services
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# Notes
|
||||||
|
|
||||||
|
This repository intentionally excludes secrets and sensitive credentials using `.gitignore`.
|
||||||
|
|
||||||
|
Some manifests may require environment-specific modifications before deployment.
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
---
|
||||||
|
- name: Install monitoring packages
|
||||||
|
hosts: monitoring_targets
|
||||||
|
become: true
|
||||||
|
|
||||||
|
tasks:
|
||||||
|
- name: Install node exporter
|
||||||
|
ansible.builtin.apt:
|
||||||
|
name:
|
||||||
|
- prometheus-node-exporter
|
||||||
|
state: present
|
||||||
|
update_cache: true
|
||||||
|
|
||||||
|
- name: Enable and start node exporter
|
||||||
|
ansible.builtin.systemd:
|
||||||
|
name: prometheus-node-exporter
|
||||||
|
enabled: true
|
||||||
|
state: started
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
---
|
||||||
|
- name: Install common packages
|
||||||
|
hosts: debian machines
|
||||||
|
become: true
|
||||||
|
|
||||||
|
vars:
|
||||||
|
common_packages:
|
||||||
|
- qemu-guest-agent
|
||||||
|
- btop
|
||||||
|
- fastfetch
|
||||||
|
- curl
|
||||||
|
- wget
|
||||||
|
- nano
|
||||||
|
|
||||||
|
tasks:
|
||||||
|
- name: Install packages
|
||||||
|
apt:
|
||||||
|
name: "{{ common_packages }}"
|
||||||
|
state: present
|
||||||
|
update_cache: true
|
||||||
@@ -0,0 +1,45 @@
|
|||||||
|
[machines]
|
||||||
|
proxmox ansible_host=10.0.2.5
|
||||||
|
mc-servers ansible_host=10.0.3.2
|
||||||
|
#backup-nas ansible_host=10.0.2.12
|
||||||
|
|
||||||
|
[debian]
|
||||||
|
plexisklo ansible_host=10.0.1.2
|
||||||
|
synch ansible_host=10.0.1.3
|
||||||
|
graylog ansible_host=10.0.1.4
|
||||||
|
kozel-tajne ansible_host=10.0.1.6
|
||||||
|
kozel-web ansible_host=10.0.1.7
|
||||||
|
qbittorrent ansible_host=10.0.1.8
|
||||||
|
docker ansible_host=10.0.1.9
|
||||||
|
pihole ansible_host=10.0.1.10
|
||||||
|
gitea ansible_host=10.0.1.11 ansible_port=2030
|
||||||
|
mariadb ansible_host=10.0.1.12
|
||||||
|
zabbix ansible_host=10.0.1.20
|
||||||
|
|
||||||
|
[k8s]
|
||||||
|
k8s-cp-00 ansible_host=10.0.4.2
|
||||||
|
k8s-cp-01 ansible_host=10.0.4.3
|
||||||
|
k8s-cp-02 ansible_host=10.0.4.4
|
||||||
|
k8s-wrk-00 ansible_host=10.0.4.12
|
||||||
|
k8s-wrk-01 ansible_host=10.0.4.13
|
||||||
|
k8s-wrk-02 ansible_host=10.0.4.14
|
||||||
|
|
||||||
|
[all:vars]
|
||||||
|
ansible_user=root
|
||||||
|
ansible_port=22
|
||||||
|
ansible_python_interpreter=/usr/bin/python3
|
||||||
|
|
||||||
|
[monitoring_targets]
|
||||||
|
proxmox
|
||||||
|
mc-servers
|
||||||
|
plexisklo
|
||||||
|
synch
|
||||||
|
graylog
|
||||||
|
kozel-tajne
|
||||||
|
kozel-web
|
||||||
|
qbittorrent
|
||||||
|
docker
|
||||||
|
pihole
|
||||||
|
gitea
|
||||||
|
mariadb
|
||||||
|
zabbix
|
||||||
@@ -0,0 +1,30 @@
|
|||||||
|
apiVersion: monitoring.coreos.com/v1
|
||||||
|
kind: PrometheusRule
|
||||||
|
metadata:
|
||||||
|
name: blackbox-alerts
|
||||||
|
namespace: monitoring
|
||||||
|
labels:
|
||||||
|
release: kube-prometheus-stack
|
||||||
|
|
||||||
|
spec:
|
||||||
|
groups:
|
||||||
|
- name: blackbox
|
||||||
|
rules:
|
||||||
|
|
||||||
|
- alert: DeviceDown
|
||||||
|
expr: probe_success == 0
|
||||||
|
for: 1m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Device down"
|
||||||
|
description: "{{ $labels.instance }} is DOWN"
|
||||||
|
|
||||||
|
- alert: HighPing
|
||||||
|
expr: probe_duration_seconds > 0.5
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "High ping"
|
||||||
|
description: "{{ $labels.instance }} has high latency"
|
||||||
@@ -0,0 +1,331 @@
|
|||||||
|
apiVersion: monitoring.coreos.com/v1
|
||||||
|
kind: PrometheusRule
|
||||||
|
metadata:
|
||||||
|
name: ipmi-alerts
|
||||||
|
namespace: monitoring
|
||||||
|
labels:
|
||||||
|
release: kube-prometheus-stack
|
||||||
|
|
||||||
|
spec:
|
||||||
|
groups:
|
||||||
|
- name: ipmi-targets
|
||||||
|
rules:
|
||||||
|
- alert: IPMITargetDown
|
||||||
|
expr: up{job=~"ipmi.*|.*ipmi.*"} == 0
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "IPMI target down"
|
||||||
|
description: "{{ $labels.instance }} IPMI scrape target is down."
|
||||||
|
|
||||||
|
- alert: IPMIExporterCollectorDown
|
||||||
|
expr: ipmi_up{job=~"ipmi.*|.*ipmi.*"} == 0
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "IPMI collector down"
|
||||||
|
description: "{{ $labels.instance }} IPMI collector {{ $labels.collector }} is down."
|
||||||
|
|
||||||
|
- alert: IPMIScrapeSlow
|
||||||
|
expr: scrape_duration_seconds{job=~"ipmi.*|.*ipmi.*"} > 30
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "IPMI scrape slow"
|
||||||
|
description: "{{ $labels.instance }} IPMI scrape takes more than 30 seconds."
|
||||||
|
|
||||||
|
- alert: IPMIScrapeVerySlow
|
||||||
|
expr: scrape_duration_seconds{job=~"ipmi.*|.*ipmi.*"} > 90
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "IPMI scrape very slow"
|
||||||
|
description: "{{ $labels.instance }} IPMI scrape takes more than 90 seconds."
|
||||||
|
|
||||||
|
- alert: IPMILowSamples
|
||||||
|
expr: scrape_samples_scraped{job=~"ipmi.*|.*ipmi.*"} < 10
|
||||||
|
for: 10m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "IPMI low sample count"
|
||||||
|
description: "{{ $labels.instance }} IPMI scrape returns fewer than 10 samples."
|
||||||
|
|
||||||
|
- name: ipmi-chassis
|
||||||
|
rules:
|
||||||
|
- alert: IPMIChassisPowerOff
|
||||||
|
expr: ipmi_chassis_power_state{job=~"ipmi.*|.*ipmi.*"} == 0
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Server chassis power is off"
|
||||||
|
description: "{{ $labels.instance }} chassis power state is OFF."
|
||||||
|
|
||||||
|
- alert: IPMIChassisCoolingFault
|
||||||
|
expr: ipmi_chassis_cooling_fault_state{job=~"ipmi.*|.*ipmi.*"} == 1
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Chassis cooling fault"
|
||||||
|
description: "{{ $labels.instance }} reports chassis cooling fault."
|
||||||
|
|
||||||
|
- alert: IPMIChassisDriveFault
|
||||||
|
expr: ipmi_chassis_drive_fault_state{job=~"ipmi.*|.*ipmi.*"} == 1
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Chassis drive fault"
|
||||||
|
description: "{{ $labels.instance }} reports chassis drive fault."
|
||||||
|
|
||||||
|
- alert: IPMIChassisPowerControlFault
|
||||||
|
expr: ipmi_chassis_power_control_fault_state{job=~"ipmi.*|.*ipmi.*"} == 1
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Power control fault"
|
||||||
|
description: "{{ $labels.instance }} reports power control fault."
|
||||||
|
|
||||||
|
- name: ipmi-sensors
|
||||||
|
rules:
|
||||||
|
- alert: IPMISensorWarning
|
||||||
|
expr: ipmi_sensor_state{job=~"ipmi.*|.*ipmi.*"} == 1
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "IPMI sensor warning"
|
||||||
|
description: "{{ $labels.instance }} sensor {{ $labels.name }} reports WARNING state."
|
||||||
|
|
||||||
|
- alert: IPMISensorCritical
|
||||||
|
expr: ipmi_sensor_state{job=~"ipmi.*|.*ipmi.*"} >= 2
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "IPMI sensor critical"
|
||||||
|
description: "{{ $labels.instance }} sensor {{ $labels.name }} reports CRITICAL state."
|
||||||
|
|
||||||
|
- alert: IPMISensorMissingValue
|
||||||
|
expr: ipmi_sensor_value{job=~"ipmi.*|.*ipmi.*"} != ipmi_sensor_value{job=~"ipmi.*|.*ipmi.*"}
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "IPMI sensor value is NaN"
|
||||||
|
description: "{{ $labels.instance }} sensor {{ $labels.name }} returns NaN."
|
||||||
|
|
||||||
|
- name: ipmi-fans
|
||||||
|
rules:
|
||||||
|
- alert: IPMIFanStateWarning
|
||||||
|
expr: ipmi_fan_speed_state{job=~"ipmi.*|.*ipmi.*"} == 1
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Fan warning state"
|
||||||
|
description: "{{ $labels.instance }} fan {{ $labels.name }} reports WARNING state."
|
||||||
|
|
||||||
|
- alert: IPMIFanStateCritical
|
||||||
|
expr: ipmi_fan_speed_state{job=~"ipmi.*|.*ipmi.*"} >= 2
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Fan critical state"
|
||||||
|
description: "{{ $labels.instance }} fan {{ $labels.name }} reports CRITICAL state."
|
||||||
|
|
||||||
|
- alert: IPMIFanStopped
|
||||||
|
expr: ipmi_fan_speed_rpm{job=~"ipmi.*|.*ipmi.*"} == 0
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Fan stopped"
|
||||||
|
description: "{{ $labels.instance }} fan {{ $labels.name }} reports 0 RPM."
|
||||||
|
|
||||||
|
- alert: IPMIFanLowRPM
|
||||||
|
expr: ipmi_fan_speed_rpm{job=~"ipmi.*|.*ipmi.*"} > 0 and ipmi_fan_speed_rpm{job=~"ipmi.*|.*ipmi.*"} < 1000
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Fan low RPM"
|
||||||
|
description: "{{ $labels.instance }} fan {{ $labels.name }} is below 1000 RPM."
|
||||||
|
|
||||||
|
- alert: IPMIFanVeryHighRPM
|
||||||
|
expr: ipmi_fan_speed_rpm{job=~"ipmi.*|.*ipmi.*"} > 10000
|
||||||
|
for: 10m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Fan very high RPM"
|
||||||
|
description: "{{ $labels.instance }} fan {{ $labels.name }} is above 10000 RPM."
|
||||||
|
|
||||||
|
- name: ipmi-temperature
|
||||||
|
rules:
|
||||||
|
- alert: IPMITemperatureHigh
|
||||||
|
expr: ipmi_temperature_celsius{job=~"ipmi.*|.*ipmi.*"} > 70
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "High IPMI temperature"
|
||||||
|
description: "{{ $labels.instance }} sensor {{ $labels.name }} is above 70°C."
|
||||||
|
|
||||||
|
- alert: IPMITemperatureCritical
|
||||||
|
expr: ipmi_temperature_celsius{job=~"ipmi.*|.*ipmi.*"} > 85
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Critical IPMI temperature"
|
||||||
|
description: "{{ $labels.instance }} sensor {{ $labels.name }} is above 85°C."
|
||||||
|
|
||||||
|
- alert: IPMISensorTemperatureHigh
|
||||||
|
expr: ipmi_sensor_value{type=~"Temperature|temperature",job=~"ipmi.*|.*ipmi.*"} > 70
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "High sensor temperature"
|
||||||
|
description: "{{ $labels.instance }} sensor {{ $labels.name }} is above 70°C."
|
||||||
|
|
||||||
|
- alert: IPMISensorTemperatureCritical
|
||||||
|
expr: ipmi_sensor_value{type=~"Temperature|temperature",job=~"ipmi.*|.*ipmi.*"} > 85
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Critical sensor temperature"
|
||||||
|
description: "{{ $labels.instance }} sensor {{ $labels.name }} is above 85°C."
|
||||||
|
|
||||||
|
- name: ipmi-power-current-voltage
|
||||||
|
rules:
|
||||||
|
- alert: IPMIPowerHigh
|
||||||
|
expr: ipmi_power_watts{job=~"ipmi.*|.*ipmi.*"} > 600
|
||||||
|
for: 10m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "High power usage"
|
||||||
|
description: "{{ $labels.instance }} power usage is above 600 W."
|
||||||
|
|
||||||
|
- alert: IPMIPowerVeryHigh
|
||||||
|
expr: ipmi_power_watts{job=~"ipmi.*|.*ipmi.*"} > 800
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Very high power usage"
|
||||||
|
description: "{{ $labels.instance }} power usage is above 800 W."
|
||||||
|
|
||||||
|
- alert: IPMIPowerStateWarning
|
||||||
|
expr: ipmi_power_state{job=~"ipmi.*|.*ipmi.*"} == 1
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Power sensor warning"
|
||||||
|
description: "{{ $labels.instance }} power sensor {{ $labels.name }} reports WARNING state."
|
||||||
|
|
||||||
|
- alert: IPMIPowerStateCritical
|
||||||
|
expr: ipmi_power_state{job=~"ipmi.*|.*ipmi.*"} >= 2
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Power sensor critical"
|
||||||
|
description: "{{ $labels.instance }} power sensor {{ $labels.name }} reports CRITICAL state."
|
||||||
|
|
||||||
|
- alert: IPMICurrentStateWarning
|
||||||
|
expr: ipmi_current_state{job=~"ipmi.*|.*ipmi.*"} == 1
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Current sensor warning"
|
||||||
|
description: "{{ $labels.instance }} current sensor {{ $labels.name }} reports WARNING state."
|
||||||
|
|
||||||
|
- alert: IPMICurrentStateCritical
|
||||||
|
expr: ipmi_current_state{job=~"ipmi.*|.*ipmi.*"} >= 2
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Current sensor critical"
|
||||||
|
description: "{{ $labels.instance }} current sensor {{ $labels.name }} reports CRITICAL state."
|
||||||
|
|
||||||
|
- alert: IPMIVoltageStateWarning
|
||||||
|
expr: ipmi_voltage_state{job=~"ipmi.*|.*ipmi.*"} == 1
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Voltage sensor warning"
|
||||||
|
description: "{{ $labels.instance }} voltage sensor {{ $labels.name }} reports WARNING state."
|
||||||
|
|
||||||
|
- alert: IPMIVoltageStateCritical
|
||||||
|
expr: ipmi_voltage_state{job=~"ipmi.*|.*ipmi.*"} >= 2
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Voltage sensor critical"
|
||||||
|
description: "{{ $labels.instance }} voltage sensor {{ $labels.name }} reports CRITICAL state."
|
||||||
|
|
||||||
|
- name: ipmi-generic-thresholds
|
||||||
|
rules:
|
||||||
|
- alert: IPMISensorVoltageVeryLow
|
||||||
|
expr: ipmi_sensor_value{type=~"Voltage|voltage",job=~"ipmi.*|.*ipmi.*"} > 0 and ipmi_sensor_value{type=~"Voltage|voltage",job=~"ipmi.*|.*ipmi.*"} < 0.8
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Voltage sensor very low"
|
||||||
|
description: "{{ $labels.instance }} voltage sensor {{ $labels.name }} is very low."
|
||||||
|
|
||||||
|
- alert: IPMISensorVoltageVeryHigh
|
||||||
|
expr: ipmi_sensor_value{type=~"Voltage|voltage",job=~"ipmi.*|.*ipmi.*"} > 14
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Voltage sensor very high"
|
||||||
|
description: "{{ $labels.instance }} voltage sensor {{ $labels.name }} is very high."
|
||||||
|
|
||||||
|
- alert: IPMISensorCurrentHigh
|
||||||
|
expr: ipmi_current_amperes{job=~"ipmi.*|.*ipmi.*"} > 20
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "High current"
|
||||||
|
description: "{{ $labels.instance }} current sensor {{ $labels.name }} is above 20 A."
|
||||||
|
|
||||||
|
- name: ipmi-bmc-info
|
||||||
|
rules:
|
||||||
|
- alert: IPMIBMCInfoMissing
|
||||||
|
expr: absent(ipmi_bmc_info{job=~"ipmi.*|.*ipmi.*"})
|
||||||
|
for: 10m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "IPMI BMC info missing"
|
||||||
|
description: "No ipmi_bmc_info metrics are present."
|
||||||
|
|
||||||
|
- alert: IPMIBMCFirmwareUnknown
|
||||||
|
expr: ipmi_bmc_info{firmware_revision="",job=~"ipmi.*|.*ipmi.*"} == 1
|
||||||
|
for: 10m
|
||||||
|
labels:
|
||||||
|
severity: info
|
||||||
|
annotations:
|
||||||
|
summary: "BMC firmware revision unknown"
|
||||||
|
description: "{{ $labels.instance }} does not report BMC firmware revision."
|
||||||
@@ -0,0 +1,470 @@
|
|||||||
|
apiVersion: monitoring.coreos.com/v1
|
||||||
|
kind: PrometheusRule
|
||||||
|
metadata:
|
||||||
|
name: node-exporter-alerts
|
||||||
|
namespace: monitoring
|
||||||
|
labels:
|
||||||
|
release: kube-prometheus-stack
|
||||||
|
|
||||||
|
spec:
|
||||||
|
groups:
|
||||||
|
- name: node-exporter-availability
|
||||||
|
rules:
|
||||||
|
- alert: NodeDown
|
||||||
|
expr: up{job=~".*node.*|external-node-exporters"} == 0
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Node down"
|
||||||
|
description: "{{ $labels.instance }} is down."
|
||||||
|
|
||||||
|
- alert: NodeExporterMissing
|
||||||
|
expr: absent(up{job=~".*node.*|external-node-exporters"})
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Node exporter missing"
|
||||||
|
description: "No node-exporter targets are currently being scraped."
|
||||||
|
|
||||||
|
- name: node-exporter-cpu-load
|
||||||
|
rules:
|
||||||
|
- alert: NodeHighCPU
|
||||||
|
expr: 100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 85
|
||||||
|
for: 10m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "High CPU usage"
|
||||||
|
description: "{{ $labels.instance }} CPU usage is above 85% for 10 minutes."
|
||||||
|
|
||||||
|
- alert: NodeCriticalCPU
|
||||||
|
expr: 100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 95
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Critical CPU usage"
|
||||||
|
description: "{{ $labels.instance }} CPU usage is above 95%."
|
||||||
|
|
||||||
|
- alert: NodeHighLoad1
|
||||||
|
expr: node_load1 / count by(instance) (node_cpu_seconds_total{mode="idle"}) > 2
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "High 1m load"
|
||||||
|
description: "{{ $labels.instance }} has high 1-minute load."
|
||||||
|
|
||||||
|
- alert: NodeHighLoad5
|
||||||
|
expr: node_load5 / count by(instance) (node_cpu_seconds_total{mode="idle"}) > 1.75
|
||||||
|
for: 10m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "High 5m load"
|
||||||
|
description: "{{ $labels.instance }} has high 5-minute load."
|
||||||
|
|
||||||
|
- alert: NodeHighLoad15
|
||||||
|
expr: node_load15 / count by(instance) (node_cpu_seconds_total{mode="idle"}) > 1.5
|
||||||
|
for: 15m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "High 15m load"
|
||||||
|
description: "{{ $labels.instance }} has high 15-minute load."
|
||||||
|
|
||||||
|
- alert: NodeCriticalLoad15
|
||||||
|
expr: node_load15 / count by(instance) (node_cpu_seconds_total{mode="idle"}) > 3
|
||||||
|
for: 10m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Critical 15m load"
|
||||||
|
description: "{{ $labels.instance }} has critically high 15-minute load."
|
||||||
|
|
||||||
|
- name: node-exporter-memory-swap
|
||||||
|
rules:
|
||||||
|
- alert: NodeMemoryLow
|
||||||
|
expr: (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) < 0.10
|
||||||
|
for: 10m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Low available memory"
|
||||||
|
description: "{{ $labels.instance }} has less than 10% memory available."
|
||||||
|
|
||||||
|
- alert: NodeMemoryCritical
|
||||||
|
expr: (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) < 0.05
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Critical available memory"
|
||||||
|
description: "{{ $labels.instance }} has less than 5% memory available."
|
||||||
|
|
||||||
|
- alert: NodeSwapHigh
|
||||||
|
expr: (node_memory_SwapTotal_bytes > 0) and ((node_memory_SwapTotal_bytes - node_memory_SwapFree_bytes) / node_memory_SwapTotal_bytes > 0.50)
|
||||||
|
for: 10m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "High swap usage"
|
||||||
|
description: "{{ $labels.instance }} swap usage is above 50%."
|
||||||
|
|
||||||
|
- alert: NodeSwapCritical
|
||||||
|
expr: (node_memory_SwapTotal_bytes > 0) and ((node_memory_SwapTotal_bytes - node_memory_SwapFree_bytes) / node_memory_SwapTotal_bytes > 0.80)
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Critical swap usage"
|
||||||
|
description: "{{ $labels.instance }} swap usage is above 80%."
|
||||||
|
|
||||||
|
- alert: NodeOOMKills
|
||||||
|
expr: increase(node_vmstat_oom_kill[10m]) > 0
|
||||||
|
for: 1m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "OOM kill detected"
|
||||||
|
description: "{{ $labels.instance }} had OOM kill event in last 10 minutes."
|
||||||
|
|
||||||
|
- name: node-exporter-filesystems
|
||||||
|
rules:
|
||||||
|
- alert: NodeRootFilesystemLow
|
||||||
|
expr: (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs|nsfs|ramfs",mountpoint="/"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay|squashfs|nsfs|ramfs",mountpoint="/"}) < 0.15
|
||||||
|
for: 10m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Root filesystem low"
|
||||||
|
description: "{{ $labels.instance }} root filesystem has less than 15% free space."
|
||||||
|
|
||||||
|
- alert: NodeRootFilesystemCritical
|
||||||
|
expr: (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs|nsfs|ramfs",mountpoint="/"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay|squashfs|nsfs|ramfs",mountpoint="/"}) < 0.05
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Root filesystem critical"
|
||||||
|
description: "{{ $labels.instance }} root filesystem has less than 5% free space."
|
||||||
|
|
||||||
|
- alert: NodeFilesystemLow
|
||||||
|
expr: (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs|nsfs|ramfs"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay|squashfs|nsfs|ramfs"}) < 0.15
|
||||||
|
for: 15m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Filesystem low"
|
||||||
|
description: "{{ $labels.instance }} {{ $labels.mountpoint }} has less than 15% free space."
|
||||||
|
|
||||||
|
- alert: NodeFilesystemCritical
|
||||||
|
expr: (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs|nsfs|ramfs"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay|squashfs|nsfs|ramfs"}) < 0.05
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Filesystem critical"
|
||||||
|
description: "{{ $labels.instance }} {{ $labels.mountpoint }} has less than 5% free space."
|
||||||
|
|
||||||
|
- alert: NodeFilesystemWillFillSoon
|
||||||
|
expr: predict_linear(node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs|nsfs|ramfs"}[6h], 24 * 3600) < 0
|
||||||
|
for: 30m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Filesystem will fill soon"
|
||||||
|
description: "{{ $labels.instance }} {{ $labels.mountpoint }} is predicted to fill within 24 hours."
|
||||||
|
|
||||||
|
- alert: NodeFilesystemWillFillVerySoon
|
||||||
|
expr: predict_linear(node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs|nsfs|ramfs"}[2h], 6 * 3600) < 0
|
||||||
|
for: 15m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Filesystem will fill very soon"
|
||||||
|
description: "{{ $labels.instance }} {{ $labels.mountpoint }} is predicted to fill within 6 hours."
|
||||||
|
|
||||||
|
- alert: NodeInodesLow
|
||||||
|
expr: (node_filesystem_files_free{fstype!~"tmpfs|overlay|squashfs|nsfs|ramfs"} / node_filesystem_files{fstype!~"tmpfs|overlay|squashfs|nsfs|ramfs"}) < 0.10
|
||||||
|
for: 15m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Filesystem inodes low"
|
||||||
|
description: "{{ $labels.instance }} {{ $labels.mountpoint }} has less than 10% free inodes."
|
||||||
|
|
||||||
|
- alert: NodeInodesCritical
|
||||||
|
expr: (node_filesystem_files_free{fstype!~"tmpfs|overlay|squashfs|nsfs|ramfs"} / node_filesystem_files{fstype!~"tmpfs|overlay|squashfs|nsfs|ramfs"}) < 0.05
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Filesystem inodes critical"
|
||||||
|
description: "{{ $labels.instance }} {{ $labels.mountpoint }} has less than 5% free inodes."
|
||||||
|
|
||||||
|
- alert: NodeFilesystemReadOnly
|
||||||
|
expr: node_filesystem_readonly{fstype!~"tmpfs|overlay|squashfs|nsfs|ramfs"} == 1
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Filesystem read-only"
|
||||||
|
description: "{{ $labels.instance }} {{ $labels.mountpoint }} is read-only."
|
||||||
|
|
||||||
|
- name: node-exporter-disk-io
|
||||||
|
rules:
|
||||||
|
- alert: NodeDiskIOHigh
|
||||||
|
expr: rate(node_disk_io_time_seconds_total{device!~"loop.*|ram.*|dm-.*"}[5m]) > 0.90
|
||||||
|
for: 10m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "High disk I/O utilization"
|
||||||
|
description: "{{ $labels.instance }} disk {{ $labels.device }} I/O utilization is high."
|
||||||
|
|
||||||
|
- alert: NodeDiskReadLatencyHigh
|
||||||
|
expr: rate(node_disk_read_time_seconds_total{device!~"loop.*|ram.*"}[5m]) / rate(node_disk_reads_completed_total{device!~"loop.*|ram.*"}[5m]) > 0.1
|
||||||
|
for: 10m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "High disk read latency"
|
||||||
|
description: "{{ $labels.instance }} disk {{ $labels.device }} read latency is high."
|
||||||
|
|
||||||
|
- alert: NodeDiskWriteLatencyHigh
|
||||||
|
expr: rate(node_disk_write_time_seconds_total{device!~"loop.*|ram.*"}[5m]) / rate(node_disk_writes_completed_total{device!~"loop.*|ram.*"}[5m]) > 0.1
|
||||||
|
for: 10m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "High disk write latency"
|
||||||
|
description: "{{ $labels.instance }} disk {{ $labels.device }} write latency is high."
|
||||||
|
|
||||||
|
- alert: NodeDiskReadErrorsOrWeirdness
|
||||||
|
expr: increase(node_disk_reads_completed_total{device!~"loop.*|ram.*"}[10m]) == 0 and rate(node_disk_read_time_seconds_total{device!~"loop.*|ram.*"}[10m]) > 0
|
||||||
|
for: 10m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Possible disk read issue"
|
||||||
|
description: "{{ $labels.instance }} disk {{ $labels.device }} reports read time without completed reads."
|
||||||
|
|
||||||
|
- name: node-exporter-network
|
||||||
|
rules:
|
||||||
|
- alert: NodeNetworkReceiveErrors
|
||||||
|
expr: increase(node_network_receive_errs_total{device!~"lo|veth.*|docker.*|br-.*|cni.*|flannel.*|calico.*"}[10m]) > 10
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Network RX errors"
|
||||||
|
description: "{{ $labels.instance }} interface {{ $labels.device }} has RX errors."
|
||||||
|
|
||||||
|
- alert: NodeNetworkTransmitErrors
|
||||||
|
expr: increase(node_network_transmit_errs_total{device!~"lo|veth.*|docker.*|br-.*|cni.*|flannel.*|calico.*"}[10m]) > 10
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Network TX errors"
|
||||||
|
description: "{{ $labels.instance }} interface {{ $labels.device }} has TX errors."
|
||||||
|
|
||||||
|
- alert: NodeNetworkReceiveDrops
|
||||||
|
expr: increase(node_network_receive_drop_total{device!~"lo|veth.*|docker.*|br-.*|cni.*|flannel.*|calico.*"}[10m]) > 50
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Network RX drops"
|
||||||
|
description: "{{ $labels.instance }} interface {{ $labels.device }} has RX drops."
|
||||||
|
|
||||||
|
- alert: NodeNetworkTransmitDrops
|
||||||
|
expr: increase(node_network_transmit_drop_total{device!~"lo|veth.*|docker.*|br-.*|cni.*|flannel.*|calico.*"}[10m]) > 50
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Network TX drops"
|
||||||
|
description: "{{ $labels.instance }} interface {{ $labels.device }} has TX drops."
|
||||||
|
|
||||||
|
- alert: NodeNetworkInterfaceFlapping
|
||||||
|
expr: changes(node_network_up{device!~"lo|veth.*|docker.*|br-.*|cni.*|flannel.*|calico.*"}[10m]) > 2
|
||||||
|
for: 1m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Network interface flapping"
|
||||||
|
description: "{{ $labels.instance }} interface {{ $labels.device }} changed link state more than twice in 10 minutes."
|
||||||
|
|
||||||
|
- alert: NodeNetworkInterfaceDown
|
||||||
|
expr: node_network_up{device!~"lo|veth.*|docker.*|br-.*|cni.*|flannel.*|calico.*"} == 0
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Network interface down"
|
||||||
|
description: "{{ $labels.instance }} interface {{ $labels.device }} is down."
|
||||||
|
|
||||||
|
- name: node-exporter-system-health
|
||||||
|
rules:
|
||||||
|
- alert: NodeRebooted
|
||||||
|
expr: time() - node_boot_time_seconds < 600
|
||||||
|
for: 1m
|
||||||
|
labels:
|
||||||
|
severity: info
|
||||||
|
annotations:
|
||||||
|
summary: "Node recently rebooted"
|
||||||
|
description: "{{ $labels.instance }} rebooted less than 10 minutes ago."
|
||||||
|
|
||||||
|
- alert: NodeTimeDrift
|
||||||
|
expr: abs(node_timex_offset_seconds) > 0.1
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Node time drift"
|
||||||
|
description: "{{ $labels.instance }} has time drift greater than 100 ms."
|
||||||
|
|
||||||
|
- alert: NodeTimeDriftCritical
|
||||||
|
expr: abs(node_timex_offset_seconds) > 1
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Critical node time drift"
|
||||||
|
description: "{{ $labels.instance }} has time drift greater than 1 second."
|
||||||
|
|
||||||
|
- alert: NodeClockNotSynchronized
|
||||||
|
expr: node_timex_sync_status == 0
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Clock not synchronized"
|
||||||
|
description: "{{ $labels.instance }} clock is not synchronized."
|
||||||
|
|
||||||
|
- alert: NodeEntropyLow
|
||||||
|
expr: node_entropy_available_bits < 256
|
||||||
|
for: 10m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Low entropy"
|
||||||
|
description: "{{ $labels.instance }} has low available entropy."
|
||||||
|
|
||||||
|
- alert: NodeFileDescriptorsHigh
|
||||||
|
expr: node_filefd_allocated / node_filefd_maximum > 0.80
|
||||||
|
for: 10m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "High file descriptor usage"
|
||||||
|
description: "{{ $labels.instance }} file descriptor usage is above 80%."
|
||||||
|
|
||||||
|
- alert: NodeFileDescriptorsCritical
|
||||||
|
expr: node_filefd_allocated / node_filefd_maximum > 0.95
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Critical file descriptor usage"
|
||||||
|
description: "{{ $labels.instance }} file descriptor usage is above 95%."
|
||||||
|
|
||||||
|
- alert: NodeProcessesHigh
|
||||||
|
expr: node_procs_running > 300
|
||||||
|
for: 10m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "High running process count"
|
||||||
|
description: "{{ $labels.instance }} has more than 300 running processes."
|
||||||
|
|
||||||
|
- alert: NodeProcessesBlocked
|
||||||
|
expr: node_procs_blocked > 5
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Blocked processes"
|
||||||
|
description: "{{ $labels.instance }} has more than 5 blocked processes."
|
||||||
|
|
||||||
|
- name: node-exporter-systemd
|
||||||
|
rules:
|
||||||
|
- alert: NodeSystemdUnitFailed
|
||||||
|
expr: node_systemd_unit_state{state="failed"} == 1
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Systemd unit failed"
|
||||||
|
description: "{{ $labels.instance }} systemd unit {{ $labels.name }} is failed."
|
||||||
|
|
||||||
|
- alert: NodeSystemdServiceRestartingTooOften
|
||||||
|
expr: increase(node_systemd_service_restart_total[15m]) > 3
|
||||||
|
for: 1m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Systemd service restarting too often"
|
||||||
|
description: "{{ $labels.instance }} systemd service {{ $labels.name }} restarted more than 3 times in 15 minutes."
|
||||||
|
|
||||||
|
- name: node-exporter-hardware
|
||||||
|
rules:
|
||||||
|
- alert: NodeTemperatureHigh
|
||||||
|
expr: node_hwmon_temp_celsius > 80
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "High hardware temperature"
|
||||||
|
description: "{{ $labels.instance }} sensor {{ $labels.chip }} {{ $labels.sensor }} is above 80°C."
|
||||||
|
|
||||||
|
- alert: NodeTemperatureCritical
|
||||||
|
expr: node_hwmon_temp_celsius > 90
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Critical hardware temperature"
|
||||||
|
description: "{{ $labels.instance }} sensor {{ $labels.chip }} {{ $labels.sensor }} is above 90°C."
|
||||||
|
|
||||||
|
- alert: NodeThermalZoneHigh
|
||||||
|
expr: node_thermal_zone_temp > 80000
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "High thermal zone temperature"
|
||||||
|
description: "{{ $labels.instance }} thermal zone {{ $labels.zone }} is above 80°C."
|
||||||
|
|
||||||
|
- alert: NodeThermalZoneCritical
|
||||||
|
expr: node_thermal_zone_temp > 90000
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Critical thermal zone temperature"
|
||||||
|
description: "{{ $labels.instance }} thermal zone {{ $labels.zone }} is above 90°C."
|
||||||
|
|
||||||
|
- name: node-exporter-prometheus-scrape
|
||||||
|
rules:
|
||||||
|
- alert: NodeExporterScrapeSlow
|
||||||
|
expr: scrape_duration_seconds{job=~".*node.*|external-node-exporters"} > 10
|
||||||
|
for: 10m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Node exporter scrape slow"
|
||||||
|
description: "Scrape of {{ $labels.instance }} takes more than 10 seconds."
|
||||||
|
|
||||||
|
- alert: NodeExporterLowSamples
|
||||||
|
expr: scrape_samples_scraped{job=~".*node.*|external-node-exporters"} < 100
|
||||||
|
for: 10m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Node exporter low sample count"
|
||||||
|
description: "{{ $labels.instance }} returns fewer than 100 samples."
|
||||||
@@ -0,0 +1,423 @@
|
|||||||
|
apiVersion: monitoring.coreos.com/v1
|
||||||
|
kind: PrometheusRule
|
||||||
|
metadata:
|
||||||
|
name: proxmox-alerts
|
||||||
|
namespace: monitoring
|
||||||
|
labels:
|
||||||
|
release: kube-prometheus-stack
|
||||||
|
|
||||||
|
spec:
|
||||||
|
groups:
|
||||||
|
- name: proxmox-targets
|
||||||
|
rules:
|
||||||
|
- alert: ProxmoxTargetDown
|
||||||
|
expr: up{job=~"proxmox.*|.*pve.*"} == 0
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Proxmox exporter target down"
|
||||||
|
description: "{{ $labels.instance }} Proxmox/PVE scrape is down."
|
||||||
|
|
||||||
|
- alert: ProxmoxScrapeSlow
|
||||||
|
expr: scrape_duration_seconds{job=~"proxmox.*|.*pve.*"} > 20
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Proxmox scrape slow"
|
||||||
|
description: "{{ $labels.instance }} Proxmox scrape takes more than 20 seconds."
|
||||||
|
|
||||||
|
- alert: ProxmoxScrapeVerySlow
|
||||||
|
expr: scrape_duration_seconds{job=~"proxmox.*|.*pve.*"} > 60
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Proxmox scrape very slow"
|
||||||
|
description: "{{ $labels.instance }} Proxmox scrape takes more than 60 seconds."
|
||||||
|
|
||||||
|
- alert: ProxmoxLowSamples
|
||||||
|
expr: scrape_samples_scraped{job=~"proxmox.*|.*pve.*"} < 20
|
||||||
|
for: 10m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Proxmox low sample count"
|
||||||
|
description: "{{ $labels.instance }} Proxmox scrape returns fewer than 20 samples."
|
||||||
|
|
||||||
|
- name: proxmox-nodes
|
||||||
|
rules:
|
||||||
|
- alert: ProxmoxNodeDown
|
||||||
|
expr: pve_node_status == 0
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Proxmox node down"
|
||||||
|
description: "Proxmox node {{ $labels.node }} is down."
|
||||||
|
|
||||||
|
- alert: ProxmoxNodeHighCPU
|
||||||
|
expr: pve_node_cpu_usage_ratio > 0.85
|
||||||
|
for: 10m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Proxmox node high CPU"
|
||||||
|
description: "Node {{ $labels.node }} CPU usage is above 85%."
|
||||||
|
|
||||||
|
- alert: ProxmoxNodeCriticalCPU
|
||||||
|
expr: pve_node_cpu_usage_ratio > 0.95
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Proxmox node critical CPU"
|
||||||
|
description: "Node {{ $labels.node }} CPU usage is above 95%."
|
||||||
|
|
||||||
|
- alert: ProxmoxNodeHighMemory
|
||||||
|
expr: pve_node_memory_usage_bytes / pve_node_memory_size_bytes > 0.90
|
||||||
|
for: 10m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Proxmox node high memory"
|
||||||
|
description: "Node {{ $labels.node }} memory usage is above 90%."
|
||||||
|
|
||||||
|
- alert: ProxmoxNodeCriticalMemory
|
||||||
|
expr: pve_node_memory_usage_bytes / pve_node_memory_size_bytes > 0.97
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Proxmox node critical memory"
|
||||||
|
description: "Node {{ $labels.node }} memory usage is above 97%."
|
||||||
|
|
||||||
|
- alert: ProxmoxNodeLowFreeMemory
|
||||||
|
expr: pve_node_memory_free_bytes / pve_node_memory_size_bytes < 0.10
|
||||||
|
for: 10m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Proxmox node low free memory"
|
||||||
|
description: "Node {{ $labels.node }} has less than 10% free memory."
|
||||||
|
|
||||||
|
- alert: ProxmoxNodeVeryLowFreeMemory
|
||||||
|
expr: pve_node_memory_free_bytes / pve_node_memory_size_bytes < 0.05
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Proxmox node very low free memory"
|
||||||
|
description: "Node {{ $labels.node }} has less than 5% free memory."
|
||||||
|
|
||||||
|
- alert: ProxmoxNodeHighLoad
|
||||||
|
expr: pve_node_loadavg_15 / pve_node_cpu_count > 1.5
|
||||||
|
for: 15m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Proxmox node high load"
|
||||||
|
description: "Node {{ $labels.node }} has high 15m load."
|
||||||
|
|
||||||
|
- alert: ProxmoxNodeCriticalLoad
|
||||||
|
expr: pve_node_loadavg_15 / pve_node_cpu_count > 3
|
||||||
|
for: 10m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Proxmox node critical load"
|
||||||
|
description: "Node {{ $labels.node }} has critically high 15m load."
|
||||||
|
|
||||||
|
- alert: ProxmoxNodeUptimeTooLow
|
||||||
|
expr: pve_node_uptime_seconds < 600
|
||||||
|
for: 1m
|
||||||
|
labels:
|
||||||
|
severity: info
|
||||||
|
annotations:
|
||||||
|
summary: "Proxmox node recently rebooted"
|
||||||
|
description: "Node {{ $labels.node }} uptime is less than 10 minutes."
|
||||||
|
|
||||||
|
- name: proxmox-guests
|
||||||
|
rules:
|
||||||
|
- alert: ProxmoxGuestDown
|
||||||
|
expr: pve_guest_status == 0
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Proxmox guest down"
|
||||||
|
description: "Guest {{ $labels.id }} {{ $labels.name }} on node {{ $labels.node }} is down."
|
||||||
|
|
||||||
|
- alert: ProxmoxGuestHighCPU
|
||||||
|
expr: pve_guest_cpu_usage_ratio > 0.90
|
||||||
|
for: 15m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Proxmox guest high CPU"
|
||||||
|
description: "Guest {{ $labels.id }} {{ $labels.name }} CPU usage is above 90%."
|
||||||
|
|
||||||
|
- alert: ProxmoxGuestCriticalCPU
|
||||||
|
expr: pve_guest_cpu_usage_ratio > 0.98
|
||||||
|
for: 10m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Proxmox guest critical CPU"
|
||||||
|
description: "Guest {{ $labels.id }} {{ $labels.name }} CPU usage is above 98%."
|
||||||
|
|
||||||
|
- alert: ProxmoxGuestHighMemory
|
||||||
|
expr: pve_guest_memory_usage_bytes / pve_guest_memory_size_bytes > 0.90
|
||||||
|
for: 15m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Proxmox guest high memory"
|
||||||
|
description: "Guest {{ $labels.id }} {{ $labels.name }} memory usage is above 90%."
|
||||||
|
|
||||||
|
- alert: ProxmoxGuestCriticalMemory
|
||||||
|
expr: pve_guest_memory_usage_bytes / pve_guest_memory_size_bytes > 0.97
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Proxmox guest critical memory"
|
||||||
|
description: "Guest {{ $labels.id }} {{ $labels.name }} memory usage is above 97%."
|
||||||
|
|
||||||
|
- alert: ProxmoxGuestLowFreeMemory
|
||||||
|
expr: pve_guest_memory_free_bytes / pve_guest_memory_size_bytes < 0.10
|
||||||
|
for: 15m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Proxmox guest low free memory"
|
||||||
|
description: "Guest {{ $labels.id }} {{ $labels.name }} has less than 10% free memory."
|
||||||
|
|
||||||
|
- alert: ProxmoxGuestVeryLowFreeMemory
|
||||||
|
expr: pve_guest_memory_free_bytes / pve_guest_memory_size_bytes < 0.05
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Proxmox guest very low free memory"
|
||||||
|
description: "Guest {{ $labels.id }} {{ $labels.name }} has less than 5% free memory."
|
||||||
|
|
||||||
|
- alert: ProxmoxGuestHighDiskUsage
|
||||||
|
expr: pve_guest_disk_usage_bytes / pve_guest_disk_size_bytes > 0.85
|
||||||
|
for: 15m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Proxmox guest disk high usage"
|
||||||
|
description: "Guest {{ $labels.id }} {{ $labels.name }} disk usage is above 85%."
|
||||||
|
|
||||||
|
- alert: ProxmoxGuestCriticalDiskUsage
|
||||||
|
expr: pve_guest_disk_usage_bytes / pve_guest_disk_size_bytes > 0.95
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Proxmox guest disk critical usage"
|
||||||
|
description: "Guest {{ $labels.id }} {{ $labels.name }} disk usage is above 95%."
|
||||||
|
|
||||||
|
- alert: ProxmoxGuestUnexpectedlyRunning
|
||||||
|
expr: pve_guest_status == 1 and on(id) pve_guest_template == 1
|
||||||
|
for: 10m
|
||||||
|
labels:
|
||||||
|
severity: info
|
||||||
|
annotations:
|
||||||
|
summary: "Proxmox template seems running"
|
||||||
|
description: "Template guest {{ $labels.id }} {{ $labels.name }} appears to be running."
|
||||||
|
|
||||||
|
- alert: ProxmoxGuestRecentlyRebooted
|
||||||
|
expr: pve_guest_uptime_seconds > 0 and pve_guest_uptime_seconds < 600
|
||||||
|
for: 1m
|
||||||
|
labels:
|
||||||
|
severity: info
|
||||||
|
annotations:
|
||||||
|
summary: "Proxmox guest recently rebooted"
|
||||||
|
description: "Guest {{ $labels.id }} {{ $labels.name }} uptime is less than 10 minutes."
|
||||||
|
|
||||||
|
- name: proxmox-storage
|
||||||
|
rules:
|
||||||
|
- alert: ProxmoxStorageHighUsage
|
||||||
|
expr: pve_storage_usage_bytes / pve_storage_size_bytes > 0.85
|
||||||
|
for: 10m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Proxmox storage high usage"
|
||||||
|
description: "Storage {{ $labels.storage }} on node {{ $labels.node }} is above 85%."
|
||||||
|
|
||||||
|
- alert: ProxmoxStorageCriticalUsage
|
||||||
|
expr: pve_storage_usage_bytes / pve_storage_size_bytes > 0.95
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Proxmox storage critical usage"
|
||||||
|
description: "Storage {{ $labels.storage }} on node {{ $labels.node }} is above 95%."
|
||||||
|
|
||||||
|
- alert: ProxmoxStorageLowFree
|
||||||
|
expr: pve_storage_free_bytes / pve_storage_size_bytes < 0.15
|
||||||
|
for: 10m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Proxmox storage low free"
|
||||||
|
description: "Storage {{ $labels.storage }} on node {{ $labels.node }} has less than 15% free space."
|
||||||
|
|
||||||
|
- alert: ProxmoxStorageVeryLowFree
|
||||||
|
expr: pve_storage_free_bytes / pve_storage_size_bytes < 0.05
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Proxmox storage very low free"
|
||||||
|
description: "Storage {{ $labels.storage }} on node {{ $labels.node }} has less than 5% free space."
|
||||||
|
|
||||||
|
- alert: ProxmoxStorageDisabled
|
||||||
|
expr: pve_storage_status == 0
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Proxmox storage disabled/unavailable"
|
||||||
|
description: "Storage {{ $labels.storage }} on node {{ $labels.node }} is disabled or unavailable."
|
||||||
|
|
||||||
|
- name: proxmox-cluster
|
||||||
|
rules:
|
||||||
|
- alert: ProxmoxClusterQuorumLost
|
||||||
|
expr: pve_cluster_quorate == 0
|
||||||
|
for: 1m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Proxmox cluster quorum lost"
|
||||||
|
description: "Proxmox cluster has lost quorum."
|
||||||
|
|
||||||
|
- alert: ProxmoxClusterNodeOffline
|
||||||
|
expr: pve_cluster_node_online == 0
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Proxmox cluster node offline"
|
||||||
|
description: "Cluster node {{ $labels.node }} is offline."
|
||||||
|
|
||||||
|
- alert: ProxmoxClusterExpectedVotesLow
|
||||||
|
expr: pve_cluster_expected_votes < 2
|
||||||
|
for: 10m
|
||||||
|
labels:
|
||||||
|
severity: info
|
||||||
|
annotations:
|
||||||
|
summary: "Proxmox cluster expected votes low"
|
||||||
|
description: "Proxmox cluster expected votes is below 2."
|
||||||
|
|
||||||
|
- name: proxmox-backups
|
||||||
|
rules:
|
||||||
|
- alert: ProxmoxBackupFailed
|
||||||
|
expr: pve_guest_backup_last_status == 0
|
||||||
|
for: 10m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Proxmox backup failed"
|
||||||
|
description: "Backup for guest {{ $labels.id }} {{ $labels.name }} failed."
|
||||||
|
|
||||||
|
- alert: ProxmoxBackupTooOld
|
||||||
|
expr: time() - pve_guest_backup_last_timestamp_seconds > 7 * 24 * 3600
|
||||||
|
for: 1h
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Proxmox backup too old"
|
||||||
|
description: "Backup for guest {{ $labels.id }} {{ $labels.name }} is older than 7 days."
|
||||||
|
|
||||||
|
- alert: ProxmoxNoRecentBackup
|
||||||
|
expr: absent(pve_guest_backup_last_timestamp_seconds)
|
||||||
|
for: 1h
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "No Proxmox backup metric found"
|
||||||
|
description: "No Proxmox backup timestamp metrics are present."
|
||||||
|
|
||||||
|
- name: proxmox-replication-ha
|
||||||
|
rules:
|
||||||
|
- alert: ProxmoxReplicationFailed
|
||||||
|
expr: pve_replication_status == 0
|
||||||
|
for: 10m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Proxmox replication failed"
|
||||||
|
description: "Replication job {{ $labels.job }} on node {{ $labels.node }} failed."
|
||||||
|
|
||||||
|
- alert: ProxmoxReplicationTooOld
|
||||||
|
expr: time() - pve_replication_last_sync_timestamp_seconds > 6 * 3600
|
||||||
|
for: 30m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Proxmox replication too old"
|
||||||
|
description: "Replication job {{ $labels.job }} last sync is older than 6 hours."
|
||||||
|
|
||||||
|
- alert: ProxmoxHAServiceError
|
||||||
|
expr: pve_ha_service_status == 0
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Proxmox HA service error"
|
||||||
|
description: "HA service {{ $labels.sid }} is not healthy."
|
||||||
|
|
||||||
|
- name: proxmox-tasks
|
||||||
|
rules:
|
||||||
|
- alert: ProxmoxTaskFailed
|
||||||
|
expr: increase(pve_tasks_failed_total[30m]) > 0
|
||||||
|
for: 1m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Proxmox task failed"
|
||||||
|
description: "A Proxmox task failed on {{ $labels.instance }}."
|
||||||
|
|
||||||
|
- alert: ProxmoxTasksFailingOften
|
||||||
|
expr: increase(pve_tasks_failed_total[6h]) > 5
|
||||||
|
for: 10m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Proxmox tasks failing often"
|
||||||
|
description: "More than 5 Proxmox tasks failed in 6 hours on {{ $labels.instance }}."
|
||||||
|
|
||||||
|
- name: proxmox-exporter-health
|
||||||
|
rules:
|
||||||
|
- alert: ProxmoxExporterNoNodeMetrics
|
||||||
|
expr: absent(pve_node_status)
|
||||||
|
for: 10m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "No Proxmox node metrics"
|
||||||
|
description: "PVE exporter is not returning node metrics."
|
||||||
|
|
||||||
|
- alert: ProxmoxExporterNoGuestMetrics
|
||||||
|
expr: absent(pve_guest_status)
|
||||||
|
for: 10m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "No Proxmox guest metrics"
|
||||||
|
description: "PVE exporter is not returning guest metrics."
|
||||||
|
|
||||||
|
- alert: ProxmoxExporterNoStorageMetrics
|
||||||
|
expr: absent(pve_storage_size_bytes)
|
||||||
|
for: 10m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "No Proxmox storage metrics"
|
||||||
|
description: "PVE exporter is not returning storage metrics."
|
||||||
@@ -0,0 +1,211 @@
|
|||||||
|
apiVersion: monitoring.coreos.com/v1
|
||||||
|
kind: PrometheusRule
|
||||||
|
metadata:
|
||||||
|
name: snmp-alerts
|
||||||
|
namespace: monitoring
|
||||||
|
labels:
|
||||||
|
release: kube-prometheus-stack
|
||||||
|
|
||||||
|
spec:
|
||||||
|
groups:
|
||||||
|
- name: snmp-targets
|
||||||
|
rules:
|
||||||
|
- alert: SNMPTargetDown
|
||||||
|
expr: up{job=~"snmp.*"} == 0
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "SNMP target down"
|
||||||
|
description: "SNMP target {{ $labels.instance }} in job {{ $labels.job }} is down."
|
||||||
|
|
||||||
|
- alert: SNMPScrapeSlow
|
||||||
|
expr: scrape_duration_seconds{job=~"snmp.*"} > 20
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "SNMP scrape is slow"
|
||||||
|
description: "SNMP scrape for {{ $labels.instance }} in job {{ $labels.job }} takes more than 20 seconds."
|
||||||
|
|
||||||
|
- alert: SNMPScrapeVerySlow
|
||||||
|
expr: scrape_duration_seconds{job=~"snmp.*"} > 60
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "SNMP scrape is very slow"
|
||||||
|
description: "SNMP scrape for {{ $labels.instance }} in job {{ $labels.job }} takes more than 60 seconds."
|
||||||
|
|
||||||
|
- alert: SNMPLowSamples
|
||||||
|
expr: scrape_samples_scraped{job=~"snmp.*"} < 5
|
||||||
|
for: 10m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "SNMP scrape returns very few samples"
|
||||||
|
description: "SNMP target {{ $labels.instance }} in job {{ $labels.job }} returns fewer than 5 samples."
|
||||||
|
|
||||||
|
- name: snmp-interfaces
|
||||||
|
rules:
|
||||||
|
- alert: InterfaceAdminUpButDown
|
||||||
|
expr: ifAdminStatus{job=~"snmp.*"} == 1 and ifOperStatus{job=~"snmp.*"} == 2
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Interface admin up but operationally down"
|
||||||
|
description: "{{ $labels.instance }} {{ $labels.ifName }} is enabled but operationally down."
|
||||||
|
|
||||||
|
- alert: InterfaceDown
|
||||||
|
expr: ifOperStatus{job=~"snmp.*"} == 2
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Interface down"
|
||||||
|
description: "{{ $labels.instance }} {{ $labels.ifName }} is down."
|
||||||
|
|
||||||
|
- alert: InterfaceHighRXUtilization
|
||||||
|
expr: ((rate(ifHCInOctets{job=~"snmp.*"}[5m]) * 8) / (ifHighSpeed{job=~"snmp.*"} * 1000000)) > 0.80
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "High RX interface utilization"
|
||||||
|
description: "{{ $labels.instance }} {{ $labels.ifName }} RX utilization is above 80%."
|
||||||
|
|
||||||
|
- alert: InterfaceHighTXUtilization
|
||||||
|
expr: ((rate(ifHCOutOctets{job=~"snmp.*"}[5m]) * 8) / (ifHighSpeed{job=~"snmp.*"} * 1000000)) > 0.80
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "High TX interface utilization"
|
||||||
|
description: "{{ $labels.instance }} {{ $labels.ifName }} TX utilization is above 80%."
|
||||||
|
|
||||||
|
- alert: InterfaceCriticalRXUtilization
|
||||||
|
expr: ((rate(ifHCInOctets{job=~"snmp.*"}[5m]) * 8) / (ifHighSpeed{job=~"snmp.*"} * 1000000)) > 0.95
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Critical RX interface utilization"
|
||||||
|
description: "{{ $labels.instance }} {{ $labels.ifName }} RX utilization is above 95%."
|
||||||
|
|
||||||
|
- alert: InterfaceCriticalTXUtilization
|
||||||
|
expr: ((rate(ifHCOutOctets{job=~"snmp.*"}[5m]) * 8) / (ifHighSpeed{job=~"snmp.*"} * 1000000)) > 0.95
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Critical TX interface utilization"
|
||||||
|
description: "{{ $labels.instance }} {{ $labels.ifName }} TX utilization is above 95%."
|
||||||
|
|
||||||
|
- alert: InterfaceHighRXFixedThreshold
|
||||||
|
expr: (rate(ifHCInOctets{job=~"snmp.*"}[5m]) * 8) > 800000000
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "High RX traffic"
|
||||||
|
description: "{{ $labels.instance }} {{ $labels.ifName }} RX traffic is above 800 Mbit/s."
|
||||||
|
|
||||||
|
- alert: InterfaceHighTXFixedThreshold
|
||||||
|
expr: (rate(ifHCOutOctets{job=~"snmp.*"}[5m]) * 8) > 800000000
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "High TX traffic"
|
||||||
|
description: "{{ $labels.instance }} {{ $labels.ifName }} TX traffic is above 800 Mbit/s."
|
||||||
|
|
||||||
|
- alert: InterfaceRXErrors
|
||||||
|
expr: increase(ifInErrors{job=~"snmp.*"}[10m]) > 10
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Interface RX errors"
|
||||||
|
description: "{{ $labels.instance }} {{ $labels.ifName }} has more than 10 RX errors in 10 minutes."
|
||||||
|
|
||||||
|
- alert: InterfaceTXErrors
|
||||||
|
expr: increase(ifOutErrors{job=~"snmp.*"}[10m]) > 10
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Interface TX errors"
|
||||||
|
description: "{{ $labels.instance }} {{ $labels.ifName }} has more than 10 TX errors in 10 minutes."
|
||||||
|
|
||||||
|
- alert: InterfaceRXDiscards
|
||||||
|
expr: increase(ifInDiscards{job=~"snmp.*"}[10m]) > 50
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Interface RX discards"
|
||||||
|
description: "{{ $labels.instance }} {{ $labels.ifName }} has more than 50 RX discards in 10 minutes."
|
||||||
|
|
||||||
|
- alert: InterfaceTXDiscards
|
||||||
|
expr: increase(ifOutDiscards{job=~"snmp.*"}[10m]) > 50
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Interface TX discards"
|
||||||
|
description: "{{ $labels.instance }} {{ $labels.ifName }} has more than 50 TX discards in 10 minutes."
|
||||||
|
|
||||||
|
- alert: InterfaceBroadcastStorm
|
||||||
|
expr: increase(ifHCInBroadcastPkts{job=~"snmp.*"}[5m]) > 10000
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Possible broadcast storm"
|
||||||
|
description: "{{ $labels.instance }} {{ $labels.ifName }} received more than 10000 broadcast packets in 5 minutes."
|
||||||
|
|
||||||
|
- alert: InterfaceMulticastStorm
|
||||||
|
expr: increase(ifHCInMulticastPkts{job=~"snmp.*"}[5m]) > 50000
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Possible multicast storm"
|
||||||
|
description: "{{ $labels.instance }} {{ $labels.ifName }} received more than 50000 multicast packets in 5 minutes."
|
||||||
|
|
||||||
|
- alert: InterfaceNoTraffic
|
||||||
|
expr: rate(ifHCInOctets{job=~"snmp.*"}[30m]) == 0 and rate(ifHCOutOctets{job=~"snmp.*"}[30m]) == 0 and ifOperStatus{job=~"snmp.*"} == 1
|
||||||
|
for: 30m
|
||||||
|
labels:
|
||||||
|
severity: info
|
||||||
|
annotations:
|
||||||
|
summary: "Interface has no traffic"
|
||||||
|
description: "{{ $labels.instance }} {{ $labels.ifName }} is up but has no RX/TX traffic for 30 minutes."
|
||||||
|
|
||||||
|
- alert: InterfaceCounterReset
|
||||||
|
expr: resets(ifHCInOctets{job=~"snmp.*"}[15m]) > 0 or resets(ifHCOutOctets{job=~"snmp.*"}[15m]) > 0
|
||||||
|
for: 1m
|
||||||
|
labels:
|
||||||
|
severity: info
|
||||||
|
annotations:
|
||||||
|
summary: "Interface counter reset"
|
||||||
|
description: "{{ $labels.instance }} {{ $labels.ifName }} interface counter reset detected."
|
||||||
|
|
||||||
|
- alert: InterfaceSpeedUnknown
|
||||||
|
expr: ifHighSpeed{job=~"snmp.*"} == 0
|
||||||
|
for: 10m
|
||||||
|
labels:
|
||||||
|
severity: info
|
||||||
|
annotations:
|
||||||
|
summary: "Interface speed unknown"
|
||||||
|
description: "{{ $labels.instance }} {{ $labels.ifName }} reports unknown interface speed."
|
||||||
|
|
||||||
|
- alert: InterfacePossibleFlapping
|
||||||
|
expr: changes(ifOperStatus{job=~"snmp.*"}[10m]) > 2
|
||||||
|
for: 1m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Possible interface flapping"
|
||||||
|
description: "{{ $labels.instance }} {{ $labels.ifName }} changed operational state more than twice in 10 minutes."
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
config:
|
||||||
|
modules:
|
||||||
|
redfish:
|
||||||
|
prober: http
|
||||||
|
timeout: 10s
|
||||||
|
http:
|
||||||
|
method: GET
|
||||||
|
|
||||||
|
tls_config:
|
||||||
|
insecure_skip_verify: true
|
||||||
|
|
||||||
|
preferred_ip_protocol: ip4
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
config:
|
||||||
|
modules:
|
||||||
|
icmp:
|
||||||
|
prober: icmp
|
||||||
|
timeout: 5s
|
||||||
|
icmp:
|
||||||
|
preferred_ip_protocol: ip4
|
||||||
|
|
||||||
|
http_2xx:
|
||||||
|
prober: http
|
||||||
|
timeout: 5s
|
||||||
|
http:
|
||||||
|
preferred_ip_protocol: ip4
|
||||||
@@ -0,0 +1,36 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: pve-exporter
|
||||||
|
namespace: monitoring
|
||||||
|
spec:
|
||||||
|
replicas: 1
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: pve-exporter
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: pve-exporter
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: pve-exporter
|
||||||
|
image: prompve/prometheus-pve-exporter:latest
|
||||||
|
ports:
|
||||||
|
- containerPort: 9221
|
||||||
|
envFrom:
|
||||||
|
- secretRef:
|
||||||
|
name: pve-exporter-secret
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: pve-exporter
|
||||||
|
namespace: monitoring
|
||||||
|
spec:
|
||||||
|
selector:
|
||||||
|
app: pve-exporter
|
||||||
|
ports:
|
||||||
|
- name: http
|
||||||
|
port: 9221
|
||||||
|
targetPort: 9221
|
||||||
@@ -0,0 +1,44 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: snmp-exporter-manual
|
||||||
|
namespace: monitoring
|
||||||
|
spec:
|
||||||
|
replicas: 1
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: snmp-exporter-manual
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: snmp-exporter-manual
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: snmp-exporter
|
||||||
|
image: quay.io/prometheus/snmp-exporter:v0.30.1
|
||||||
|
args:
|
||||||
|
- --config.file=/etc/snmp_exporter/snmp.yml
|
||||||
|
ports:
|
||||||
|
- name: http
|
||||||
|
containerPort: 9116
|
||||||
|
volumeMounts:
|
||||||
|
- name: snmp-config
|
||||||
|
mountPath: /etc/snmp_exporter/snmp.yml
|
||||||
|
subPath: snmp.yml
|
||||||
|
volumes:
|
||||||
|
- name: snmp-config
|
||||||
|
configMap:
|
||||||
|
name: snmp-exporter-config
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: snmp-exporter-manual
|
||||||
|
namespace: monitoring
|
||||||
|
spec:
|
||||||
|
selector:
|
||||||
|
app: snmp-exporter-manual
|
||||||
|
ports:
|
||||||
|
- name: http
|
||||||
|
port: 9116
|
||||||
|
targetPort: 9116
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,25 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: ConfigMap
|
||||||
|
metadata:
|
||||||
|
name: external-node-exporters
|
||||||
|
namespace: monitoring
|
||||||
|
|
||||||
|
data:
|
||||||
|
prometheus-additional.yaml: |
|
||||||
|
- job_name: 'external-node-exporters'
|
||||||
|
|
||||||
|
static_configs:
|
||||||
|
- targets:
|
||||||
|
- 10.0.1.2:9100
|
||||||
|
- 10.0.1.3:9100
|
||||||
|
- 10.0.1.4:9100
|
||||||
|
- 10.0.1.6:9100
|
||||||
|
- 10.0.1.7:9100
|
||||||
|
- 10.0.1.8:9100
|
||||||
|
- 10.0.1.9:9100
|
||||||
|
- 10.0.1.10:9100
|
||||||
|
- 10.0.1.11:9100
|
||||||
|
- 10.0.1.12:9100
|
||||||
|
- 10.0.1.20:9100
|
||||||
|
- 10.0.2.5:9100
|
||||||
|
- 10.0.3.2:9100
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
config:
|
||||||
|
modules:
|
||||||
|
default:
|
||||||
|
user: "Admin"
|
||||||
|
pass: "Richa1000"
|
||||||
|
privilege: "administrator"
|
||||||
|
driver: "LAN_2_0"
|
||||||
|
|
||||||
|
collectors:
|
||||||
|
- ipmi
|
||||||
|
- chassis
|
||||||
|
- bmc
|
||||||
@@ -0,0 +1,415 @@
|
|||||||
|
prometheus:
|
||||||
|
prometheusSpec:
|
||||||
|
retention: 7d
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
memory: 512Mi
|
||||||
|
limits:
|
||||||
|
memory: 1Gi
|
||||||
|
additionalScrapeConfigs:
|
||||||
|
|
||||||
|
- job_name: 'blackbox-icmp'
|
||||||
|
|
||||||
|
metrics_path: /probe
|
||||||
|
|
||||||
|
params:
|
||||||
|
module: [icmp]
|
||||||
|
|
||||||
|
static_configs:
|
||||||
|
- targets: ["10.0.0.1"]
|
||||||
|
labels:
|
||||||
|
hostname: router
|
||||||
|
device_type: router
|
||||||
|
|
||||||
|
- targets: ["10.0.0.123"]
|
||||||
|
labels:
|
||||||
|
hostname: switch
|
||||||
|
device_type: switch
|
||||||
|
|
||||||
|
- targets: ["10.0.0.232"]
|
||||||
|
labels:
|
||||||
|
hostname: ap-loznice
|
||||||
|
device_type: ap
|
||||||
|
|
||||||
|
- targets: ["10.0.0.233"]
|
||||||
|
labels:
|
||||||
|
hostname: ap-det-pokoj
|
||||||
|
device_type: ap
|
||||||
|
|
||||||
|
- targets: ["10.0.0.13"]
|
||||||
|
labels:
|
||||||
|
hostname: konica-minolta-bizhub-227
|
||||||
|
device_type: printer
|
||||||
|
|
||||||
|
- targets: ["10.0.0.152"]
|
||||||
|
labels:
|
||||||
|
hostname: hp-officejet-pro-6830
|
||||||
|
device_type: printer
|
||||||
|
|
||||||
|
- targets: ["10.0.2.2"]
|
||||||
|
labels:
|
||||||
|
hostname: main-nas
|
||||||
|
device_type: nas
|
||||||
|
|
||||||
|
- targets: ["10.0.2.12"]
|
||||||
|
labels:
|
||||||
|
hostname: backup-nas
|
||||||
|
device_type: nas
|
||||||
|
|
||||||
|
- targets: ["10.0.2.6"]
|
||||||
|
labels:
|
||||||
|
hostname: nas-raid-card
|
||||||
|
device_type: storage-controller
|
||||||
|
|
||||||
|
- targets: ["10.0.2.4"]
|
||||||
|
labels:
|
||||||
|
hostname: mac-mini
|
||||||
|
device_type: mac
|
||||||
|
|
||||||
|
- targets: ["10.0.0.139"]
|
||||||
|
labels:
|
||||||
|
hostname: klipper-rpi4b
|
||||||
|
device_type: raspberry-pi
|
||||||
|
|
||||||
|
- targets: ["10.0.2.130"]
|
||||||
|
labels:
|
||||||
|
hostname: r720-overlord-idrac
|
||||||
|
device_type: idrac
|
||||||
|
|
||||||
|
- targets: ["10.0.2.131"]
|
||||||
|
labels:
|
||||||
|
hostname: r620-idrac
|
||||||
|
device_type: idrac
|
||||||
|
|
||||||
|
- targets: ["10.0.2.132"]
|
||||||
|
labels:
|
||||||
|
hostname: r720-proxmox-idrac
|
||||||
|
device_type: idrac
|
||||||
|
|
||||||
|
- targets: ["10.0.2.133"]
|
||||||
|
labels:
|
||||||
|
hostname: hp-dl360p-g8-ilo
|
||||||
|
device_type: ilo
|
||||||
|
|
||||||
|
- targets: ["10.0.2.134"]
|
||||||
|
labels:
|
||||||
|
hostname: dell-precision-rack-7910-idrac
|
||||||
|
device_type: idrac
|
||||||
|
|
||||||
|
- targets: ["10.0.2.136"]
|
||||||
|
labels:
|
||||||
|
hostname: r420-idrac
|
||||||
|
device_type: idrac
|
||||||
|
|
||||||
|
- targets: ["10.0.2.137"]
|
||||||
|
labels:
|
||||||
|
hostname: t620-idrac
|
||||||
|
device_type: idrac
|
||||||
|
|
||||||
|
- targets: ["10.0.1.2"]
|
||||||
|
labels:
|
||||||
|
hostname: plexisklo
|
||||||
|
device_type: lxc
|
||||||
|
|
||||||
|
- targets: ["10.0.1.3"]
|
||||||
|
labels:
|
||||||
|
hostname: synch
|
||||||
|
device_type: lxc
|
||||||
|
|
||||||
|
- targets: ["10.0.1.4"]
|
||||||
|
labels:
|
||||||
|
hostname: graylog
|
||||||
|
device_type: vm
|
||||||
|
|
||||||
|
- targets: ["10.0.1.6"]
|
||||||
|
labels:
|
||||||
|
hostname: kozel-tajne
|
||||||
|
device_type: lxc
|
||||||
|
|
||||||
|
- targets: ["10.0.1.7"]
|
||||||
|
labels:
|
||||||
|
hostname: kozel-web
|
||||||
|
device_type: lxc
|
||||||
|
|
||||||
|
- targets: ["10.0.1.8"]
|
||||||
|
labels:
|
||||||
|
hostname: qbittorrent
|
||||||
|
device_type: lxc
|
||||||
|
|
||||||
|
- targets: ["10.0.1.9"]
|
||||||
|
labels:
|
||||||
|
hostname: docker
|
||||||
|
device_type: lxc
|
||||||
|
|
||||||
|
- targets: ["10.0.1.10"]
|
||||||
|
labels:
|
||||||
|
hostname: pihole
|
||||||
|
device_type: lxc
|
||||||
|
|
||||||
|
- targets: ["10.0.1.11"]
|
||||||
|
labels:
|
||||||
|
hostname: gitea
|
||||||
|
device_type: lxc
|
||||||
|
|
||||||
|
- targets: ["10.0.1.12"]
|
||||||
|
labels:
|
||||||
|
hostname: mariadb
|
||||||
|
device_type: lxc
|
||||||
|
|
||||||
|
- targets: ["10.0.1.20"]
|
||||||
|
labels:
|
||||||
|
hostname: zabbix
|
||||||
|
device_type: vm
|
||||||
|
|
||||||
|
- targets: ["10.0.2.5"]
|
||||||
|
labels:
|
||||||
|
hostname: proxmox
|
||||||
|
device_type: server
|
||||||
|
|
||||||
|
- targets: ["10.0.3.2"]
|
||||||
|
labels:
|
||||||
|
hostname: mc-servers
|
||||||
|
device_type: server
|
||||||
|
|
||||||
|
relabel_configs:
|
||||||
|
|
||||||
|
- source_labels: [__address__]
|
||||||
|
target_label: __param_target
|
||||||
|
|
||||||
|
- source_labels: [__param_target]
|
||||||
|
target_label: instance
|
||||||
|
|
||||||
|
- target_label: __address__
|
||||||
|
replacement: blackbox-exporter-prometheus-blackbox-exporter.monitoring.svc.cluster.local:9115
|
||||||
|
|
||||||
|
- job_name: external-node-exporters
|
||||||
|
|
||||||
|
static_configs:
|
||||||
|
- targets:
|
||||||
|
- 10.0.1.2:9100
|
||||||
|
labels:
|
||||||
|
hostname: plexisklo
|
||||||
|
|
||||||
|
- targets:
|
||||||
|
- 10.0.1.3:9100
|
||||||
|
labels:
|
||||||
|
hostname: synch
|
||||||
|
|
||||||
|
- targets:
|
||||||
|
- 10.0.1.4:9100
|
||||||
|
labels:
|
||||||
|
hostname: graylog
|
||||||
|
|
||||||
|
- targets:
|
||||||
|
- 10.0.1.6:9100
|
||||||
|
labels:
|
||||||
|
hostname: kozel-tajne
|
||||||
|
|
||||||
|
- targets:
|
||||||
|
- 10.0.1.7:9100
|
||||||
|
labels:
|
||||||
|
hostname: kozel-web
|
||||||
|
|
||||||
|
- targets:
|
||||||
|
- 10.0.1.8:9100
|
||||||
|
labels:
|
||||||
|
hostname: qbittorrent
|
||||||
|
|
||||||
|
- targets:
|
||||||
|
- 10.0.1.9:9100
|
||||||
|
labels:
|
||||||
|
hostname: docker
|
||||||
|
|
||||||
|
- targets:
|
||||||
|
- 10.0.1.10:9100
|
||||||
|
labels:
|
||||||
|
hostname: pihloe
|
||||||
|
|
||||||
|
- targets:
|
||||||
|
- 10.0.1.11:9100
|
||||||
|
labels:
|
||||||
|
hostname: gitea
|
||||||
|
|
||||||
|
- targets:
|
||||||
|
- 10.0.1.12:9100
|
||||||
|
labels:
|
||||||
|
hostname: mariadb
|
||||||
|
|
||||||
|
- targets:
|
||||||
|
- 10.0.1.20:9100
|
||||||
|
labels:
|
||||||
|
hostname: zabbix
|
||||||
|
|
||||||
|
- targets:
|
||||||
|
- 10.0.2.5:9100
|
||||||
|
labels:
|
||||||
|
hostname: proxmox
|
||||||
|
|
||||||
|
- targets:
|
||||||
|
- 10.0.3.2:9100
|
||||||
|
labels:
|
||||||
|
hostname: mc-servers
|
||||||
|
|
||||||
|
- job_name: 'snmp-if-mib'
|
||||||
|
scrape_interval: 4m
|
||||||
|
scrape_timeout: 180s
|
||||||
|
metrics_path: /snmp
|
||||||
|
params:
|
||||||
|
module: [if_mib]
|
||||||
|
auth: [public_v2]
|
||||||
|
static_configs:
|
||||||
|
- targets:
|
||||||
|
- 10.0.0.1 # Router
|
||||||
|
- 10.0.0.123 # Switch
|
||||||
|
- 10.0.0.233 # AP-Dět.Pkoj
|
||||||
|
- 10.0.0.232 # AP-Ložnice
|
||||||
|
- 10.0.2.12 # Backup-NAS
|
||||||
|
- 10.0.2.2 # Main NAS
|
||||||
|
- 10.0.0.13 # Konica Minolta
|
||||||
|
relabel_configs:
|
||||||
|
- source_labels: [__address__]
|
||||||
|
target_label: __param_target
|
||||||
|
- source_labels: [__param_target]
|
||||||
|
target_label: instance
|
||||||
|
- target_label: __address__
|
||||||
|
replacement: snmp-exporter-manual.monitoring.svc.cluster.local:9116
|
||||||
|
|
||||||
|
# - job_name: 'snmp-mib'
|
||||||
|
# scrape_interval: 4m
|
||||||
|
# scrape_timeout: 180s
|
||||||
|
# metrics_path: /snmp
|
||||||
|
# params:
|
||||||
|
# module: [mikrotik]
|
||||||
|
# auth: [public_v2]
|
||||||
|
# static_configs:
|
||||||
|
# - targets:
|
||||||
|
# - 10.0.0.1 # Router
|
||||||
|
# - 10.0.0.123 # Switch
|
||||||
|
# - 10.0.0.233 # AP-Dět.Pkoj
|
||||||
|
# - 10.0.0.232 # AP-Ložnice
|
||||||
|
# relabel_configs:
|
||||||
|
# - source_labels: [__address__]
|
||||||
|
# target_label: __param_target
|
||||||
|
# - source_labels: [__param_target]
|
||||||
|
# target_label: instance
|
||||||
|
# - target_label: __address__
|
||||||
|
# replacement: snmp-exporter-manual.monitoring.svc.cluster.local:9116
|
||||||
|
|
||||||
|
# - job_name: 'snmp-host-resources'
|
||||||
|
# scrape_interval: 4m
|
||||||
|
# scrape_timeout: 180s
|
||||||
|
# metrics_path: /snmp
|
||||||
|
# params:
|
||||||
|
# module: [resources_hst]
|
||||||
|
# auth: [public_v2]
|
||||||
|
# static_configs:
|
||||||
|
# - targets:
|
||||||
|
# - 10.0.0.1 # Router
|
||||||
|
# - 10.0.0.123 # Switch
|
||||||
|
# - 10.0.0.233 # AP-Dět.Pkoj
|
||||||
|
# - 10.0.0.232 # AP-Ložnice
|
||||||
|
# - 10.0.2.12 # Backup-NAS
|
||||||
|
# - 10.0.2.2 # Main NAS
|
||||||
|
# - 10.0.2.6 # NAS RAID karta
|
||||||
|
# - 10.0.0.13 # Konica Minolta
|
||||||
|
# relabel_configs:
|
||||||
|
# - source_labels: [__address__]
|
||||||
|
# target_label: __param_target
|
||||||
|
# - source_labels: [__param_target]
|
||||||
|
# target_label: instance
|
||||||
|
# - target_label: __address__
|
||||||
|
# replacement: snmp-exporter-manual.monitoring.svc.cluster.local:9116
|
||||||
|
|
||||||
|
# - job_name: 'snmp-dell-idrac'
|
||||||
|
# scrape_interval: 4m
|
||||||
|
# scrape_timeout: 180s
|
||||||
|
# metrics_path: /snmp
|
||||||
|
# params:
|
||||||
|
# module: [dell_rac]
|
||||||
|
# auth: [public_v2]
|
||||||
|
# static_configs:
|
||||||
|
# - targets:
|
||||||
|
# - 10.0.2.130
|
||||||
|
# - 10.0.2.131
|
||||||
|
# - 10.0.2.132
|
||||||
|
# - 10.0.2.134
|
||||||
|
# - 10.0.2.136
|
||||||
|
# - 10.0.2.137
|
||||||
|
# relabel_configs:
|
||||||
|
# - source_labels: [__address__]
|
||||||
|
# target_label: __param_target
|
||||||
|
# - source_labels: [__param_target]
|
||||||
|
# target_label: instance
|
||||||
|
# - target_label: __address__
|
||||||
|
# replacement: snmp-exporter-manual.monitoring.svc.cluster.local:9116
|
||||||
|
|
||||||
|
# - job_name: 'snmp-hpe-ilo'
|
||||||
|
# scrape_interval: 4m
|
||||||
|
# scrape_timeout: 180s
|
||||||
|
# metrics_path: /snmp
|
||||||
|
# params:
|
||||||
|
# module: [hpe]
|
||||||
|
# auth: [public_v2]
|
||||||
|
# static_configs:
|
||||||
|
# - targets:
|
||||||
|
# - 10.0.2.133
|
||||||
|
# relabel_configs:
|
||||||
|
# - source_labels: [__address__]
|
||||||
|
# target_label: __param_target
|
||||||
|
# - source_labels: [__param_target]
|
||||||
|
# target_label: instance
|
||||||
|
# - target_label: __address__
|
||||||
|
# replacement: snmp-exporter-manual.monitoring.svc.cluster.local:9116
|
||||||
|
|
||||||
|
- job_name: 'proxmox'
|
||||||
|
metrics_path: /pve
|
||||||
|
params:
|
||||||
|
module: [default]
|
||||||
|
cluster: ['1']
|
||||||
|
node: ['1']
|
||||||
|
static_configs:
|
||||||
|
- targets:
|
||||||
|
- 10.0.2.5
|
||||||
|
relabel_configs:
|
||||||
|
- source_labels: [__address__]
|
||||||
|
target_label: __param_target
|
||||||
|
- source_labels: [__param_target]
|
||||||
|
target_label: instance
|
||||||
|
- target_label: __address__
|
||||||
|
replacement: pve-exporter.monitoring.svc.cluster.local:9221
|
||||||
|
|
||||||
|
- job_name: 'ipmi-dell-ilo'
|
||||||
|
metrics_path: /ipmi
|
||||||
|
params:
|
||||||
|
module: [default]
|
||||||
|
static_configs:
|
||||||
|
- targets:
|
||||||
|
- 10.0.2.130
|
||||||
|
- 10.0.2.131
|
||||||
|
- 10.0.2.132
|
||||||
|
- 10.0.2.133
|
||||||
|
- 10.0.2.134
|
||||||
|
- 10.0.2.136
|
||||||
|
- 10.0.2.137
|
||||||
|
relabel_configs:
|
||||||
|
- source_labels: [__address__]
|
||||||
|
target_label: __param_target
|
||||||
|
- source_labels: [__param_target]
|
||||||
|
target_label: instance
|
||||||
|
- target_label: __address__
|
||||||
|
replacement: ipmi-exporter-prometheus-ipmi-exporter.monitoring.svc.cluster.local:9290
|
||||||
|
|
||||||
|
grafana:
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
memory: 128Mi
|
||||||
|
limits:
|
||||||
|
memory: 256Mi
|
||||||
|
|
||||||
|
alertmanager:
|
||||||
|
alertmanagerSpec:
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
memory: 128Mi
|
||||||
|
limits:
|
||||||
|
memory: 256Mi
|
||||||
Reference in New Issue
Block a user