feat(monitoring): add OPNsense CPU, disk, load, uptime metrics and alerts
ansible-deploy / plan (push) Successful in 2s
ansible-lint / lint (push) Successful in 3s
ansible-deploy / deploy (core-mon, ) (push) Successful in 33s

This commit is contained in:
karpenok committed 2026-11-17 16:00:19 +07:00
1 parent 28ae06834b
commit 376006e809
3 files changed
+146 -2

No files matched your search

@@ -93,7 +93,10 @@
{
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
"fieldConfig": {
"defaults": { "unit": "short" },
"defaults": {
"unit": "short",
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "orange", "value": 50000 }, { "color": "red", "value": 200000 }] }
},
"overrides": []
},
"gridPos": { "h": 4, "w": 4, "x": 16, "y": 0 },
@@ -221,6 +224,65 @@
"targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_service_running{name=~\"unbound|xray|netbird|dhcpd|pf|node_exporter\"}", "legendFormat": "{{name}}", "refId": "A" }],
"title": "OPNsense Services",
"type": "stat"
},
{
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
"fieldConfig": {
"defaults": { "unit": "percent", "min": 0, "max": 100, "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 15 } },
"overrides": []
},
"gridPos": { "h": 6, "w": 8, "x": 0, "y": 29 },
"id": 14,
"options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi" } },
"targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_cpu_usage_percent", "legendFormat": "CPU %", "refId": "A" }],
"title": "CPU Usage (%)",
"type": "timeseries"
},
{
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
"fieldConfig": {
"defaults": { "unit": "short", "custom": { "drawStyle": "line", "lineWidth": 1.5, "fillOpacity": 10 } },
"overrides": []
},
"gridPos": { "h": 6, "w": 8, "x": 8, "y": 29 },
"id": 15,
"options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi" } },
"targets": [
{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_load1", "legendFormat": "1m", "refId": "A" },
{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_load5", "legendFormat": "5m", "refId": "B" },
{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_load15", "legendFormat": "15m", "refId": "C" }
],
"title": "Load Average",
"type": "timeseries"
},
{
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
"fieldConfig": {
"defaults": {
"unit": "percent",
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "orange", "value": 80 }, { "color": "red", "value": 95 }] }
},
"overrides": []
},
"gridPos": { "h": 6, "w": 4, "x": 16, "y": 29 },
"id": 16,
"options": { "colorMode": "value", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" },
"targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_disk_used_percent{mountpoint=\"/\"}", "refId": "A" }],
"title": "Disk Used (/)",
"type": "stat"
},
{
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
"fieldConfig": {
"defaults": { "unit": "s" },
"overrides": []
},
"gridPos": { "h": 6, "w": 4, "x": 20, "y": 29 },
"id": 17,
"options": { "colorMode": "value", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" },
"targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_uptime_seconds", "refId": "A" }],
"title": "Uptime",
"type": "stat"
}
],
"refresh": "15s",
@@ -65,6 +65,16 @@ def _parse_pct(text: str):
return float(m.group(1)) / 100.0 if m else None
def _parse_uptime(text: str):
# Examples: "3 days, 16:05:44", "16:05:44", "1 day, 02:10:00"
m = re.match(r"^(?:(\d+)\s+days?,\s*)?(\d+):(\d+):(\d+)$", str(text or "").strip())
if not m:
return None
days = int(m.group(1) or 0)
hours, mins, secs = int(m.group(2)), int(m.group(3)), int(m.group(4))
return days * 86400 + hours * 3600 + mins * 60 + secs
def collect_metrics() -> str:
lines: list[str] = []
@@ -75,6 +85,8 @@ def collect_metrics() -> str:
pf_data = _api_get("/api/diagnostics/firewall/pfStatistics/info").get("info", {})
sys_res = _api_get("/api/diagnostics/system/systemResources")
sys_time = _api_get("/api/diagnostics/system/systemTime")
sys_disk = _api_get("/api/diagnostics/system/systemDisk").get("devices", [])
sys_act = _api_get("/api/diagnostics/activity/getActivity").get("headers", [])
services_data = _api_get("/api/core/service/search").get("rows", [])
# Active IPv4 default route from kernel routing table
@@ -209,7 +221,7 @@ def collect_metrics() -> str:
f"opnsense_pf_state_removals_total {pf_removals}",
])
# System memory & load
# System memory, load, CPU, disk & uptime
mem = sys_res.get("memory", {})
if mem.get("total") is not None and mem.get("used") is not None:
lines.extend([
@@ -235,6 +247,40 @@ def collect_metrics() -> str:
f"opnsense_load15 {float(load_parts[2])}",
])
uptime_sec = _parse_uptime(sys_time.get("uptime"))
if uptime_sec is not None:
lines.extend([
"# HELP opnsense_uptime_seconds System uptime in seconds",
"# TYPE opnsense_uptime_seconds gauge",
f"opnsense_uptime_seconds {uptime_sec}",
])
for h in sys_act:
m = re.search(r"([0-9]+(?:\.[0-9]+)?)%\s*idle", str(h))
if m:
cpu_used = round(max(0.0, 100.0 - float(m.group(1))), 2)
lines.extend([
"# HELP opnsense_cpu_usage_percent Total CPU usage percentage",
"# TYPE opnsense_cpu_usage_percent gauge",
f"opnsense_cpu_usage_percent {cpu_used}",
])
break
lines.extend([
"# HELP opnsense_disk_used_percent Disk space used percentage",
"# TYPE opnsense_disk_used_percent gauge",
])
for dev in sys_disk:
mp = dev.get("mountpoint")
pct = dev.get("used_pct")
if mp and pct is not None:
lbl = _labels({
"mountpoint": mp,
"device": dev.get("device", ""),
"fstype": dev.get("type", ""),
})
lines.append(f"opnsense_disk_used_percent{lbl} {pct}")
# Services
lines.extend([
"# HELP opnsense_service_running OPNsense service status (1=running, 0=stopped)",
@@ -279,4 +279,40 @@ groups:
annotations:
summary: "OPNsense переключился на резерв: {{ $labels.gateway }} ({{ $labels.descr }})"
description: "Активный default-маршрут ушёл с NORCOM_DHCP на {{ $labels.gateway }} ({{ $labels.descr }}, {{ $labels.device }})."
- alert: OpnsenseDiskSpaceLow
expr: opnsense_disk_used_percent{mountpoint="/"} > 85 and opnsense_disk_used_percent{mountpoint="/"} <= 95
for: 5m
labels:
severity: warning
annotations:
summary: "Мало свободного места на OPNsense (/)"
description: "Корневой раздел / на OPNsense заполнен более чем на 85%."
- alert: OpnsenseDiskSpaceCritical
expr: opnsense_disk_used_percent{mountpoint="/"} > 95
for: 5m
labels:
severity: critical
annotations:
summary: "Заканчивается место на OPNsense (/)"
description: "Корневой раздел / на OPNsense заполнен более чем на 95%."
- alert: OpnsenseHighCpuLoad
expr: opnsense_cpu_usage_percent > 90
for: 10m
labels:
severity: warning
annotations:
summary: "Высокая нагрузка CPU на OPNsense"
description: "Утилизация CPU на OPNsense превышает 90% более 10 минут."
- alert: OpnsenseHighMemoryUsage
expr: (opnsense_memory_used_bytes / opnsense_memory_total_bytes) * 100 > 90
for: 10m
labels:
severity: warning
annotations:
summary: "Мало оперативной памяти на OPNsense"
description: "Использование RAM на OPNsense превышает 90% более 10 минут."
[% endif %]