feat(monitoring): add OPNsense CPU, disk, load, uptime metrics and alerts
This commit is contained in:
1 parent
28ae06834b
commit
376006e809
3 files changed
+146
-2
No files matched your search
@@ -93,7 +93,10 @@
|
|||||||
{
|
{
|
||||||
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
||||||
"fieldConfig": {
|
"fieldConfig": {
|
||||||
"defaults": { "unit": "short" },
|
"defaults": {
|
||||||
|
"unit": "short",
|
||||||
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "orange", "value": 50000 }, { "color": "red", "value": 200000 }] }
|
||||||
|
},
|
||||||
"overrides": []
|
"overrides": []
|
||||||
},
|
},
|
||||||
"gridPos": { "h": 4, "w": 4, "x": 16, "y": 0 },
|
"gridPos": { "h": 4, "w": 4, "x": 16, "y": 0 },
|
||||||
@@ -221,6 +224,65 @@
|
|||||||
"targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_service_running{name=~\"unbound|xray|netbird|dhcpd|pf|node_exporter\"}", "legendFormat": "{{name}}", "refId": "A" }],
|
"targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_service_running{name=~\"unbound|xray|netbird|dhcpd|pf|node_exporter\"}", "legendFormat": "{{name}}", "refId": "A" }],
|
||||||
"title": "OPNsense Services",
|
"title": "OPNsense Services",
|
||||||
"type": "stat"
|
"type": "stat"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": { "unit": "percent", "min": 0, "max": 100, "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 15 } },
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": { "h": 6, "w": 8, "x": 0, "y": 29 },
|
||||||
|
"id": 14,
|
||||||
|
"options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi" } },
|
||||||
|
"targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_cpu_usage_percent", "legendFormat": "CPU %", "refId": "A" }],
|
||||||
|
"title": "CPU Usage (%)",
|
||||||
|
"type": "timeseries"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": { "unit": "short", "custom": { "drawStyle": "line", "lineWidth": 1.5, "fillOpacity": 10 } },
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": { "h": 6, "w": 8, "x": 8, "y": 29 },
|
||||||
|
"id": 15,
|
||||||
|
"options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi" } },
|
||||||
|
"targets": [
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_load1", "legendFormat": "1m", "refId": "A" },
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_load5", "legendFormat": "5m", "refId": "B" },
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_load15", "legendFormat": "15m", "refId": "C" }
|
||||||
|
],
|
||||||
|
"title": "Load Average",
|
||||||
|
"type": "timeseries"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "percent",
|
||||||
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "orange", "value": 80 }, { "color": "red", "value": 95 }] }
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": { "h": 6, "w": 4, "x": 16, "y": 29 },
|
||||||
|
"id": 16,
|
||||||
|
"options": { "colorMode": "value", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" },
|
||||||
|
"targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_disk_used_percent{mountpoint=\"/\"}", "refId": "A" }],
|
||||||
|
"title": "Disk Used (/)",
|
||||||
|
"type": "stat"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": { "unit": "s" },
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": { "h": 6, "w": 4, "x": 20, "y": 29 },
|
||||||
|
"id": 17,
|
||||||
|
"options": { "colorMode": "value", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" },
|
||||||
|
"targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_uptime_seconds", "refId": "A" }],
|
||||||
|
"title": "Uptime",
|
||||||
|
"type": "stat"
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"refresh": "15s",
|
"refresh": "15s",
|
||||||
|
|||||||
@@ -65,6 +65,16 @@ def _parse_pct(text: str):
|
|||||||
return float(m.group(1)) / 100.0 if m else None
|
return float(m.group(1)) / 100.0 if m else None
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_uptime(text: str):
|
||||||
|
# Examples: "3 days, 16:05:44", "16:05:44", "1 day, 02:10:00"
|
||||||
|
m = re.match(r"^(?:(\d+)\s+days?,\s*)?(\d+):(\d+):(\d+)$", str(text or "").strip())
|
||||||
|
if not m:
|
||||||
|
return None
|
||||||
|
days = int(m.group(1) or 0)
|
||||||
|
hours, mins, secs = int(m.group(2)), int(m.group(3)), int(m.group(4))
|
||||||
|
return days * 86400 + hours * 3600 + mins * 60 + secs
|
||||||
|
|
||||||
|
|
||||||
def collect_metrics() -> str:
|
def collect_metrics() -> str:
|
||||||
lines: list[str] = []
|
lines: list[str] = []
|
||||||
|
|
||||||
@@ -75,6 +85,8 @@ def collect_metrics() -> str:
|
|||||||
pf_data = _api_get("/api/diagnostics/firewall/pfStatistics/info").get("info", {})
|
pf_data = _api_get("/api/diagnostics/firewall/pfStatistics/info").get("info", {})
|
||||||
sys_res = _api_get("/api/diagnostics/system/systemResources")
|
sys_res = _api_get("/api/diagnostics/system/systemResources")
|
||||||
sys_time = _api_get("/api/diagnostics/system/systemTime")
|
sys_time = _api_get("/api/diagnostics/system/systemTime")
|
||||||
|
sys_disk = _api_get("/api/diagnostics/system/systemDisk").get("devices", [])
|
||||||
|
sys_act = _api_get("/api/diagnostics/activity/getActivity").get("headers", [])
|
||||||
services_data = _api_get("/api/core/service/search").get("rows", [])
|
services_data = _api_get("/api/core/service/search").get("rows", [])
|
||||||
|
|
||||||
# Active IPv4 default route from kernel routing table
|
# Active IPv4 default route from kernel routing table
|
||||||
@@ -209,7 +221,7 @@ def collect_metrics() -> str:
|
|||||||
f"opnsense_pf_state_removals_total {pf_removals}",
|
f"opnsense_pf_state_removals_total {pf_removals}",
|
||||||
])
|
])
|
||||||
|
|
||||||
# System memory & load
|
# System memory, load, CPU, disk & uptime
|
||||||
mem = sys_res.get("memory", {})
|
mem = sys_res.get("memory", {})
|
||||||
if mem.get("total") is not None and mem.get("used") is not None:
|
if mem.get("total") is not None and mem.get("used") is not None:
|
||||||
lines.extend([
|
lines.extend([
|
||||||
@@ -235,6 +247,40 @@ def collect_metrics() -> str:
|
|||||||
f"opnsense_load15 {float(load_parts[2])}",
|
f"opnsense_load15 {float(load_parts[2])}",
|
||||||
])
|
])
|
||||||
|
|
||||||
|
uptime_sec = _parse_uptime(sys_time.get("uptime"))
|
||||||
|
if uptime_sec is not None:
|
||||||
|
lines.extend([
|
||||||
|
"# HELP opnsense_uptime_seconds System uptime in seconds",
|
||||||
|
"# TYPE opnsense_uptime_seconds gauge",
|
||||||
|
f"opnsense_uptime_seconds {uptime_sec}",
|
||||||
|
])
|
||||||
|
|
||||||
|
for h in sys_act:
|
||||||
|
m = re.search(r"([0-9]+(?:\.[0-9]+)?)%\s*idle", str(h))
|
||||||
|
if m:
|
||||||
|
cpu_used = round(max(0.0, 100.0 - float(m.group(1))), 2)
|
||||||
|
lines.extend([
|
||||||
|
"# HELP opnsense_cpu_usage_percent Total CPU usage percentage",
|
||||||
|
"# TYPE opnsense_cpu_usage_percent gauge",
|
||||||
|
f"opnsense_cpu_usage_percent {cpu_used}",
|
||||||
|
])
|
||||||
|
break
|
||||||
|
|
||||||
|
lines.extend([
|
||||||
|
"# HELP opnsense_disk_used_percent Disk space used percentage",
|
||||||
|
"# TYPE opnsense_disk_used_percent gauge",
|
||||||
|
])
|
||||||
|
for dev in sys_disk:
|
||||||
|
mp = dev.get("mountpoint")
|
||||||
|
pct = dev.get("used_pct")
|
||||||
|
if mp and pct is not None:
|
||||||
|
lbl = _labels({
|
||||||
|
"mountpoint": mp,
|
||||||
|
"device": dev.get("device", ""),
|
||||||
|
"fstype": dev.get("type", ""),
|
||||||
|
})
|
||||||
|
lines.append(f"opnsense_disk_used_percent{lbl} {pct}")
|
||||||
|
|
||||||
# Services
|
# Services
|
||||||
lines.extend([
|
lines.extend([
|
||||||
"# HELP opnsense_service_running OPNsense service status (1=running, 0=stopped)",
|
"# HELP opnsense_service_running OPNsense service status (1=running, 0=stopped)",
|
||||||
|
|||||||
@@ -279,4 +279,40 @@ groups:
|
|||||||
annotations:
|
annotations:
|
||||||
summary: "OPNsense переключился на резерв: {{ $labels.gateway }} ({{ $labels.descr }})"
|
summary: "OPNsense переключился на резерв: {{ $labels.gateway }} ({{ $labels.descr }})"
|
||||||
description: "Активный default-маршрут ушёл с NORCOM_DHCP на {{ $labels.gateway }} ({{ $labels.descr }}, {{ $labels.device }})."
|
description: "Активный default-маршрут ушёл с NORCOM_DHCP на {{ $labels.gateway }} ({{ $labels.descr }}, {{ $labels.device }})."
|
||||||
|
|
||||||
|
- alert: OpnsenseDiskSpaceLow
|
||||||
|
expr: opnsense_disk_used_percent{mountpoint="/"} > 85 and opnsense_disk_used_percent{mountpoint="/"} <= 95
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Мало свободного места на OPNsense (/)"
|
||||||
|
description: "Корневой раздел / на OPNsense заполнен более чем на 85%."
|
||||||
|
|
||||||
|
- alert: OpnsenseDiskSpaceCritical
|
||||||
|
expr: opnsense_disk_used_percent{mountpoint="/"} > 95
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Заканчивается место на OPNsense (/)"
|
||||||
|
description: "Корневой раздел / на OPNsense заполнен более чем на 95%."
|
||||||
|
|
||||||
|
- alert: OpnsenseHighCpuLoad
|
||||||
|
expr: opnsense_cpu_usage_percent > 90
|
||||||
|
for: 10m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Высокая нагрузка CPU на OPNsense"
|
||||||
|
description: "Утилизация CPU на OPNsense превышает 90% более 10 минут."
|
||||||
|
|
||||||
|
- alert: OpnsenseHighMemoryUsage
|
||||||
|
expr: (opnsense_memory_used_bytes / opnsense_memory_total_bytes) * 100 > 90
|
||||||
|
for: 10m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Мало оперативной памяти на OPNsense"
|
||||||
|
description: "Использование RAM на OPNsense превышает 90% более 10 минут."
|
||||||
[% endif %]
|
[% endif %]
|
||||||
Reference in new issue
Block a user