diff --git a/ansible/roles/core-mon/monitoring/files/dashboards/opnsense-wan.json b/ansible/roles/core-mon/monitoring/files/dashboards/opnsense-wan.json index 6e17a58..831a0bc 100644 --- a/ansible/roles/core-mon/monitoring/files/dashboards/opnsense-wan.json +++ b/ansible/roles/core-mon/monitoring/files/dashboards/opnsense-wan.json @@ -93,7 +93,10 @@ { "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "fieldConfig": { - "defaults": { "unit": "short" }, + "defaults": { + "unit": "short", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "orange", "value": 50000 }, { "color": "red", "value": 200000 }] } + }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 16, "y": 0 }, @@ -221,6 +224,65 @@ "targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_service_running{name=~\"unbound|xray|netbird|dhcpd|pf|node_exporter\"}", "legendFormat": "{{name}}", "refId": "A" }], "title": "OPNsense Services", "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "victoriametrics" }, + "fieldConfig": { + "defaults": { "unit": "percent", "min": 0, "max": 100, "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 15 } }, + "overrides": [] + }, + "gridPos": { "h": 6, "w": 8, "x": 0, "y": 29 }, + "id": 14, + "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi" } }, + "targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_cpu_usage_percent", "legendFormat": "CPU %", "refId": "A" }], + "title": "CPU Usage (%)", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "victoriametrics" }, + "fieldConfig": { + "defaults": { "unit": "short", "custom": { "drawStyle": "line", "lineWidth": 1.5, "fillOpacity": 10 } }, + "overrides": [] + }, + "gridPos": { "h": 6, "w": 8, "x": 8, "y": 29 }, + "id": 15, + "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi" } }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_load1", "legendFormat": "1m", "refId": "A" }, + { "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_load5", "legendFormat": "5m", "refId": "B" }, + { "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_load15", "legendFormat": "15m", "refId": "C" } + ], + "title": "Load Average", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "victoriametrics" }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "orange", "value": 80 }, { "color": "red", "value": 95 }] } + }, + "overrides": [] + }, + "gridPos": { "h": 6, "w": 4, "x": 16, "y": 29 }, + "id": 16, + "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" }, + "targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_disk_used_percent{mountpoint=\"/\"}", "refId": "A" }], + "title": "Disk Used (/)", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "victoriametrics" }, + "fieldConfig": { + "defaults": { "unit": "s" }, + "overrides": [] + }, + "gridPos": { "h": 6, "w": 4, "x": 20, "y": 29 }, + "id": 17, + "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" }, + "targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_uptime_seconds", "refId": "A" }], + "title": "Uptime", + "type": "stat" } ], "refresh": "15s", diff --git a/ansible/roles/core-mon/monitoring/files/opnsense-exporter/exporter.py b/ansible/roles/core-mon/monitoring/files/opnsense-exporter/exporter.py index 136ec53..e12953e 100644 --- a/ansible/roles/core-mon/monitoring/files/opnsense-exporter/exporter.py +++ b/ansible/roles/core-mon/monitoring/files/opnsense-exporter/exporter.py @@ -65,6 +65,16 @@ def _parse_pct(text: str): return float(m.group(1)) / 100.0 if m else None +def _parse_uptime(text: str): + # Examples: "3 days, 16:05:44", "16:05:44", "1 day, 02:10:00" + m = re.match(r"^(?:(\d+)\s+days?,\s*)?(\d+):(\d+):(\d+)$", str(text or "").strip()) + if not m: + return None + days = int(m.group(1) or 0) + hours, mins, secs = int(m.group(2)), int(m.group(3)), int(m.group(4)) + return days * 86400 + hours * 3600 + mins * 60 + secs + + def collect_metrics() -> str: lines: list[str] = [] @@ -75,6 +85,8 @@ def collect_metrics() -> str: pf_data = _api_get("/api/diagnostics/firewall/pfStatistics/info").get("info", {}) sys_res = _api_get("/api/diagnostics/system/systemResources") sys_time = _api_get("/api/diagnostics/system/systemTime") + sys_disk = _api_get("/api/diagnostics/system/systemDisk").get("devices", []) + sys_act = _api_get("/api/diagnostics/activity/getActivity").get("headers", []) services_data = _api_get("/api/core/service/search").get("rows", []) # Active IPv4 default route from kernel routing table @@ -209,7 +221,7 @@ def collect_metrics() -> str: f"opnsense_pf_state_removals_total {pf_removals}", ]) - # System memory & load + # System memory, load, CPU, disk & uptime mem = sys_res.get("memory", {}) if mem.get("total") is not None and mem.get("used") is not None: lines.extend([ @@ -235,6 +247,40 @@ def collect_metrics() -> str: f"opnsense_load15 {float(load_parts[2])}", ]) + uptime_sec = _parse_uptime(sys_time.get("uptime")) + if uptime_sec is not None: + lines.extend([ + "# HELP opnsense_uptime_seconds System uptime in seconds", + "# TYPE opnsense_uptime_seconds gauge", + f"opnsense_uptime_seconds {uptime_sec}", + ]) + + for h in sys_act: + m = re.search(r"([0-9]+(?:\.[0-9]+)?)%\s*idle", str(h)) + if m: + cpu_used = round(max(0.0, 100.0 - float(m.group(1))), 2) + lines.extend([ + "# HELP opnsense_cpu_usage_percent Total CPU usage percentage", + "# TYPE opnsense_cpu_usage_percent gauge", + f"opnsense_cpu_usage_percent {cpu_used}", + ]) + break + + lines.extend([ + "# HELP opnsense_disk_used_percent Disk space used percentage", + "# TYPE opnsense_disk_used_percent gauge", + ]) + for dev in sys_disk: + mp = dev.get("mountpoint") + pct = dev.get("used_pct") + if mp and pct is not None: + lbl = _labels({ + "mountpoint": mp, + "device": dev.get("device", ""), + "fstype": dev.get("type", ""), + }) + lines.append(f"opnsense_disk_used_percent{lbl} {pct}") + # Services lines.extend([ "# HELP opnsense_service_running OPNsense service status (1=running, 0=stopped)", diff --git a/ansible/roles/core-mon/monitoring/templates/alerts.yml.j2 b/ansible/roles/core-mon/monitoring/templates/alerts.yml.j2 index 814e9b9..fde2a57 100644 --- a/ansible/roles/core-mon/monitoring/templates/alerts.yml.j2 +++ b/ansible/roles/core-mon/monitoring/templates/alerts.yml.j2 @@ -279,4 +279,40 @@ groups: annotations: summary: "OPNsense переключился на резерв: {{ $labels.gateway }} ({{ $labels.descr }})" description: "Активный default-маршрут ушёл с NORCOM_DHCP на {{ $labels.gateway }} ({{ $labels.descr }}, {{ $labels.device }})." + + - alert: OpnsenseDiskSpaceLow + expr: opnsense_disk_used_percent{mountpoint="/"} > 85 and opnsense_disk_used_percent{mountpoint="/"} <= 95 + for: 5m + labels: + severity: warning + annotations: + summary: "Мало свободного места на OPNsense (/)" + description: "Корневой раздел / на OPNsense заполнен более чем на 85%." + + - alert: OpnsenseDiskSpaceCritical + expr: opnsense_disk_used_percent{mountpoint="/"} > 95 + for: 5m + labels: + severity: critical + annotations: + summary: "Заканчивается место на OPNsense (/)" + description: "Корневой раздел / на OPNsense заполнен более чем на 95%." + + - alert: OpnsenseHighCpuLoad + expr: opnsense_cpu_usage_percent > 90 + for: 10m + labels: + severity: warning + annotations: + summary: "Высокая нагрузка CPU на OPNsense" + description: "Утилизация CPU на OPNsense превышает 90% более 10 минут." + + - alert: OpnsenseHighMemoryUsage + expr: (opnsense_memory_used_bytes / opnsense_memory_total_bytes) * 100 > 90 + for: 10m + labels: + severity: warning + annotations: + summary: "Мало оперативной памяти на OPNsense" + description: "Использование RAM на OPNsense превышает 90% более 10 минут." [% endif %]