diff --git a/ansible/ci/deploy-machine.sh b/ansible/ci/deploy-machine.sh index e8f259f..762f5f9 100755 --- a/ansible/ci/deploy-machine.sh +++ b/ansible/ci/deploy-machine.sh @@ -89,6 +89,9 @@ case "$machine" in fi eval "$(infisical export --domain="${INFISICAL_DOMAIN}" --token="${TOKEN}" --env=prod --projectId="${INFISICAL_PROJECT_ID}" --path="/services/remnawave" --format=dotenv-export --silent 2>/dev/null || true)" export XRAY_CHECKER_SUBSCRIPTION_URL="${XRAY_CHECKER_SUBSCRIPTION_URL:-}" + eval "$(infisical export --domain="${INFISICAL_DOMAIN}" --token="${TOKEN}" --env=prod --projectId="${INFISICAL_PROJECT_ID}" --path="/services/opnsense" --format=dotenv-export --silent 2>/dev/null || true)" + export OPNSENSE_API_KEY="${OPNSENSE_API_KEY:-}" + export OPNSENSE_API_SECRET="${OPNSENSE_API_SECRET:-}" ;; rus-central) input_email="${INPUT_LETSENCRYPT_EMAIL:-}" diff --git a/ansible/roles/core-mon/monitoring/defaults/main.yml b/ansible/roles/core-mon/monitoring/defaults/main.yml index efd3769..7d71956 100644 --- a/ansible/roles/core-mon/monitoring/defaults/main.yml +++ b/ansible/roles/core-mon/monitoring/defaults/main.yml @@ -19,6 +19,11 @@ vmagent_central_scrape_skip: cadvisor_image: gcr.io/cadvisor/cadvisor:v0.49.2 xray_checker_image: kutovoys/xray-checker:v1.3.1 xray_checker_subscription_url: "{{ lookup('env', 'XRAY_CHECKER_SUBSCRIPTION_URL', default='') }}" +opnsense_exporter_image: "python:3.12-alpine" +opnsense_api_url: "https://192.168.1.1:4433" +opnsense_api_key: "{{ lookup('env', 'OPNSENSE_API_KEY', default='') }}" +opnsense_api_secret: "{{ lookup('env', 'OPNSENSE_API_SECRET', default='') }}" +opnsense_exporter_enabled: "{{ (opnsense_api_key | length > 0) and (opnsense_api_secret | length > 0) }}" vmagent_cluster: homelab # cAdvisor этих хостов слушает :9101. core-mon, core-net и ger-nod-02 пишут его своим агентом. monitoring_cadvisor_port: 9101 diff --git a/ansible/roles/core-mon/monitoring/files/dashboards/opnsense-wan.json b/ansible/roles/core-mon/monitoring/files/dashboards/opnsense-wan.json new file mode 100644 index 0000000..6e17a58 --- /dev/null +++ b/ansible/roles/core-mon/monitoring/files/dashboards/opnsense-wan.json @@ -0,0 +1,237 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "datasource": { "type": "prometheus", "uid": "victoriametrics" }, + "fieldConfig": { + "defaults": { + "mappings": [ + { "options": { "0": { "color": "red", "index": 1, "text": "Offline" }, "1": { "color": "green", "index": 0, "text": "Online" } }, "type": "value" } + ], + "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] } + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 4, "x": 0, "y": 0 }, + "id": 1, + "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" }, + "targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_gateway_status{gateway=\"NORCOM_DHCP\"}", "refId": "A" }], + "title": "NorCom (Primary)", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "victoriametrics" }, + "fieldConfig": { + "defaults": { + "mappings": [ + { "options": { "0": { "color": "red", "index": 1, "text": "Offline" }, "1": { "color": "green", "index": 0, "text": "Online" } }, "type": "value" } + ], + "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] } + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 4, "x": 4, "y": 0 }, + "id": 2, + "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" }, + "targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_gateway_status{gateway=\"WAN1\"}", "refId": "A" }], + "title": "MTC (WAN1)", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "victoriametrics" }, + "fieldConfig": { + "defaults": { + "mappings": [ + { "options": { "0": { "color": "blue", "index": 1, "text": "Offline (sched)" }, "1": { "color": "green", "index": 0, "text": "Online" } }, "type": "value" } + ], + "thresholds": { "mode": "absolute", "steps": [{ "color": "blue", "value": null }, { "color": "green", "value": 1 }] } + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 4, "x": 8, "y": 0 }, + "id": 3, + "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" }, + "targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_gateway_status{gateway=\"WAN_0\"}", "refId": "A" }], + "title": "Iterika (WAN_0)", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "victoriametrics" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "fixed", "fixedColor": "super-light-blue" } + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 4, "x": 12, "y": 0 }, + "id": 4, + "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "name" }, + "targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_gateway_default == 1", "legendFormat": "{{gateway}} ({{descr}})", "refId": "A" }], + "title": "Active Default GW", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "victoriametrics" }, + "fieldConfig": { + "defaults": { "unit": "short" }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 4, "x": 16, "y": 0 }, + "id": 5, + "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" }, + "targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_pf_states_current", "refId": "A" }], + "title": "pf Active States", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "victoriametrics" }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "orange", "value": 75 }, { "color": "red", "value": 90 }] } + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 4, "x": 20, "y": 0 }, + "id": 6, + "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" }, + "targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "(opnsense_memory_used_bytes / opnsense_memory_total_bytes) * 100", "refId": "A" }], + "title": "RAM Usage", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "victoriametrics" }, + "fieldConfig": { + "defaults": { + "mappings": [ + { "options": { "0": { "color": "red", "index": 1, "text": "Offline" }, "1": { "color": "green", "index": 0, "text": "Online" } }, "type": "value" } + ] + }, + "overrides": [] + }, + "gridPos": { "h": 5, "w": 24, "x": 0, "y": 4 }, + "id": 7, + "options": { "alignValue": "left", "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "mergeValues": true, "rowHeight": 0.85, "showValue": "auto" }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_gateway_status{gateway=~\"NORCOM_DHCP|WAN1|WAN_0\"}", "legendFormat": "{{gateway}} ({{descr}})", "refId": "A" }, + { "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_gateway_default{gateway=~\"NORCOM_DHCP|WAN1|WAN_0\"}", "legendFormat": "default: {{gateway}}", "refId": "B" } + ], + "title": "Gateway Availability & Active Default Route Timeline", + "type": "state-timeline" + }, + { + "datasource": { "type": "prometheus", "uid": "victoriametrics" }, + "fieldConfig": { + "defaults": { "unit": "ms", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10 } }, + "overrides": [] + }, + "gridPos": { "h": 7, "w": 12, "x": 0, "y": 9 }, + "id": 8, + "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": ["mean", "max", "lastNotNull"] }, "tooltip": { "mode": "multi" } }, + "targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_gateway_delay_seconds{gateway=~\"NORCOM_DHCP|WAN1|WAN_0\"} * 1000", "legendFormat": "{{gateway}} ({{descr}} -> {{monitor}})", "refId": "A" }], + "title": "Gateway Latency (RTT, dpinger)", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "victoriametrics" }, + "fieldConfig": { + "defaults": { "unit": "percent", "min": 0, "max": 100, "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 15 } }, + "overrides": [] + }, + "gridPos": { "h": 7, "w": 12, "x": 12, "y": 9 }, + "id": 9, + "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": ["mean", "max", "lastNotNull"] }, "tooltip": { "mode": "multi" } }, + "targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_gateway_loss_ratio{gateway=~\"NORCOM_DHCP|WAN1|WAN_0\"} * 100", "legendFormat": "{{gateway}} ({{descr}})", "refId": "A" }], + "title": "Gateway Packet Loss (%)", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "victoriametrics" }, + "fieldConfig": { + "defaults": { "unit": "bps", "custom": { "drawStyle": "line", "lineWidth": 1.5, "fillOpacity": 15 } }, + "overrides": [] + }, + "gridPos": { "h": 7, "w": 12, "x": 0, "y": 16 }, + "id": 10, + "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": ["mean", "max", "lastNotNull"] }, "tooltip": { "mode": "multi" } }, + "targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "rate(opnsense_interface_rx_bytes_total{device=~\"igc0|igc1|igc2|proxytun2socks0|vtnet0\"}[1m]) * 8", "legendFormat": "RX {{descr}} ({{device}})", "refId": "A" }], + "title": "Interface Download (RX)", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "victoriametrics" }, + "fieldConfig": { + "defaults": { "unit": "bps", "custom": { "drawStyle": "line", "lineWidth": 1.5, "fillOpacity": 15 } }, + "overrides": [] + }, + "gridPos": { "h": 7, "w": 12, "x": 12, "y": 16 }, + "id": 11, + "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": ["mean", "max", "lastNotNull"] }, "tooltip": { "mode": "multi" } }, + "targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "rate(opnsense_interface_tx_bytes_total{device=~\"igc0|igc1|igc2|proxytun2socks0|vtnet0\"}[1m]) * 8", "legendFormat": "TX {{descr}} ({{device}})", "refId": "A" }], + "title": "Interface Upload (TX)", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "victoriametrics" }, + "fieldConfig": { + "defaults": { "unit": "pps", "custom": { "drawStyle": "line", "lineWidth": 1.5, "fillOpacity": 10 } }, + "overrides": [] + }, + "gridPos": { "h": 6, "w": 12, "x": 0, "y": 23 }, + "id": 12, + "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi" } }, + "targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "rate(opnsense_interface_rx_errors_total{device=~\"igc0|igc1|igc2|vtnet0\"}[5m]) + rate(opnsense_interface_tx_errors_total{device=~\"igc0|igc1|igc2|vtnet0\"}[5m]) + rate(opnsense_interface_dropped_packets_total{device=~\"igc0|igc1|igc2|vtnet0\"}[5m])", "legendFormat": "{{descr}} ({{device}})", "refId": "A" }], + "title": "Interface Errors & Drops", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "victoriametrics" }, + "fieldConfig": { + "defaults": { + "mappings": [ + { "options": { "0": { "color": "red", "index": 1, "text": "Stopped" }, "1": { "color": "green", "index": 0, "text": "Running" } }, "type": "value" } + ], + "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] } + }, + "overrides": [] + }, + "gridPos": { "h": 6, "w": 12, "x": 12, "y": 23 }, + "id": 13, + "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "value_and_name" }, + "targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_service_running{name=~\"unbound|xray|netbird|dhcpd|pf|node_exporter\"}", "legendFormat": "{{name}}", "refId": "A" }], + "title": "OPNsense Services", + "type": "stat" + } + ], + "refresh": "15s", + "schemaVersion": 39, + "tags": ["opnsense", "wan", "network"], + "templating": { "list": [] }, + "time": { "from": "now-6h", "to": "now" }, + "timepicker": {}, + "timezone": "", + "title": "OPNsense — Multi-WAN & Router", + "uid": "opnsense-wan", + "version": 1 +} + diff --git a/ansible/roles/core-mon/monitoring/files/opnsense-exporter/exporter.py b/ansible/roles/core-mon/monitoring/files/opnsense-exporter/exporter.py new file mode 100644 index 0000000..136ec53 --- /dev/null +++ b/ansible/roles/core-mon/monitoring/files/opnsense-exporter/exporter.py @@ -0,0 +1,292 @@ +#!/usr/bin/env python3 +"""Lightweight Prometheus exporter for OPNsense gateways, interfaces, pf, and system metrics.""" + +import base64 +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import json +import os +import re +import ssl +import sys +import urllib.error +import urllib.request + + +API_URL = os.environ.get("OPNSENSE_API_URL", "https://192.168.1.1:4433").rstrip("/") +API_KEY = os.environ.get("OPNSENSE_API_KEY", "") +API_SECRET = os.environ.get("OPNSENSE_API_SECRET", "") +VERIFY_TLS = os.environ.get("OPNSENSE_VERIFY_TLS", "false").lower() in ("1", "true", "yes") +LISTEN_PORT = int(os.environ.get("OPNSENSE_EXPORTER_PORT", "9105")) +TIMEOUT = float(os.environ.get("OPNSENSE_API_TIMEOUT", "5")) + + +def _ssl_context() -> ssl.SSLContext: + ctx = ssl.create_default_context() + if not VERIFY_TLS: + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + return ctx + + +def _auth_header() -> str: + raw = f"{API_KEY}:{API_SECRET}".encode("utf-8") + return "Basic " + base64.b64encode(raw).decode("ascii") + + +def _api_get(path: str): + req = urllib.request.Request( + f"{API_URL}{path}", + headers={ + "Authorization": _auth_header(), + "Accept": "application/json", + }, + method="GET", + ) + with urllib.request.urlopen(req, context=_ssl_context(), timeout=TIMEOUT) as resp: + return json.loads(resp.read().decode("utf-8")) + + +def _esc(val: str) -> str: + return str(val).replace("\\", "\\\\").replace('"', '\\"').replace("\n", "\\n") + + +def _labels(mapping: dict[str, str]) -> str: + parts = [f'{k}="{_esc(v)}"' for k, v in mapping.items()] + return "{" + ",".join(parts) + "}" + + +def _parse_ms(text: str): + m = re.match(r"^\s*([0-9]+(?:\.[0-9]+)?)\s*ms\s*$", str(text or "")) + return float(m.group(1)) / 1000.0 if m else None + + +def _parse_pct(text: str): + m = re.match(r"^\s*([0-9]+(?:\.[0-9]+)?)\s*%\s*$", str(text or "")) + return float(m.group(1)) / 100.0 if m else None + + +def collect_metrics() -> str: + lines: list[str] = [] + + gateways_data = _api_get("/api/routing/settings/searchGateway").get("rows", []) + routes_data = _api_get("/api/diagnostics/interface/getRoutes") + ifaces_data = _api_get("/api/interfaces/overview/interfacesInfo").get("rows", []) + ifstats_data = _api_get("/api/diagnostics/interface/getInterfaceStatistics").get("statistics", {}) + pf_data = _api_get("/api/diagnostics/firewall/pfStatistics/info").get("info", {}) + sys_res = _api_get("/api/diagnostics/system/systemResources") + sys_time = _api_get("/api/diagnostics/system/systemTime") + services_data = _api_get("/api/core/service/search").get("rows", []) + + # Active IPv4 default route from kernel routing table + active_default_gw = None + active_default_if = None + for r in routes_data: + if r.get("proto") == "ipv4" and r.get("destination") == "default": + active_default_gw = r.get("gateway") + active_default_if = r.get("netif") + break + + # Gateways (dpinger) + lines.extend([ + "# HELP opnsense_gateway_status Gateway online status (1=Online, 0=Offline)", + "# TYPE opnsense_gateway_status gauge", + "# HELP opnsense_gateway_configured_default Configured default gateway flag", + "# TYPE opnsense_gateway_configured_default gauge", + "# HELP opnsense_gateway_default Active kernel default route flag (1=active default)", + "# TYPE opnsense_gateway_default gauge", + "# HELP opnsense_gateway_delay_seconds Gateway monitor RTT in seconds", + "# TYPE opnsense_gateway_delay_seconds gauge", + "# HELP opnsense_gateway_stddev_seconds Gateway monitor RTT stddev in seconds", + "# TYPE opnsense_gateway_stddev_seconds gauge", + "# HELP opnsense_gateway_loss_ratio Gateway monitor packet loss ratio (0.0-1.0)", + "# TYPE opnsense_gateway_loss_ratio gauge", + ]) + for gw in gateways_data: + name = gw.get("name", "") + descr = gw.get("interface_descr") or gw.get("descr") or name + iface = gw.get("interface", "") + device = gw.get("if", "") + monitor = gw.get("monitor", "") + gw_addr = gw.get("gateway", "") + lbl = _labels({ + "gateway": name, + "descr": descr, + "interface": iface, + "device": device, + "monitor": monitor, + }) + online = 1 if gw.get("status") == "Online" else 0 + lines.append(f"opnsense_gateway_status{lbl} {online}") + lines.append(f"opnsense_gateway_configured_default{lbl} {1 if gw.get('defaultgw') else 0}") + + is_active_default = 1 if ( + active_default_gw and active_default_if + and gw_addr == active_default_gw and device == active_default_if + ) else 0 + lines.append(f"opnsense_gateway_default{lbl} {is_active_default}") + + loss = _parse_pct(gw.get("loss")) + if loss is not None: + lines.append(f"opnsense_gateway_loss_ratio{lbl} {loss}") + + if online: + delay = _parse_ms(gw.get("delay")) + stddev = _parse_ms(gw.get("stddev")) + if delay is not None: + lines.append(f"opnsense_gateway_delay_seconds{lbl} {delay}") + if stddev is not None: + lines.append(f"opnsense_gateway_stddev_seconds{lbl} {stddev}") + + # Interfaces status & traffic counters + device_meta: dict[str, tuple[str, str]] = {} + lines.extend([ + "# HELP opnsense_interface_up Interface operational status (1=up, 0=down)", + "# TYPE opnsense_interface_up gauge", + ]) + for iface in ifaces_data: + ident = iface.get("identifier") or "" + device = iface.get("device") or "" + descr = iface.get("description") or ident or device + if not ident or not device: + continue + device_meta[device] = (ident, descr) + up = 1 if iface.get("status") == "up" else 0 + lbl = _labels({"interface": ident, "device": device, "descr": descr}) + lines.append(f"opnsense_interface_up{lbl} {up}") + + lines.extend([ + "# HELP opnsense_interface_rx_bytes_total Received bytes on interface", + "# TYPE opnsense_interface_rx_bytes_total counter", + "# HELP opnsense_interface_tx_bytes_total Transmitted bytes on interface", + "# TYPE opnsense_interface_tx_bytes_total counter", + "# HELP opnsense_interface_rx_packets_total Received packets on interface", + "# TYPE opnsense_interface_rx_packets_total counter", + "# HELP opnsense_interface_tx_packets_total Transmitted packets on interface", + "# TYPE opnsense_interface_tx_packets_total counter", + "# HELP opnsense_interface_rx_errors_total Receive errors on interface", + "# TYPE opnsense_interface_rx_errors_total counter", + "# HELP opnsense_interface_tx_errors_total Transmit errors on interface", + "# TYPE opnsense_interface_tx_errors_total counter", + "# HELP opnsense_interface_dropped_packets_total Dropped packets on interface", + "# TYPE opnsense_interface_dropped_packets_total counter", + ]) + seen_devices: set[str] = set() + for stat in ifstats_data.values(): + device = stat.get("name") + if not device or device not in device_meta or device in seen_devices: + continue + if stat.get("received-bytes") is None: + continue + seen_devices.add(device) + ident, descr = device_meta[device] + lbl = _labels({"interface": ident, "device": device, "descr": descr}) + lines.append(f"opnsense_interface_rx_bytes_total{lbl} {stat['received-bytes']}") + lines.append(f"opnsense_interface_tx_bytes_total{lbl} {stat['sent-bytes']}") + lines.append(f"opnsense_interface_rx_packets_total{lbl} {stat['received-packets']}") + lines.append(f"opnsense_interface_tx_packets_total{lbl} {stat['sent-packets']}") + lines.append(f"opnsense_interface_rx_errors_total{lbl} {stat.get('received-errors') or 0}") + lines.append(f"opnsense_interface_tx_errors_total{lbl} {stat.get('send-errors') or 0}") + lines.append(f"opnsense_interface_dropped_packets_total{lbl} {stat.get('dropped-packets') or 0}") + + # pf state table + state_table = pf_data.get("state-table", {}) + pf_current = state_table.get("current-entries", {}).get("total") + pf_inserts = state_table.get("inserts", {}).get("total") + pf_removals = state_table.get("removals", {}).get("total") + if pf_current is not None: + lines.extend([ + "# HELP opnsense_pf_states_current Current active pf states", + "# TYPE opnsense_pf_states_current gauge", + f"opnsense_pf_states_current {pf_current}", + ]) + if pf_inserts is not None and pf_removals is not None: + lines.extend([ + "# HELP opnsense_pf_state_inserts_total Total pf state insertions", + "# TYPE opnsense_pf_state_inserts_total counter", + f"opnsense_pf_state_inserts_total {pf_inserts}", + "# HELP opnsense_pf_state_removals_total Total pf state removals", + "# TYPE opnsense_pf_state_removals_total counter", + f"opnsense_pf_state_removals_total {pf_removals}", + ]) + + # System memory & load + mem = sys_res.get("memory", {}) + if mem.get("total") is not None and mem.get("used") is not None: + lines.extend([ + "# HELP opnsense_memory_total_bytes Total physical memory in bytes", + "# TYPE opnsense_memory_total_bytes gauge", + f"opnsense_memory_total_bytes {mem['total']}", + "# HELP opnsense_memory_used_bytes Used memory in bytes", + "# TYPE opnsense_memory_used_bytes gauge", + f"opnsense_memory_used_bytes {mem['used']}", + ]) + + load_parts = [p.strip() for p in str(sys_time.get("loadavg", "")).split(",") if p.strip()] + if len(load_parts) == 3: + lines.extend([ + "# HELP opnsense_load1 1-minute system load average", + "# TYPE opnsense_load1 gauge", + f"opnsense_load1 {float(load_parts[0])}", + "# HELP opnsense_load5 5-minute system load average", + "# TYPE opnsense_load5 gauge", + f"opnsense_load5 {float(load_parts[1])}", + "# HELP opnsense_load15 15-minute system load average", + "# TYPE opnsense_load15 gauge", + f"opnsense_load15 {float(load_parts[2])}", + ]) + + # Services + lines.extend([ + "# HELP opnsense_service_running OPNsense service status (1=running, 0=stopped)", + "# TYPE opnsense_service_running gauge", + ]) + for svc in services_data: + lbl = _labels({ + "service_id": svc.get("id", ""), + "name": svc.get("name", ""), + }) + lines.append(f"opnsense_service_running{lbl} {1 if svc.get('running') else 0}") + + return "\n".join(lines) + "\n" + + +class MetricsHandler(BaseHTTPRequestHandler): + def do_GET(self): + if self.path == "/health": + self.send_response(200) + self.send_header("Content-Type", "text/plain") + self.end_headers() + self.wfile.write(b"ok\n") + return + if self.path != "/metrics": + self.send_response(404) + self.end_headers() + return + try: + body = collect_metrics().encode("utf-8") + self.send_response(200) + self.send_header("Content-Type", "text/plain; version=0.0.4; charset=utf-8") + self.end_headers() + self.wfile.write(body) + except (urllib.error.URLError, TimeoutError, json.JSONDecodeError, KeyError) as exc: + msg = f"# error collecting OPNsense metrics: {exc}\n".encode("utf-8") + self.send_response(503) + self.send_header("Content-Type", "text/plain; charset=utf-8") + self.end_headers() + self.wfile.write(msg) + + def log_message(self, format, *args): + return + + +def main(): + if not API_KEY or not API_SECRET: + print("OPNSENSE_API_KEY and OPNSENSE_API_SECRET must be set", file=sys.stderr) + sys.exit(1) + server = ThreadingHTTPServer(("0.0.0.0", LISTEN_PORT), MetricsHandler) + server.serve_forever() + + +if __name__ == "__main__": + main() + diff --git a/ansible/roles/core-mon/monitoring/tasks/compose.yml b/ansible/roles/core-mon/monitoring/tasks/compose.yml index a01ba77..b2d3c4a 100644 --- a/ansible/roles/core-mon/monitoring/tasks/compose.yml +++ b/ansible/roles/core-mon/monitoring/tasks/compose.yml @@ -18,6 +18,13 @@ when: xray_checker_env is defined and xray_checker_env is changed changed_when: true +- name: Пересоздать экспортер OPNsense при обновлении скрипта или окружения + ansible.builtin.command: docker compose -f {{ monitoring_dir }}/compose.yaml up -d --force-recreate --no-deps opnsense-exporter + when: > + (opnsense_exporter_env is defined and opnsense_exporter_env is changed) or + (opnsense_exporter_script is defined and opnsense_exporter_script is changed) + changed_when: true + - name: Дождаться VictoriaMetrics ansible.builtin.uri: url: "http://{{ ansible_host }}:8428/health" diff --git a/ansible/roles/core-mon/monitoring/tasks/dirs.yml b/ansible/roles/core-mon/monitoring/tasks/dirs.yml index ca096f9..200b5c4 100644 --- a/ansible/roles/core-mon/monitoring/tasks/dirs.yml +++ b/ansible/roles/core-mon/monitoring/tasks/dirs.yml @@ -21,3 +21,4 @@ - { path: "{{ monitoring_dir }}/vmagent", owner: "root", group: "root", mode: "0755" } - { path: "{{ monitoring_dir }}/vmagent/conf.d", owner: "root", group: "root", mode: "0755" } - { path: "{{ monitoring_dir }}/vmalert", owner: "root", group: "root", mode: "0755" } + - { path: "{{ monitoring_dir }}/opnsense-exporter", owner: "root", group: "root", mode: "0755" } diff --git a/ansible/roles/core-mon/monitoring/tasks/vm.yml b/ansible/roles/core-mon/monitoring/tasks/vm.yml index a372956..04f9c47 100644 --- a/ansible/roles/core-mon/monitoring/tasks/vm.yml +++ b/ansible/roles/core-mon/monitoring/tasks/vm.yml @@ -96,3 +96,42 @@ mode: "0644" when: xray_checker_subscription_url | length > 0 +- name: Убрать scrape OPNsense без API-ключей + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: + - "{{ monitoring_dir }}/vmagent/conf.d/opnsense.yml" + - "{{ monitoring_dir }}/opnsense-exporter.env" + when: not (opnsense_exporter_enabled | bool) + +- name: Скрипт экспортера OPNsense + ansible.builtin.copy: + src: opnsense-exporter/exporter.py + dest: "{{ monitoring_dir }}/opnsense-exporter/exporter.py" + owner: root + group: root + mode: "0755" + register: opnsense_exporter_script + when: opnsense_exporter_enabled | bool + +- name: Файл окружения экспортера OPNsense + ansible.builtin.template: + src: opnsense-exporter.env.j2 + dest: "{{ monitoring_dir }}/opnsense-exporter.env" + owner: root + group: root + mode: "0600" + no_log: true + register: opnsense_exporter_env + when: opnsense_exporter_enabled | bool + +- name: Scrape OPNsense + ansible.builtin.template: + src: vmagent-opnsense.yml.j2 + dest: "{{ monitoring_dir }}/vmagent/conf.d/opnsense.yml" + owner: root + group: root + mode: "0644" + when: opnsense_exporter_enabled | bool + diff --git a/ansible/roles/core-mon/monitoring/templates/alerts.yml.j2 b/ansible/roles/core-mon/monitoring/templates/alerts.yml.j2 index b511395..814e9b9 100644 --- a/ansible/roles/core-mon/monitoring/templates/alerts.yml.j2 +++ b/ansible/roles/core-mon/monitoring/templates/alerts.yml.j2 @@ -218,3 +218,65 @@ groups: annotations: summary: "Прокси {{ $labels.name }} не проходит проверку" description: "xray_proxy_status {{ $labels.name }} ({{ $labels.address }}) не равен 1." +[% if opnsense_exporter_enabled | bool %] + + - alert: OpnsenseExporterDown + expr: up{job="opnsense"} == 0 + for: 2m + labels: + severity: critical + annotations: + summary: "Экспортер OPNsense недоступен" + description: "Scrape opnsense ({{ $labels.instance }}) возвращает up=0 более 2 минут." + + - alert: OpnsenseMetricsStale + expr: | + ( + time() - tlast_over_time(up{job="opnsense"}[30m]) > 180 + ) + or absent_over_time(up{job="opnsense",instance="opnsense"}[30m]) + for: 1m + labels: + severity: critical + annotations: + summary: "Нет свежих метрик OPNsense" + description: "opnsense-exporter не присылает up дольше 3 минут." + + # Мониторим отвал только постоянных каналов (NorCom и MTC); WAN_0 (Iterika) работает по расписанию. + - alert: OpnsenseGatewayDown + expr: opnsense_gateway_status{gateway=~"NORCOM_DHCP|WAN1"} == 0 + for: 2m + labels: + severity: critical + annotations: + summary: "Провайдер OPNsense {{ $labels.gateway }} ({{ $labels.descr }}) недоступен" + description: "Шлюз {{ $labels.gateway }} (монитор {{ $labels.monitor }}) в статусе Offline более 2 минут." + + - alert: OpnsenseGatewayPacketLoss + expr: opnsense_gateway_loss_ratio{gateway=~"NORCOM_DHCP|WAN1"} > 0.1 + for: 3m + labels: + severity: warning + annotations: + summary: "Потери пакетов на {{ $labels.gateway }} ({{ $labels.descr }})" + description: "Потери на шлюзе {{ $labels.gateway }} превышают 10% более 3 минут." + + - alert: OpnsenseGatewayHighLatency + expr: opnsense_gateway_delay_seconds{gateway=~"NORCOM_DHCP|WAN1"} > 0.25 + for: 5m + labels: + severity: warning + annotations: + summary: "Высокая задержка на {{ $labels.gateway }} ({{ $labels.descr }})" + description: "RTT до {{ $labels.monitor }} через {{ $labels.gateway }} превышает 250 ms более 5 минут." + + # Уведомление при переключении default-маршрута на резерв (включая WAN_0 / Iterika). + - alert: OpnsenseFailoverActive + expr: opnsense_gateway_default{gateway!="NORCOM_DHCP"} == 1 + for: 30s + labels: + severity: warning + annotations: + summary: "OPNsense переключился на резерв: {{ $labels.gateway }} ({{ $labels.descr }})" + description: "Активный default-маршрут ушёл с NORCOM_DHCP на {{ $labels.gateway }} ({{ $labels.descr }}, {{ $labels.device }})." +[% endif %] diff --git a/ansible/roles/core-mon/monitoring/templates/compose.yaml.j2 b/ansible/roles/core-mon/monitoring/templates/compose.yaml.j2 index a0b5cf4..82592fe 100644 --- a/ansible/roles/core-mon/monitoring/templates/compose.yaml.j2 +++ b/ansible/roles/core-mon/monitoring/templates/compose.yaml.j2 @@ -103,6 +103,20 @@ services: ports: - "127.0.0.1:2112:2112" {% endif %} +{% if opnsense_exporter_enabled | bool %} + + opnsense-exporter: + image: {{ opnsense_exporter_image }} + container_name: monitoring-opnsense-exporter + restart: unless-stopped + command: ["python3", "/app/exporter.py"] + env_file: + - ./opnsense-exporter.env + volumes: + - ./opnsense-exporter/exporter.py:/app/exporter.py:ro + ports: + - "127.0.0.1:9105:9105" +{% endif %} vmagent: image: {{ vmagent_image }} diff --git a/ansible/roles/core-mon/monitoring/templates/opnsense-exporter.env.j2 b/ansible/roles/core-mon/monitoring/templates/opnsense-exporter.env.j2 new file mode 100644 index 0000000..02508ac --- /dev/null +++ b/ansible/roles/core-mon/monitoring/templates/opnsense-exporter.env.j2 @@ -0,0 +1,6 @@ +OPNSENSE_API_URL={{ opnsense_api_url }} +OPNSENSE_API_KEY={{ opnsense_api_key }} +OPNSENSE_API_SECRET={{ opnsense_api_secret }} +OPNSENSE_VERIFY_TLS=false +OPNSENSE_EXPORTER_PORT=9105 + diff --git a/ansible/roles/core-mon/monitoring/templates/vmagent-opnsense.yml.j2 b/ansible/roles/core-mon/monitoring/templates/vmagent-opnsense.yml.j2 new file mode 100644 index 0000000..a4944bd --- /dev/null +++ b/ansible/roles/core-mon/monitoring/templates/vmagent-opnsense.yml.j2 @@ -0,0 +1,9 @@ +- job_name: opnsense + scrape_interval: 15s + static_configs: + - targets: ["opnsense-exporter:9105"] + labels: + instance: "opnsense" + cluster: "{{ vmagent_cluster }}" + host: "opnsense" +