feat(monitoring): add OPNsense multi-WAN exporter, alerts, and Grafana dashboard
ansible-deploy / plan (push) Successful in 1s
ansible-lint / lint (push) Successful in 3s
ansible-deploy / deploy (core-mon, ) (push) Successful in 43s

This commit is contained in:
karpenok committed 2026-11-17 15:53:22 +07:00
1 parent c7adefd95a
commit 28ae06834b
11 files changed
+675

No files matched your search

@@ -19,6 +19,11 @@ vmagent_central_scrape_skip:
cadvisor_image: gcr.io/cadvisor/cadvisor:v0.49.2
xray_checker_image: kutovoys/xray-checker:v1.3.1
xray_checker_subscription_url: "{{ lookup('env', 'XRAY_CHECKER_SUBSCRIPTION_URL', default='') }}"
opnsense_exporter_image: "python:3.12-alpine"
opnsense_api_url: "https://192.168.1.1:4433"
opnsense_api_key: "{{ lookup('env', 'OPNSENSE_API_KEY', default='') }}"
opnsense_api_secret: "{{ lookup('env', 'OPNSENSE_API_SECRET', default='') }}"
opnsense_exporter_enabled: "{{ (opnsense_api_key | length > 0) and (opnsense_api_secret | length > 0) }}"
vmagent_cluster: homelab
# cAdvisor этих хостов слушает :9101. core-mon, core-net и ger-nod-02 пишут его своим агентом.
monitoring_cadvisor_port: 9101
@@ -0,0 +1,237 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
"id": null,
"links": [],
"panels": [
{
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
"fieldConfig": {
"defaults": {
"mappings": [
{ "options": { "0": { "color": "red", "index": 1, "text": "Offline" }, "1": { "color": "green", "index": 0, "text": "Online" } }, "type": "value" }
],
"thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] }
},
"overrides": []
},
"gridPos": { "h": 4, "w": 4, "x": 0, "y": 0 },
"id": 1,
"options": { "colorMode": "background", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" },
"targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_gateway_status{gateway=\"NORCOM_DHCP\"}", "refId": "A" }],
"title": "NorCom (Primary)",
"type": "stat"
},
{
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
"fieldConfig": {
"defaults": {
"mappings": [
{ "options": { "0": { "color": "red", "index": 1, "text": "Offline" }, "1": { "color": "green", "index": 0, "text": "Online" } }, "type": "value" }
],
"thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] }
},
"overrides": []
},
"gridPos": { "h": 4, "w": 4, "x": 4, "y": 0 },
"id": 2,
"options": { "colorMode": "background", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" },
"targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_gateway_status{gateway=\"WAN1\"}", "refId": "A" }],
"title": "MTC (WAN1)",
"type": "stat"
},
{
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
"fieldConfig": {
"defaults": {
"mappings": [
{ "options": { "0": { "color": "blue", "index": 1, "text": "Offline (sched)" }, "1": { "color": "green", "index": 0, "text": "Online" } }, "type": "value" }
],
"thresholds": { "mode": "absolute", "steps": [{ "color": "blue", "value": null }, { "color": "green", "value": 1 }] }
},
"overrides": []
},
"gridPos": { "h": 4, "w": 4, "x": 8, "y": 0 },
"id": 3,
"options": { "colorMode": "background", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" },
"targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_gateway_status{gateway=\"WAN_0\"}", "refId": "A" }],
"title": "Iterika (WAN_0)",
"type": "stat"
},
{
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
"fieldConfig": {
"defaults": {
"color": { "mode": "fixed", "fixedColor": "super-light-blue" }
},
"overrides": []
},
"gridPos": { "h": 4, "w": 4, "x": 12, "y": 0 },
"id": 4,
"options": { "colorMode": "value", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "name" },
"targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_gateway_default == 1", "legendFormat": "{{gateway}} ({{descr}})", "refId": "A" }],
"title": "Active Default GW",
"type": "stat"
},
{
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
"fieldConfig": {
"defaults": { "unit": "short" },
"overrides": []
},
"gridPos": { "h": 4, "w": 4, "x": 16, "y": 0 },
"id": 5,
"options": { "colorMode": "value", "graphMode": "area", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" },
"targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_pf_states_current", "refId": "A" }],
"title": "pf Active States",
"type": "stat"
},
{
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
"fieldConfig": {
"defaults": {
"unit": "percent",
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "orange", "value": 75 }, { "color": "red", "value": 90 }] }
},
"overrides": []
},
"gridPos": { "h": 4, "w": 4, "x": 20, "y": 0 },
"id": 6,
"options": { "colorMode": "value", "graphMode": "area", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" },
"targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "(opnsense_memory_used_bytes / opnsense_memory_total_bytes) * 100", "refId": "A" }],
"title": "RAM Usage",
"type": "stat"
},
{
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
"fieldConfig": {
"defaults": {
"mappings": [
{ "options": { "0": { "color": "red", "index": 1, "text": "Offline" }, "1": { "color": "green", "index": 0, "text": "Online" } }, "type": "value" }
]
},
"overrides": []
},
"gridPos": { "h": 5, "w": 24, "x": 0, "y": 4 },
"id": 7,
"options": { "alignValue": "left", "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "mergeValues": true, "rowHeight": 0.85, "showValue": "auto" },
"targets": [
{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_gateway_status{gateway=~\"NORCOM_DHCP|WAN1|WAN_0\"}", "legendFormat": "{{gateway}} ({{descr}})", "refId": "A" },
{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_gateway_default{gateway=~\"NORCOM_DHCP|WAN1|WAN_0\"}", "legendFormat": "default: {{gateway}}", "refId": "B" }
],
"title": "Gateway Availability & Active Default Route Timeline",
"type": "state-timeline"
},
{
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
"fieldConfig": {
"defaults": { "unit": "ms", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10 } },
"overrides": []
},
"gridPos": { "h": 7, "w": 12, "x": 0, "y": 9 },
"id": 8,
"options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": ["mean", "max", "lastNotNull"] }, "tooltip": { "mode": "multi" } },
"targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_gateway_delay_seconds{gateway=~\"NORCOM_DHCP|WAN1|WAN_0\"} * 1000", "legendFormat": "{{gateway}} ({{descr}} -> {{monitor}})", "refId": "A" }],
"title": "Gateway Latency (RTT, dpinger)",
"type": "timeseries"
},
{
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
"fieldConfig": {
"defaults": { "unit": "percent", "min": 0, "max": 100, "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 15 } },
"overrides": []
},
"gridPos": { "h": 7, "w": 12, "x": 12, "y": 9 },
"id": 9,
"options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": ["mean", "max", "lastNotNull"] }, "tooltip": { "mode": "multi" } },
"targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_gateway_loss_ratio{gateway=~\"NORCOM_DHCP|WAN1|WAN_0\"} * 100", "legendFormat": "{{gateway}} ({{descr}})", "refId": "A" }],
"title": "Gateway Packet Loss (%)",
"type": "timeseries"
},
{
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
"fieldConfig": {
"defaults": { "unit": "bps", "custom": { "drawStyle": "line", "lineWidth": 1.5, "fillOpacity": 15 } },
"overrides": []
},
"gridPos": { "h": 7, "w": 12, "x": 0, "y": 16 },
"id": 10,
"options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": ["mean", "max", "lastNotNull"] }, "tooltip": { "mode": "multi" } },
"targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "rate(opnsense_interface_rx_bytes_total{device=~\"igc0|igc1|igc2|proxytun2socks0|vtnet0\"}[1m]) * 8", "legendFormat": "RX {{descr}} ({{device}})", "refId": "A" }],
"title": "Interface Download (RX)",
"type": "timeseries"
},
{
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
"fieldConfig": {
"defaults": { "unit": "bps", "custom": { "drawStyle": "line", "lineWidth": 1.5, "fillOpacity": 15 } },
"overrides": []
},
"gridPos": { "h": 7, "w": 12, "x": 12, "y": 16 },
"id": 11,
"options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": ["mean", "max", "lastNotNull"] }, "tooltip": { "mode": "multi" } },
"targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "rate(opnsense_interface_tx_bytes_total{device=~\"igc0|igc1|igc2|proxytun2socks0|vtnet0\"}[1m]) * 8", "legendFormat": "TX {{descr}} ({{device}})", "refId": "A" }],
"title": "Interface Upload (TX)",
"type": "timeseries"
},
{
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
"fieldConfig": {
"defaults": { "unit": "pps", "custom": { "drawStyle": "line", "lineWidth": 1.5, "fillOpacity": 10 } },
"overrides": []
},
"gridPos": { "h": 6, "w": 12, "x": 0, "y": 23 },
"id": 12,
"options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi" } },
"targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "rate(opnsense_interface_rx_errors_total{device=~\"igc0|igc1|igc2|vtnet0\"}[5m]) + rate(opnsense_interface_tx_errors_total{device=~\"igc0|igc1|igc2|vtnet0\"}[5m]) + rate(opnsense_interface_dropped_packets_total{device=~\"igc0|igc1|igc2|vtnet0\"}[5m])", "legendFormat": "{{descr}} ({{device}})", "refId": "A" }],
"title": "Interface Errors & Drops",
"type": "timeseries"
},
{
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
"fieldConfig": {
"defaults": {
"mappings": [
{ "options": { "0": { "color": "red", "index": 1, "text": "Stopped" }, "1": { "color": "green", "index": 0, "text": "Running" } }, "type": "value" }
],
"thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] }
},
"overrides": []
},
"gridPos": { "h": 6, "w": 12, "x": 12, "y": 23 },
"id": 13,
"options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "value_and_name" },
"targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_service_running{name=~\"unbound|xray|netbird|dhcpd|pf|node_exporter\"}", "legendFormat": "{{name}}", "refId": "A" }],
"title": "OPNsense Services",
"type": "stat"
}
],
"refresh": "15s",
"schemaVersion": 39,
"tags": ["opnsense", "wan", "network"],
"templating": { "list": [] },
"time": { "from": "now-6h", "to": "now" },
"timepicker": {},
"timezone": "",
"title": "OPNsense — Multi-WAN & Router",
"uid": "opnsense-wan",
"version": 1
}
@@ -0,0 +1,292 @@
#!/usr/bin/env python3
"""Lightweight Prometheus exporter for OPNsense gateways, interfaces, pf, and system metrics."""
import base64
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
import json
import os
import re
import ssl
import sys
import urllib.error
import urllib.request
API_URL = os.environ.get("OPNSENSE_API_URL", "https://192.168.1.1:4433").rstrip("/")
API_KEY = os.environ.get("OPNSENSE_API_KEY", "")
API_SECRET = os.environ.get("OPNSENSE_API_SECRET", "")
VERIFY_TLS = os.environ.get("OPNSENSE_VERIFY_TLS", "false").lower() in ("1", "true", "yes")
LISTEN_PORT = int(os.environ.get("OPNSENSE_EXPORTER_PORT", "9105"))
TIMEOUT = float(os.environ.get("OPNSENSE_API_TIMEOUT", "5"))
def _ssl_context() -> ssl.SSLContext:
ctx = ssl.create_default_context()
if not VERIFY_TLS:
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
return ctx
def _auth_header() -> str:
raw = f"{API_KEY}:{API_SECRET}".encode("utf-8")
return "Basic " + base64.b64encode(raw).decode("ascii")
def _api_get(path: str):
req = urllib.request.Request(
f"{API_URL}{path}",
headers={
"Authorization": _auth_header(),
"Accept": "application/json",
},
method="GET",
)
with urllib.request.urlopen(req, context=_ssl_context(), timeout=TIMEOUT) as resp:
return json.loads(resp.read().decode("utf-8"))
def _esc(val: str) -> str:
return str(val).replace("\\", "\\\\").replace('"', '\\"').replace("\n", "\\n")
def _labels(mapping: dict[str, str]) -> str:
parts = [f'{k}="{_esc(v)}"' for k, v in mapping.items()]
return "{" + ",".join(parts) + "}"
def _parse_ms(text: str):
m = re.match(r"^\s*([0-9]+(?:\.[0-9]+)?)\s*ms\s*$", str(text or ""))
return float(m.group(1)) / 1000.0 if m else None
def _parse_pct(text: str):
m = re.match(r"^\s*([0-9]+(?:\.[0-9]+)?)\s*%\s*$", str(text or ""))
return float(m.group(1)) / 100.0 if m else None
def collect_metrics() -> str:
lines: list[str] = []
gateways_data = _api_get("/api/routing/settings/searchGateway").get("rows", [])
routes_data = _api_get("/api/diagnostics/interface/getRoutes")
ifaces_data = _api_get("/api/interfaces/overview/interfacesInfo").get("rows", [])
ifstats_data = _api_get("/api/diagnostics/interface/getInterfaceStatistics").get("statistics", {})
pf_data = _api_get("/api/diagnostics/firewall/pfStatistics/info").get("info", {})
sys_res = _api_get("/api/diagnostics/system/systemResources")
sys_time = _api_get("/api/diagnostics/system/systemTime")
services_data = _api_get("/api/core/service/search").get("rows", [])
# Active IPv4 default route from kernel routing table
active_default_gw = None
active_default_if = None
for r in routes_data:
if r.get("proto") == "ipv4" and r.get("destination") == "default":
active_default_gw = r.get("gateway")
active_default_if = r.get("netif")
break
# Gateways (dpinger)
lines.extend([
"# HELP opnsense_gateway_status Gateway online status (1=Online, 0=Offline)",
"# TYPE opnsense_gateway_status gauge",
"# HELP opnsense_gateway_configured_default Configured default gateway flag",
"# TYPE opnsense_gateway_configured_default gauge",
"# HELP opnsense_gateway_default Active kernel default route flag (1=active default)",
"# TYPE opnsense_gateway_default gauge",
"# HELP opnsense_gateway_delay_seconds Gateway monitor RTT in seconds",
"# TYPE opnsense_gateway_delay_seconds gauge",
"# HELP opnsense_gateway_stddev_seconds Gateway monitor RTT stddev in seconds",
"# TYPE opnsense_gateway_stddev_seconds gauge",
"# HELP opnsense_gateway_loss_ratio Gateway monitor packet loss ratio (0.0-1.0)",
"# TYPE opnsense_gateway_loss_ratio gauge",
])
for gw in gateways_data:
name = gw.get("name", "")
descr = gw.get("interface_descr") or gw.get("descr") or name
iface = gw.get("interface", "")
device = gw.get("if", "")
monitor = gw.get("monitor", "")
gw_addr = gw.get("gateway", "")
lbl = _labels({
"gateway": name,
"descr": descr,
"interface": iface,
"device": device,
"monitor": monitor,
})
online = 1 if gw.get("status") == "Online" else 0
lines.append(f"opnsense_gateway_status{lbl} {online}")
lines.append(f"opnsense_gateway_configured_default{lbl} {1 if gw.get('defaultgw') else 0}")
is_active_default = 1 if (
active_default_gw and active_default_if
and gw_addr == active_default_gw and device == active_default_if
) else 0
lines.append(f"opnsense_gateway_default{lbl} {is_active_default}")
loss = _parse_pct(gw.get("loss"))
if loss is not None:
lines.append(f"opnsense_gateway_loss_ratio{lbl} {loss}")
if online:
delay = _parse_ms(gw.get("delay"))
stddev = _parse_ms(gw.get("stddev"))
if delay is not None:
lines.append(f"opnsense_gateway_delay_seconds{lbl} {delay}")
if stddev is not None:
lines.append(f"opnsense_gateway_stddev_seconds{lbl} {stddev}")
# Interfaces status & traffic counters
device_meta: dict[str, tuple[str, str]] = {}
lines.extend([
"# HELP opnsense_interface_up Interface operational status (1=up, 0=down)",
"# TYPE opnsense_interface_up gauge",
])
for iface in ifaces_data:
ident = iface.get("identifier") or ""
device = iface.get("device") or ""
descr = iface.get("description") or ident or device
if not ident or not device:
continue
device_meta[device] = (ident, descr)
up = 1 if iface.get("status") == "up" else 0
lbl = _labels({"interface": ident, "device": device, "descr": descr})
lines.append(f"opnsense_interface_up{lbl} {up}")
lines.extend([
"# HELP opnsense_interface_rx_bytes_total Received bytes on interface",
"# TYPE opnsense_interface_rx_bytes_total counter",
"# HELP opnsense_interface_tx_bytes_total Transmitted bytes on interface",
"# TYPE opnsense_interface_tx_bytes_total counter",
"# HELP opnsense_interface_rx_packets_total Received packets on interface",
"# TYPE opnsense_interface_rx_packets_total counter",
"# HELP opnsense_interface_tx_packets_total Transmitted packets on interface",
"# TYPE opnsense_interface_tx_packets_total counter",
"# HELP opnsense_interface_rx_errors_total Receive errors on interface",
"# TYPE opnsense_interface_rx_errors_total counter",
"# HELP opnsense_interface_tx_errors_total Transmit errors on interface",
"# TYPE opnsense_interface_tx_errors_total counter",
"# HELP opnsense_interface_dropped_packets_total Dropped packets on interface",
"# TYPE opnsense_interface_dropped_packets_total counter",
])
seen_devices: set[str] = set()
for stat in ifstats_data.values():
device = stat.get("name")
if not device or device not in device_meta or device in seen_devices:
continue
if stat.get("received-bytes") is None:
continue
seen_devices.add(device)
ident, descr = device_meta[device]
lbl = _labels({"interface": ident, "device": device, "descr": descr})
lines.append(f"opnsense_interface_rx_bytes_total{lbl} {stat['received-bytes']}")
lines.append(f"opnsense_interface_tx_bytes_total{lbl} {stat['sent-bytes']}")
lines.append(f"opnsense_interface_rx_packets_total{lbl} {stat['received-packets']}")
lines.append(f"opnsense_interface_tx_packets_total{lbl} {stat['sent-packets']}")
lines.append(f"opnsense_interface_rx_errors_total{lbl} {stat.get('received-errors') or 0}")
lines.append(f"opnsense_interface_tx_errors_total{lbl} {stat.get('send-errors') or 0}")
lines.append(f"opnsense_interface_dropped_packets_total{lbl} {stat.get('dropped-packets') or 0}")
# pf state table
state_table = pf_data.get("state-table", {})
pf_current = state_table.get("current-entries", {}).get("total")
pf_inserts = state_table.get("inserts", {}).get("total")
pf_removals = state_table.get("removals", {}).get("total")
if pf_current is not None:
lines.extend([
"# HELP opnsense_pf_states_current Current active pf states",
"# TYPE opnsense_pf_states_current gauge",
f"opnsense_pf_states_current {pf_current}",
])
if pf_inserts is not None and pf_removals is not None:
lines.extend([
"# HELP opnsense_pf_state_inserts_total Total pf state insertions",
"# TYPE opnsense_pf_state_inserts_total counter",
f"opnsense_pf_state_inserts_total {pf_inserts}",
"# HELP opnsense_pf_state_removals_total Total pf state removals",
"# TYPE opnsense_pf_state_removals_total counter",
f"opnsense_pf_state_removals_total {pf_removals}",
])
# System memory & load
mem = sys_res.get("memory", {})
if mem.get("total") is not None and mem.get("used") is not None:
lines.extend([
"# HELP opnsense_memory_total_bytes Total physical memory in bytes",
"# TYPE opnsense_memory_total_bytes gauge",
f"opnsense_memory_total_bytes {mem['total']}",
"# HELP opnsense_memory_used_bytes Used memory in bytes",
"# TYPE opnsense_memory_used_bytes gauge",
f"opnsense_memory_used_bytes {mem['used']}",
])
load_parts = [p.strip() for p in str(sys_time.get("loadavg", "")).split(",") if p.strip()]
if len(load_parts) == 3:
lines.extend([
"# HELP opnsense_load1 1-minute system load average",
"# TYPE opnsense_load1 gauge",
f"opnsense_load1 {float(load_parts[0])}",
"# HELP opnsense_load5 5-minute system load average",
"# TYPE opnsense_load5 gauge",
f"opnsense_load5 {float(load_parts[1])}",
"# HELP opnsense_load15 15-minute system load average",
"# TYPE opnsense_load15 gauge",
f"opnsense_load15 {float(load_parts[2])}",
])
# Services
lines.extend([
"# HELP opnsense_service_running OPNsense service status (1=running, 0=stopped)",
"# TYPE opnsense_service_running gauge",
])
for svc in services_data:
lbl = _labels({
"service_id": svc.get("id", ""),
"name": svc.get("name", ""),
})
lines.append(f"opnsense_service_running{lbl} {1 if svc.get('running') else 0}")
return "\n".join(lines) + "\n"
class MetricsHandler(BaseHTTPRequestHandler):
def do_GET(self):
if self.path == "/health":
self.send_response(200)
self.send_header("Content-Type", "text/plain")
self.end_headers()
self.wfile.write(b"ok\n")
return
if self.path != "/metrics":
self.send_response(404)
self.end_headers()
return
try:
body = collect_metrics().encode("utf-8")
self.send_response(200)
self.send_header("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
self.end_headers()
self.wfile.write(body)
except (urllib.error.URLError, TimeoutError, json.JSONDecodeError, KeyError) as exc:
msg = f"# error collecting OPNsense metrics: {exc}\n".encode("utf-8")
self.send_response(503)
self.send_header("Content-Type", "text/plain; charset=utf-8")
self.end_headers()
self.wfile.write(msg)
def log_message(self, format, *args):
return
def main():
if not API_KEY or not API_SECRET:
print("OPNSENSE_API_KEY and OPNSENSE_API_SECRET must be set", file=sys.stderr)
sys.exit(1)
server = ThreadingHTTPServer(("0.0.0.0", LISTEN_PORT), MetricsHandler)
server.serve_forever()
if __name__ == "__main__":
main()
@@ -18,6 +18,13 @@
when: xray_checker_env is defined and xray_checker_env is changed
changed_when: true
- name: Пересоздать экспортер OPNsense при обновлении скрипта или окружения
ansible.builtin.command: docker compose -f {{ monitoring_dir }}/compose.yaml up -d --force-recreate --no-deps opnsense-exporter
when: >
(opnsense_exporter_env is defined and opnsense_exporter_env is changed) or
(opnsense_exporter_script is defined and opnsense_exporter_script is changed)
changed_when: true
- name: Дождаться VictoriaMetrics
ansible.builtin.uri:
url: "http://{{ ansible_host }}:8428/health"
@@ -21,3 +21,4 @@
- { path: "{{ monitoring_dir }}/vmagent", owner: "root", group: "root", mode: "0755" }
- { path: "{{ monitoring_dir }}/vmagent/conf.d", owner: "root", group: "root", mode: "0755" }
- { path: "{{ monitoring_dir }}/vmalert", owner: "root", group: "root", mode: "0755" }
- { path: "{{ monitoring_dir }}/opnsense-exporter", owner: "root", group: "root", mode: "0755" }
@@ -96,3 +96,42 @@
mode: "0644"
when: xray_checker_subscription_url | length > 0
- name: Убрать scrape OPNsense без API-ключей
ansible.builtin.file:
path: "{{ item }}"
state: absent
loop:
- "{{ monitoring_dir }}/vmagent/conf.d/opnsense.yml"
- "{{ monitoring_dir }}/opnsense-exporter.env"
when: not (opnsense_exporter_enabled | bool)
- name: Скрипт экспортера OPNsense
ansible.builtin.copy:
src: opnsense-exporter/exporter.py
dest: "{{ monitoring_dir }}/opnsense-exporter/exporter.py"
owner: root
group: root
mode: "0755"
register: opnsense_exporter_script
when: opnsense_exporter_enabled | bool
- name: Файл окружения экспортера OPNsense
ansible.builtin.template:
src: opnsense-exporter.env.j2
dest: "{{ monitoring_dir }}/opnsense-exporter.env"
owner: root
group: root
mode: "0600"
no_log: true
register: opnsense_exporter_env
when: opnsense_exporter_enabled | bool
- name: Scrape OPNsense
ansible.builtin.template:
src: vmagent-opnsense.yml.j2
dest: "{{ monitoring_dir }}/vmagent/conf.d/opnsense.yml"
owner: root
group: root
mode: "0644"
when: opnsense_exporter_enabled | bool
@@ -218,3 +218,65 @@ groups:
annotations:
summary: "Прокси {{ $labels.name }} не проходит проверку"
description: "xray_proxy_status {{ $labels.name }} ({{ $labels.address }}) не равен 1."
[% if opnsense_exporter_enabled | bool %]
- alert: OpnsenseExporterDown
expr: up{job="opnsense"} == 0
for: 2m
labels:
severity: critical
annotations:
summary: "Экспортер OPNsense недоступен"
description: "Scrape opnsense ({{ $labels.instance }}) возвращает up=0 более 2 минут."
- alert: OpnsenseMetricsStale
expr: |
(
time() - tlast_over_time(up{job="opnsense"}[30m]) > 180
)
or absent_over_time(up{job="opnsense",instance="opnsense"}[30m])
for: 1m
labels:
severity: critical
annotations:
summary: "Нет свежих метрик OPNsense"
description: "opnsense-exporter не присылает up дольше 3 минут."
# Мониторим отвал только постоянных каналов (NorCom и MTC); WAN_0 (Iterika) работает по расписанию.
- alert: OpnsenseGatewayDown
expr: opnsense_gateway_status{gateway=~"NORCOM_DHCP|WAN1"} == 0
for: 2m
labels:
severity: critical
annotations:
summary: "Провайдер OPNsense {{ $labels.gateway }} ({{ $labels.descr }}) недоступен"
description: "Шлюз {{ $labels.gateway }} (монитор {{ $labels.monitor }}) в статусе Offline более 2 минут."
- alert: OpnsenseGatewayPacketLoss
expr: opnsense_gateway_loss_ratio{gateway=~"NORCOM_DHCP|WAN1"} > 0.1
for: 3m
labels:
severity: warning
annotations:
summary: "Потери пакетов на {{ $labels.gateway }} ({{ $labels.descr }})"
description: "Потери на шлюзе {{ $labels.gateway }} превышают 10% более 3 минут."
- alert: OpnsenseGatewayHighLatency
expr: opnsense_gateway_delay_seconds{gateway=~"NORCOM_DHCP|WAN1"} > 0.25
for: 5m
labels:
severity: warning
annotations:
summary: "Высокая задержка на {{ $labels.gateway }} ({{ $labels.descr }})"
description: "RTT до {{ $labels.monitor }} через {{ $labels.gateway }} превышает 250 ms более 5 минут."
# Уведомление при переключении default-маршрута на резерв (включая WAN_0 / Iterika).
- alert: OpnsenseFailoverActive
expr: opnsense_gateway_default{gateway!="NORCOM_DHCP"} == 1
for: 30s
labels:
severity: warning
annotations:
summary: "OPNsense переключился на резерв: {{ $labels.gateway }} ({{ $labels.descr }})"
description: "Активный default-маршрут ушёл с NORCOM_DHCP на {{ $labels.gateway }} ({{ $labels.descr }}, {{ $labels.device }})."
[% endif %]
@@ -103,6 +103,20 @@ services:
ports:
- "127.0.0.1:2112:2112"
{% endif %}
{% if opnsense_exporter_enabled | bool %}
opnsense-exporter:
image: {{ opnsense_exporter_image }}
container_name: monitoring-opnsense-exporter
restart: unless-stopped
command: ["python3", "/app/exporter.py"]
env_file:
- ./opnsense-exporter.env
volumes:
- ./opnsense-exporter/exporter.py:/app/exporter.py:ro
ports:
- "127.0.0.1:9105:9105"
{% endif %}
vmagent:
image: {{ vmagent_image }}
@@ -0,0 +1,6 @@
OPNSENSE_API_URL={{ opnsense_api_url }}
OPNSENSE_API_KEY={{ opnsense_api_key }}
OPNSENSE_API_SECRET={{ opnsense_api_secret }}
OPNSENSE_VERIFY_TLS=false
OPNSENSE_EXPORTER_PORT=9105
@@ -0,0 +1,9 @@
- job_name: opnsense
scrape_interval: 15s
static_configs:
- targets: ["opnsense-exporter:9105"]
labels:
instance: "opnsense"
cluster: "{{ vmagent_cluster }}"
host: "opnsense"