feat(monitoring): add OPNsense multi-WAN exporter, alerts, and Grafana dashboard
This commit is contained in:
1 parent
c7adefd95a
commit
28ae06834b
11 files changed
+675
No files matched your search
@@ -89,6 +89,9 @@ case "$machine" in
|
|||||||
fi
|
fi
|
||||||
eval "$(infisical export --domain="${INFISICAL_DOMAIN}" --token="${TOKEN}" --env=prod --projectId="${INFISICAL_PROJECT_ID}" --path="/services/remnawave" --format=dotenv-export --silent 2>/dev/null || true)"
|
eval "$(infisical export --domain="${INFISICAL_DOMAIN}" --token="${TOKEN}" --env=prod --projectId="${INFISICAL_PROJECT_ID}" --path="/services/remnawave" --format=dotenv-export --silent 2>/dev/null || true)"
|
||||||
export XRAY_CHECKER_SUBSCRIPTION_URL="${XRAY_CHECKER_SUBSCRIPTION_URL:-}"
|
export XRAY_CHECKER_SUBSCRIPTION_URL="${XRAY_CHECKER_SUBSCRIPTION_URL:-}"
|
||||||
|
eval "$(infisical export --domain="${INFISICAL_DOMAIN}" --token="${TOKEN}" --env=prod --projectId="${INFISICAL_PROJECT_ID}" --path="/services/opnsense" --format=dotenv-export --silent 2>/dev/null || true)"
|
||||||
|
export OPNSENSE_API_KEY="${OPNSENSE_API_KEY:-}"
|
||||||
|
export OPNSENSE_API_SECRET="${OPNSENSE_API_SECRET:-}"
|
||||||
;;
|
;;
|
||||||
rus-central)
|
rus-central)
|
||||||
input_email="${INPUT_LETSENCRYPT_EMAIL:-}"
|
input_email="${INPUT_LETSENCRYPT_EMAIL:-}"
|
||||||
|
|||||||
@@ -19,6 +19,11 @@ vmagent_central_scrape_skip:
|
|||||||
cadvisor_image: gcr.io/cadvisor/cadvisor:v0.49.2
|
cadvisor_image: gcr.io/cadvisor/cadvisor:v0.49.2
|
||||||
xray_checker_image: kutovoys/xray-checker:v1.3.1
|
xray_checker_image: kutovoys/xray-checker:v1.3.1
|
||||||
xray_checker_subscription_url: "{{ lookup('env', 'XRAY_CHECKER_SUBSCRIPTION_URL', default='') }}"
|
xray_checker_subscription_url: "{{ lookup('env', 'XRAY_CHECKER_SUBSCRIPTION_URL', default='') }}"
|
||||||
|
opnsense_exporter_image: "python:3.12-alpine"
|
||||||
|
opnsense_api_url: "https://192.168.1.1:4433"
|
||||||
|
opnsense_api_key: "{{ lookup('env', 'OPNSENSE_API_KEY', default='') }}"
|
||||||
|
opnsense_api_secret: "{{ lookup('env', 'OPNSENSE_API_SECRET', default='') }}"
|
||||||
|
opnsense_exporter_enabled: "{{ (opnsense_api_key | length > 0) and (opnsense_api_secret | length > 0) }}"
|
||||||
vmagent_cluster: homelab
|
vmagent_cluster: homelab
|
||||||
# cAdvisor этих хостов слушает :9101. core-mon, core-net и ger-nod-02 пишут его своим агентом.
|
# cAdvisor этих хостов слушает :9101. core-mon, core-net и ger-nod-02 пишут его своим агентом.
|
||||||
monitoring_cadvisor_port: 9101
|
monitoring_cadvisor_port: 9101
|
||||||
|
|||||||
@@ -0,0 +1,237 @@
|
|||||||
|
{
|
||||||
|
"annotations": {
|
||||||
|
"list": [
|
||||||
|
{
|
||||||
|
"builtIn": 1,
|
||||||
|
"datasource": {
|
||||||
|
"type": "grafana",
|
||||||
|
"uid": "-- Grafana --"
|
||||||
|
},
|
||||||
|
"enable": true,
|
||||||
|
"hide": true,
|
||||||
|
"iconColor": "rgba(0, 211, 255, 1)",
|
||||||
|
"name": "Annotations & Alerts",
|
||||||
|
"type": "dashboard"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"editable": true,
|
||||||
|
"fiscalYearStartMonth": 0,
|
||||||
|
"graphTooltip": 1,
|
||||||
|
"id": null,
|
||||||
|
"links": [],
|
||||||
|
"panels": [
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"mappings": [
|
||||||
|
{ "options": { "0": { "color": "red", "index": 1, "text": "Offline" }, "1": { "color": "green", "index": 0, "text": "Online" } }, "type": "value" }
|
||||||
|
],
|
||||||
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] }
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": { "h": 4, "w": 4, "x": 0, "y": 0 },
|
||||||
|
"id": 1,
|
||||||
|
"options": { "colorMode": "background", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" },
|
||||||
|
"targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_gateway_status{gateway=\"NORCOM_DHCP\"}", "refId": "A" }],
|
||||||
|
"title": "NorCom (Primary)",
|
||||||
|
"type": "stat"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"mappings": [
|
||||||
|
{ "options": { "0": { "color": "red", "index": 1, "text": "Offline" }, "1": { "color": "green", "index": 0, "text": "Online" } }, "type": "value" }
|
||||||
|
],
|
||||||
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] }
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": { "h": 4, "w": 4, "x": 4, "y": 0 },
|
||||||
|
"id": 2,
|
||||||
|
"options": { "colorMode": "background", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" },
|
||||||
|
"targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_gateway_status{gateway=\"WAN1\"}", "refId": "A" }],
|
||||||
|
"title": "MTC (WAN1)",
|
||||||
|
"type": "stat"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"mappings": [
|
||||||
|
{ "options": { "0": { "color": "blue", "index": 1, "text": "Offline (sched)" }, "1": { "color": "green", "index": 0, "text": "Online" } }, "type": "value" }
|
||||||
|
],
|
||||||
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "blue", "value": null }, { "color": "green", "value": 1 }] }
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": { "h": 4, "w": 4, "x": 8, "y": 0 },
|
||||||
|
"id": 3,
|
||||||
|
"options": { "colorMode": "background", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" },
|
||||||
|
"targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_gateway_status{gateway=\"WAN_0\"}", "refId": "A" }],
|
||||||
|
"title": "Iterika (WAN_0)",
|
||||||
|
"type": "stat"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"color": { "mode": "fixed", "fixedColor": "super-light-blue" }
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": { "h": 4, "w": 4, "x": 12, "y": 0 },
|
||||||
|
"id": 4,
|
||||||
|
"options": { "colorMode": "value", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "name" },
|
||||||
|
"targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_gateway_default == 1", "legendFormat": "{{gateway}} ({{descr}})", "refId": "A" }],
|
||||||
|
"title": "Active Default GW",
|
||||||
|
"type": "stat"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": { "unit": "short" },
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": { "h": 4, "w": 4, "x": 16, "y": 0 },
|
||||||
|
"id": 5,
|
||||||
|
"options": { "colorMode": "value", "graphMode": "area", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" },
|
||||||
|
"targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_pf_states_current", "refId": "A" }],
|
||||||
|
"title": "pf Active States",
|
||||||
|
"type": "stat"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "percent",
|
||||||
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "orange", "value": 75 }, { "color": "red", "value": 90 }] }
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": { "h": 4, "w": 4, "x": 20, "y": 0 },
|
||||||
|
"id": 6,
|
||||||
|
"options": { "colorMode": "value", "graphMode": "area", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" },
|
||||||
|
"targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "(opnsense_memory_used_bytes / opnsense_memory_total_bytes) * 100", "refId": "A" }],
|
||||||
|
"title": "RAM Usage",
|
||||||
|
"type": "stat"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"mappings": [
|
||||||
|
{ "options": { "0": { "color": "red", "index": 1, "text": "Offline" }, "1": { "color": "green", "index": 0, "text": "Online" } }, "type": "value" }
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": { "h": 5, "w": 24, "x": 0, "y": 4 },
|
||||||
|
"id": 7,
|
||||||
|
"options": { "alignValue": "left", "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "mergeValues": true, "rowHeight": 0.85, "showValue": "auto" },
|
||||||
|
"targets": [
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_gateway_status{gateway=~\"NORCOM_DHCP|WAN1|WAN_0\"}", "legendFormat": "{{gateway}} ({{descr}})", "refId": "A" },
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_gateway_default{gateway=~\"NORCOM_DHCP|WAN1|WAN_0\"}", "legendFormat": "default: {{gateway}}", "refId": "B" }
|
||||||
|
],
|
||||||
|
"title": "Gateway Availability & Active Default Route Timeline",
|
||||||
|
"type": "state-timeline"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": { "unit": "ms", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10 } },
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": { "h": 7, "w": 12, "x": 0, "y": 9 },
|
||||||
|
"id": 8,
|
||||||
|
"options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": ["mean", "max", "lastNotNull"] }, "tooltip": { "mode": "multi" } },
|
||||||
|
"targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_gateway_delay_seconds{gateway=~\"NORCOM_DHCP|WAN1|WAN_0\"} * 1000", "legendFormat": "{{gateway}} ({{descr}} -> {{monitor}})", "refId": "A" }],
|
||||||
|
"title": "Gateway Latency (RTT, dpinger)",
|
||||||
|
"type": "timeseries"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": { "unit": "percent", "min": 0, "max": 100, "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 15 } },
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": { "h": 7, "w": 12, "x": 12, "y": 9 },
|
||||||
|
"id": 9,
|
||||||
|
"options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": ["mean", "max", "lastNotNull"] }, "tooltip": { "mode": "multi" } },
|
||||||
|
"targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_gateway_loss_ratio{gateway=~\"NORCOM_DHCP|WAN1|WAN_0\"} * 100", "legendFormat": "{{gateway}} ({{descr}})", "refId": "A" }],
|
||||||
|
"title": "Gateway Packet Loss (%)",
|
||||||
|
"type": "timeseries"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": { "unit": "bps", "custom": { "drawStyle": "line", "lineWidth": 1.5, "fillOpacity": 15 } },
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": { "h": 7, "w": 12, "x": 0, "y": 16 },
|
||||||
|
"id": 10,
|
||||||
|
"options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": ["mean", "max", "lastNotNull"] }, "tooltip": { "mode": "multi" } },
|
||||||
|
"targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "rate(opnsense_interface_rx_bytes_total{device=~\"igc0|igc1|igc2|proxytun2socks0|vtnet0\"}[1m]) * 8", "legendFormat": "RX {{descr}} ({{device}})", "refId": "A" }],
|
||||||
|
"title": "Interface Download (RX)",
|
||||||
|
"type": "timeseries"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": { "unit": "bps", "custom": { "drawStyle": "line", "lineWidth": 1.5, "fillOpacity": 15 } },
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": { "h": 7, "w": 12, "x": 12, "y": 16 },
|
||||||
|
"id": 11,
|
||||||
|
"options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": ["mean", "max", "lastNotNull"] }, "tooltip": { "mode": "multi" } },
|
||||||
|
"targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "rate(opnsense_interface_tx_bytes_total{device=~\"igc0|igc1|igc2|proxytun2socks0|vtnet0\"}[1m]) * 8", "legendFormat": "TX {{descr}} ({{device}})", "refId": "A" }],
|
||||||
|
"title": "Interface Upload (TX)",
|
||||||
|
"type": "timeseries"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": { "unit": "pps", "custom": { "drawStyle": "line", "lineWidth": 1.5, "fillOpacity": 10 } },
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": { "h": 6, "w": 12, "x": 0, "y": 23 },
|
||||||
|
"id": 12,
|
||||||
|
"options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi" } },
|
||||||
|
"targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "rate(opnsense_interface_rx_errors_total{device=~\"igc0|igc1|igc2|vtnet0\"}[5m]) + rate(opnsense_interface_tx_errors_total{device=~\"igc0|igc1|igc2|vtnet0\"}[5m]) + rate(opnsense_interface_dropped_packets_total{device=~\"igc0|igc1|igc2|vtnet0\"}[5m])", "legendFormat": "{{descr}} ({{device}})", "refId": "A" }],
|
||||||
|
"title": "Interface Errors & Drops",
|
||||||
|
"type": "timeseries"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"mappings": [
|
||||||
|
{ "options": { "0": { "color": "red", "index": 1, "text": "Stopped" }, "1": { "color": "green", "index": 0, "text": "Running" } }, "type": "value" }
|
||||||
|
],
|
||||||
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] }
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": { "h": 6, "w": 12, "x": 12, "y": 23 },
|
||||||
|
"id": 13,
|
||||||
|
"options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "value_and_name" },
|
||||||
|
"targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_service_running{name=~\"unbound|xray|netbird|dhcpd|pf|node_exporter\"}", "legendFormat": "{{name}}", "refId": "A" }],
|
||||||
|
"title": "OPNsense Services",
|
||||||
|
"type": "stat"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"refresh": "15s",
|
||||||
|
"schemaVersion": 39,
|
||||||
|
"tags": ["opnsense", "wan", "network"],
|
||||||
|
"templating": { "list": [] },
|
||||||
|
"time": { "from": "now-6h", "to": "now" },
|
||||||
|
"timepicker": {},
|
||||||
|
"timezone": "",
|
||||||
|
"title": "OPNsense — Multi-WAN & Router",
|
||||||
|
"uid": "opnsense-wan",
|
||||||
|
"version": 1
|
||||||
|
}
|
||||||
|
|
||||||
@@ -0,0 +1,292 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Lightweight Prometheus exporter for OPNsense gateways, interfaces, pf, and system metrics."""
|
||||||
|
|
||||||
|
import base64
|
||||||
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import ssl
|
||||||
|
import sys
|
||||||
|
import urllib.error
|
||||||
|
import urllib.request
|
||||||
|
|
||||||
|
|
||||||
|
API_URL = os.environ.get("OPNSENSE_API_URL", "https://192.168.1.1:4433").rstrip("/")
|
||||||
|
API_KEY = os.environ.get("OPNSENSE_API_KEY", "")
|
||||||
|
API_SECRET = os.environ.get("OPNSENSE_API_SECRET", "")
|
||||||
|
VERIFY_TLS = os.environ.get("OPNSENSE_VERIFY_TLS", "false").lower() in ("1", "true", "yes")
|
||||||
|
LISTEN_PORT = int(os.environ.get("OPNSENSE_EXPORTER_PORT", "9105"))
|
||||||
|
TIMEOUT = float(os.environ.get("OPNSENSE_API_TIMEOUT", "5"))
|
||||||
|
|
||||||
|
|
||||||
|
def _ssl_context() -> ssl.SSLContext:
|
||||||
|
ctx = ssl.create_default_context()
|
||||||
|
if not VERIFY_TLS:
|
||||||
|
ctx.check_hostname = False
|
||||||
|
ctx.verify_mode = ssl.CERT_NONE
|
||||||
|
return ctx
|
||||||
|
|
||||||
|
|
||||||
|
def _auth_header() -> str:
|
||||||
|
raw = f"{API_KEY}:{API_SECRET}".encode("utf-8")
|
||||||
|
return "Basic " + base64.b64encode(raw).decode("ascii")
|
||||||
|
|
||||||
|
|
||||||
|
def _api_get(path: str):
|
||||||
|
req = urllib.request.Request(
|
||||||
|
f"{API_URL}{path}",
|
||||||
|
headers={
|
||||||
|
"Authorization": _auth_header(),
|
||||||
|
"Accept": "application/json",
|
||||||
|
},
|
||||||
|
method="GET",
|
||||||
|
)
|
||||||
|
with urllib.request.urlopen(req, context=_ssl_context(), timeout=TIMEOUT) as resp:
|
||||||
|
return json.loads(resp.read().decode("utf-8"))
|
||||||
|
|
||||||
|
|
||||||
|
def _esc(val: str) -> str:
|
||||||
|
return str(val).replace("\\", "\\\\").replace('"', '\\"').replace("\n", "\\n")
|
||||||
|
|
||||||
|
|
||||||
|
def _labels(mapping: dict[str, str]) -> str:
|
||||||
|
parts = [f'{k}="{_esc(v)}"' for k, v in mapping.items()]
|
||||||
|
return "{" + ",".join(parts) + "}"
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_ms(text: str):
|
||||||
|
m = re.match(r"^\s*([0-9]+(?:\.[0-9]+)?)\s*ms\s*$", str(text or ""))
|
||||||
|
return float(m.group(1)) / 1000.0 if m else None
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_pct(text: str):
|
||||||
|
m = re.match(r"^\s*([0-9]+(?:\.[0-9]+)?)\s*%\s*$", str(text or ""))
|
||||||
|
return float(m.group(1)) / 100.0 if m else None
|
||||||
|
|
||||||
|
|
||||||
|
def collect_metrics() -> str:
|
||||||
|
lines: list[str] = []
|
||||||
|
|
||||||
|
gateways_data = _api_get("/api/routing/settings/searchGateway").get("rows", [])
|
||||||
|
routes_data = _api_get("/api/diagnostics/interface/getRoutes")
|
||||||
|
ifaces_data = _api_get("/api/interfaces/overview/interfacesInfo").get("rows", [])
|
||||||
|
ifstats_data = _api_get("/api/diagnostics/interface/getInterfaceStatistics").get("statistics", {})
|
||||||
|
pf_data = _api_get("/api/diagnostics/firewall/pfStatistics/info").get("info", {})
|
||||||
|
sys_res = _api_get("/api/diagnostics/system/systemResources")
|
||||||
|
sys_time = _api_get("/api/diagnostics/system/systemTime")
|
||||||
|
services_data = _api_get("/api/core/service/search").get("rows", [])
|
||||||
|
|
||||||
|
# Active IPv4 default route from kernel routing table
|
||||||
|
active_default_gw = None
|
||||||
|
active_default_if = None
|
||||||
|
for r in routes_data:
|
||||||
|
if r.get("proto") == "ipv4" and r.get("destination") == "default":
|
||||||
|
active_default_gw = r.get("gateway")
|
||||||
|
active_default_if = r.get("netif")
|
||||||
|
break
|
||||||
|
|
||||||
|
# Gateways (dpinger)
|
||||||
|
lines.extend([
|
||||||
|
"# HELP opnsense_gateway_status Gateway online status (1=Online, 0=Offline)",
|
||||||
|
"# TYPE opnsense_gateway_status gauge",
|
||||||
|
"# HELP opnsense_gateway_configured_default Configured default gateway flag",
|
||||||
|
"# TYPE opnsense_gateway_configured_default gauge",
|
||||||
|
"# HELP opnsense_gateway_default Active kernel default route flag (1=active default)",
|
||||||
|
"# TYPE opnsense_gateway_default gauge",
|
||||||
|
"# HELP opnsense_gateway_delay_seconds Gateway monitor RTT in seconds",
|
||||||
|
"# TYPE opnsense_gateway_delay_seconds gauge",
|
||||||
|
"# HELP opnsense_gateway_stddev_seconds Gateway monitor RTT stddev in seconds",
|
||||||
|
"# TYPE opnsense_gateway_stddev_seconds gauge",
|
||||||
|
"# HELP opnsense_gateway_loss_ratio Gateway monitor packet loss ratio (0.0-1.0)",
|
||||||
|
"# TYPE opnsense_gateway_loss_ratio gauge",
|
||||||
|
])
|
||||||
|
for gw in gateways_data:
|
||||||
|
name = gw.get("name", "")
|
||||||
|
descr = gw.get("interface_descr") or gw.get("descr") or name
|
||||||
|
iface = gw.get("interface", "")
|
||||||
|
device = gw.get("if", "")
|
||||||
|
monitor = gw.get("monitor", "")
|
||||||
|
gw_addr = gw.get("gateway", "")
|
||||||
|
lbl = _labels({
|
||||||
|
"gateway": name,
|
||||||
|
"descr": descr,
|
||||||
|
"interface": iface,
|
||||||
|
"device": device,
|
||||||
|
"monitor": monitor,
|
||||||
|
})
|
||||||
|
online = 1 if gw.get("status") == "Online" else 0
|
||||||
|
lines.append(f"opnsense_gateway_status{lbl} {online}")
|
||||||
|
lines.append(f"opnsense_gateway_configured_default{lbl} {1 if gw.get('defaultgw') else 0}")
|
||||||
|
|
||||||
|
is_active_default = 1 if (
|
||||||
|
active_default_gw and active_default_if
|
||||||
|
and gw_addr == active_default_gw and device == active_default_if
|
||||||
|
) else 0
|
||||||
|
lines.append(f"opnsense_gateway_default{lbl} {is_active_default}")
|
||||||
|
|
||||||
|
loss = _parse_pct(gw.get("loss"))
|
||||||
|
if loss is not None:
|
||||||
|
lines.append(f"opnsense_gateway_loss_ratio{lbl} {loss}")
|
||||||
|
|
||||||
|
if online:
|
||||||
|
delay = _parse_ms(gw.get("delay"))
|
||||||
|
stddev = _parse_ms(gw.get("stddev"))
|
||||||
|
if delay is not None:
|
||||||
|
lines.append(f"opnsense_gateway_delay_seconds{lbl} {delay}")
|
||||||
|
if stddev is not None:
|
||||||
|
lines.append(f"opnsense_gateway_stddev_seconds{lbl} {stddev}")
|
||||||
|
|
||||||
|
# Interfaces status & traffic counters
|
||||||
|
device_meta: dict[str, tuple[str, str]] = {}
|
||||||
|
lines.extend([
|
||||||
|
"# HELP opnsense_interface_up Interface operational status (1=up, 0=down)",
|
||||||
|
"# TYPE opnsense_interface_up gauge",
|
||||||
|
])
|
||||||
|
for iface in ifaces_data:
|
||||||
|
ident = iface.get("identifier") or ""
|
||||||
|
device = iface.get("device") or ""
|
||||||
|
descr = iface.get("description") or ident or device
|
||||||
|
if not ident or not device:
|
||||||
|
continue
|
||||||
|
device_meta[device] = (ident, descr)
|
||||||
|
up = 1 if iface.get("status") == "up" else 0
|
||||||
|
lbl = _labels({"interface": ident, "device": device, "descr": descr})
|
||||||
|
lines.append(f"opnsense_interface_up{lbl} {up}")
|
||||||
|
|
||||||
|
lines.extend([
|
||||||
|
"# HELP opnsense_interface_rx_bytes_total Received bytes on interface",
|
||||||
|
"# TYPE opnsense_interface_rx_bytes_total counter",
|
||||||
|
"# HELP opnsense_interface_tx_bytes_total Transmitted bytes on interface",
|
||||||
|
"# TYPE opnsense_interface_tx_bytes_total counter",
|
||||||
|
"# HELP opnsense_interface_rx_packets_total Received packets on interface",
|
||||||
|
"# TYPE opnsense_interface_rx_packets_total counter",
|
||||||
|
"# HELP opnsense_interface_tx_packets_total Transmitted packets on interface",
|
||||||
|
"# TYPE opnsense_interface_tx_packets_total counter",
|
||||||
|
"# HELP opnsense_interface_rx_errors_total Receive errors on interface",
|
||||||
|
"# TYPE opnsense_interface_rx_errors_total counter",
|
||||||
|
"# HELP opnsense_interface_tx_errors_total Transmit errors on interface",
|
||||||
|
"# TYPE opnsense_interface_tx_errors_total counter",
|
||||||
|
"# HELP opnsense_interface_dropped_packets_total Dropped packets on interface",
|
||||||
|
"# TYPE opnsense_interface_dropped_packets_total counter",
|
||||||
|
])
|
||||||
|
seen_devices: set[str] = set()
|
||||||
|
for stat in ifstats_data.values():
|
||||||
|
device = stat.get("name")
|
||||||
|
if not device or device not in device_meta or device in seen_devices:
|
||||||
|
continue
|
||||||
|
if stat.get("received-bytes") is None:
|
||||||
|
continue
|
||||||
|
seen_devices.add(device)
|
||||||
|
ident, descr = device_meta[device]
|
||||||
|
lbl = _labels({"interface": ident, "device": device, "descr": descr})
|
||||||
|
lines.append(f"opnsense_interface_rx_bytes_total{lbl} {stat['received-bytes']}")
|
||||||
|
lines.append(f"opnsense_interface_tx_bytes_total{lbl} {stat['sent-bytes']}")
|
||||||
|
lines.append(f"opnsense_interface_rx_packets_total{lbl} {stat['received-packets']}")
|
||||||
|
lines.append(f"opnsense_interface_tx_packets_total{lbl} {stat['sent-packets']}")
|
||||||
|
lines.append(f"opnsense_interface_rx_errors_total{lbl} {stat.get('received-errors') or 0}")
|
||||||
|
lines.append(f"opnsense_interface_tx_errors_total{lbl} {stat.get('send-errors') or 0}")
|
||||||
|
lines.append(f"opnsense_interface_dropped_packets_total{lbl} {stat.get('dropped-packets') or 0}")
|
||||||
|
|
||||||
|
# pf state table
|
||||||
|
state_table = pf_data.get("state-table", {})
|
||||||
|
pf_current = state_table.get("current-entries", {}).get("total")
|
||||||
|
pf_inserts = state_table.get("inserts", {}).get("total")
|
||||||
|
pf_removals = state_table.get("removals", {}).get("total")
|
||||||
|
if pf_current is not None:
|
||||||
|
lines.extend([
|
||||||
|
"# HELP opnsense_pf_states_current Current active pf states",
|
||||||
|
"# TYPE opnsense_pf_states_current gauge",
|
||||||
|
f"opnsense_pf_states_current {pf_current}",
|
||||||
|
])
|
||||||
|
if pf_inserts is not None and pf_removals is not None:
|
||||||
|
lines.extend([
|
||||||
|
"# HELP opnsense_pf_state_inserts_total Total pf state insertions",
|
||||||
|
"# TYPE opnsense_pf_state_inserts_total counter",
|
||||||
|
f"opnsense_pf_state_inserts_total {pf_inserts}",
|
||||||
|
"# HELP opnsense_pf_state_removals_total Total pf state removals",
|
||||||
|
"# TYPE opnsense_pf_state_removals_total counter",
|
||||||
|
f"opnsense_pf_state_removals_total {pf_removals}",
|
||||||
|
])
|
||||||
|
|
||||||
|
# System memory & load
|
||||||
|
mem = sys_res.get("memory", {})
|
||||||
|
if mem.get("total") is not None and mem.get("used") is not None:
|
||||||
|
lines.extend([
|
||||||
|
"# HELP opnsense_memory_total_bytes Total physical memory in bytes",
|
||||||
|
"# TYPE opnsense_memory_total_bytes gauge",
|
||||||
|
f"opnsense_memory_total_bytes {mem['total']}",
|
||||||
|
"# HELP opnsense_memory_used_bytes Used memory in bytes",
|
||||||
|
"# TYPE opnsense_memory_used_bytes gauge",
|
||||||
|
f"opnsense_memory_used_bytes {mem['used']}",
|
||||||
|
])
|
||||||
|
|
||||||
|
load_parts = [p.strip() for p in str(sys_time.get("loadavg", "")).split(",") if p.strip()]
|
||||||
|
if len(load_parts) == 3:
|
||||||
|
lines.extend([
|
||||||
|
"# HELP opnsense_load1 1-minute system load average",
|
||||||
|
"# TYPE opnsense_load1 gauge",
|
||||||
|
f"opnsense_load1 {float(load_parts[0])}",
|
||||||
|
"# HELP opnsense_load5 5-minute system load average",
|
||||||
|
"# TYPE opnsense_load5 gauge",
|
||||||
|
f"opnsense_load5 {float(load_parts[1])}",
|
||||||
|
"# HELP opnsense_load15 15-minute system load average",
|
||||||
|
"# TYPE opnsense_load15 gauge",
|
||||||
|
f"opnsense_load15 {float(load_parts[2])}",
|
||||||
|
])
|
||||||
|
|
||||||
|
# Services
|
||||||
|
lines.extend([
|
||||||
|
"# HELP opnsense_service_running OPNsense service status (1=running, 0=stopped)",
|
||||||
|
"# TYPE opnsense_service_running gauge",
|
||||||
|
])
|
||||||
|
for svc in services_data:
|
||||||
|
lbl = _labels({
|
||||||
|
"service_id": svc.get("id", ""),
|
||||||
|
"name": svc.get("name", ""),
|
||||||
|
})
|
||||||
|
lines.append(f"opnsense_service_running{lbl} {1 if svc.get('running') else 0}")
|
||||||
|
|
||||||
|
return "\n".join(lines) + "\n"
|
||||||
|
|
||||||
|
|
||||||
|
class MetricsHandler(BaseHTTPRequestHandler):
|
||||||
|
def do_GET(self):
|
||||||
|
if self.path == "/health":
|
||||||
|
self.send_response(200)
|
||||||
|
self.send_header("Content-Type", "text/plain")
|
||||||
|
self.end_headers()
|
||||||
|
self.wfile.write(b"ok\n")
|
||||||
|
return
|
||||||
|
if self.path != "/metrics":
|
||||||
|
self.send_response(404)
|
||||||
|
self.end_headers()
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
body = collect_metrics().encode("utf-8")
|
||||||
|
self.send_response(200)
|
||||||
|
self.send_header("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
|
||||||
|
self.end_headers()
|
||||||
|
self.wfile.write(body)
|
||||||
|
except (urllib.error.URLError, TimeoutError, json.JSONDecodeError, KeyError) as exc:
|
||||||
|
msg = f"# error collecting OPNsense metrics: {exc}\n".encode("utf-8")
|
||||||
|
self.send_response(503)
|
||||||
|
self.send_header("Content-Type", "text/plain; charset=utf-8")
|
||||||
|
self.end_headers()
|
||||||
|
self.wfile.write(msg)
|
||||||
|
|
||||||
|
def log_message(self, format, *args):
|
||||||
|
return
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
if not API_KEY or not API_SECRET:
|
||||||
|
print("OPNSENSE_API_KEY and OPNSENSE_API_SECRET must be set", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
server = ThreadingHTTPServer(("0.0.0.0", LISTEN_PORT), MetricsHandler)
|
||||||
|
server.serve_forever()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
|
|
||||||
@@ -18,6 +18,13 @@
|
|||||||
when: xray_checker_env is defined and xray_checker_env is changed
|
when: xray_checker_env is defined and xray_checker_env is changed
|
||||||
changed_when: true
|
changed_when: true
|
||||||
|
|
||||||
|
- name: Пересоздать экспортер OPNsense при обновлении скрипта или окружения
|
||||||
|
ansible.builtin.command: docker compose -f {{ monitoring_dir }}/compose.yaml up -d --force-recreate --no-deps opnsense-exporter
|
||||||
|
when: >
|
||||||
|
(opnsense_exporter_env is defined and opnsense_exporter_env is changed) or
|
||||||
|
(opnsense_exporter_script is defined and opnsense_exporter_script is changed)
|
||||||
|
changed_when: true
|
||||||
|
|
||||||
- name: Дождаться VictoriaMetrics
|
- name: Дождаться VictoriaMetrics
|
||||||
ansible.builtin.uri:
|
ansible.builtin.uri:
|
||||||
url: "http://{{ ansible_host }}:8428/health"
|
url: "http://{{ ansible_host }}:8428/health"
|
||||||
|
|||||||
@@ -21,3 +21,4 @@
|
|||||||
- { path: "{{ monitoring_dir }}/vmagent", owner: "root", group: "root", mode: "0755" }
|
- { path: "{{ monitoring_dir }}/vmagent", owner: "root", group: "root", mode: "0755" }
|
||||||
- { path: "{{ monitoring_dir }}/vmagent/conf.d", owner: "root", group: "root", mode: "0755" }
|
- { path: "{{ monitoring_dir }}/vmagent/conf.d", owner: "root", group: "root", mode: "0755" }
|
||||||
- { path: "{{ monitoring_dir }}/vmalert", owner: "root", group: "root", mode: "0755" }
|
- { path: "{{ monitoring_dir }}/vmalert", owner: "root", group: "root", mode: "0755" }
|
||||||
|
- { path: "{{ monitoring_dir }}/opnsense-exporter", owner: "root", group: "root", mode: "0755" }
|
||||||
@@ -96,3 +96,42 @@
|
|||||||
mode: "0644"
|
mode: "0644"
|
||||||
when: xray_checker_subscription_url | length > 0
|
when: xray_checker_subscription_url | length > 0
|
||||||
|
|
||||||
|
- name: Убрать scrape OPNsense без API-ключей
|
||||||
|
ansible.builtin.file:
|
||||||
|
path: "{{ item }}"
|
||||||
|
state: absent
|
||||||
|
loop:
|
||||||
|
- "{{ monitoring_dir }}/vmagent/conf.d/opnsense.yml"
|
||||||
|
- "{{ monitoring_dir }}/opnsense-exporter.env"
|
||||||
|
when: not (opnsense_exporter_enabled | bool)
|
||||||
|
|
||||||
|
- name: Скрипт экспортера OPNsense
|
||||||
|
ansible.builtin.copy:
|
||||||
|
src: opnsense-exporter/exporter.py
|
||||||
|
dest: "{{ monitoring_dir }}/opnsense-exporter/exporter.py"
|
||||||
|
owner: root
|
||||||
|
group: root
|
||||||
|
mode: "0755"
|
||||||
|
register: opnsense_exporter_script
|
||||||
|
when: opnsense_exporter_enabled | bool
|
||||||
|
|
||||||
|
- name: Файл окружения экспортера OPNsense
|
||||||
|
ansible.builtin.template:
|
||||||
|
src: opnsense-exporter.env.j2
|
||||||
|
dest: "{{ monitoring_dir }}/opnsense-exporter.env"
|
||||||
|
owner: root
|
||||||
|
group: root
|
||||||
|
mode: "0600"
|
||||||
|
no_log: true
|
||||||
|
register: opnsense_exporter_env
|
||||||
|
when: opnsense_exporter_enabled | bool
|
||||||
|
|
||||||
|
- name: Scrape OPNsense
|
||||||
|
ansible.builtin.template:
|
||||||
|
src: vmagent-opnsense.yml.j2
|
||||||
|
dest: "{{ monitoring_dir }}/vmagent/conf.d/opnsense.yml"
|
||||||
|
owner: root
|
||||||
|
group: root
|
||||||
|
mode: "0644"
|
||||||
|
when: opnsense_exporter_enabled | bool
|
||||||
|
|
||||||
@@ -218,3 +218,65 @@ groups:
|
|||||||
annotations:
|
annotations:
|
||||||
summary: "Прокси {{ $labels.name }} не проходит проверку"
|
summary: "Прокси {{ $labels.name }} не проходит проверку"
|
||||||
description: "xray_proxy_status {{ $labels.name }} ({{ $labels.address }}) не равен 1."
|
description: "xray_proxy_status {{ $labels.name }} ({{ $labels.address }}) не равен 1."
|
||||||
|
[% if opnsense_exporter_enabled | bool %]
|
||||||
|
|
||||||
|
- alert: OpnsenseExporterDown
|
||||||
|
expr: up{job="opnsense"} == 0
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Экспортер OPNsense недоступен"
|
||||||
|
description: "Scrape opnsense ({{ $labels.instance }}) возвращает up=0 более 2 минут."
|
||||||
|
|
||||||
|
- alert: OpnsenseMetricsStale
|
||||||
|
expr: |
|
||||||
|
(
|
||||||
|
time() - tlast_over_time(up{job="opnsense"}[30m]) > 180
|
||||||
|
)
|
||||||
|
or absent_over_time(up{job="opnsense",instance="opnsense"}[30m])
|
||||||
|
for: 1m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Нет свежих метрик OPNsense"
|
||||||
|
description: "opnsense-exporter не присылает up дольше 3 минут."
|
||||||
|
|
||||||
|
# Мониторим отвал только постоянных каналов (NorCom и MTC); WAN_0 (Iterika) работает по расписанию.
|
||||||
|
- alert: OpnsenseGatewayDown
|
||||||
|
expr: opnsense_gateway_status{gateway=~"NORCOM_DHCP|WAN1"} == 0
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Провайдер OPNsense {{ $labels.gateway }} ({{ $labels.descr }}) недоступен"
|
||||||
|
description: "Шлюз {{ $labels.gateway }} (монитор {{ $labels.monitor }}) в статусе Offline более 2 минут."
|
||||||
|
|
||||||
|
- alert: OpnsenseGatewayPacketLoss
|
||||||
|
expr: opnsense_gateway_loss_ratio{gateway=~"NORCOM_DHCP|WAN1"} > 0.1
|
||||||
|
for: 3m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Потери пакетов на {{ $labels.gateway }} ({{ $labels.descr }})"
|
||||||
|
description: "Потери на шлюзе {{ $labels.gateway }} превышают 10% более 3 минут."
|
||||||
|
|
||||||
|
- alert: OpnsenseGatewayHighLatency
|
||||||
|
expr: opnsense_gateway_delay_seconds{gateway=~"NORCOM_DHCP|WAN1"} > 0.25
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Высокая задержка на {{ $labels.gateway }} ({{ $labels.descr }})"
|
||||||
|
description: "RTT до {{ $labels.monitor }} через {{ $labels.gateway }} превышает 250 ms более 5 минут."
|
||||||
|
|
||||||
|
# Уведомление при переключении default-маршрута на резерв (включая WAN_0 / Iterika).
|
||||||
|
- alert: OpnsenseFailoverActive
|
||||||
|
expr: opnsense_gateway_default{gateway!="NORCOM_DHCP"} == 1
|
||||||
|
for: 30s
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "OPNsense переключился на резерв: {{ $labels.gateway }} ({{ $labels.descr }})"
|
||||||
|
description: "Активный default-маршрут ушёл с NORCOM_DHCP на {{ $labels.gateway }} ({{ $labels.descr }}, {{ $labels.device }})."
|
||||||
|
[% endif %]
|
||||||
@@ -103,6 +103,20 @@ services:
|
|||||||
ports:
|
ports:
|
||||||
- "127.0.0.1:2112:2112"
|
- "127.0.0.1:2112:2112"
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
{% if opnsense_exporter_enabled | bool %}
|
||||||
|
|
||||||
|
opnsense-exporter:
|
||||||
|
image: {{ opnsense_exporter_image }}
|
||||||
|
container_name: monitoring-opnsense-exporter
|
||||||
|
restart: unless-stopped
|
||||||
|
command: ["python3", "/app/exporter.py"]
|
||||||
|
env_file:
|
||||||
|
- ./opnsense-exporter.env
|
||||||
|
volumes:
|
||||||
|
- ./opnsense-exporter/exporter.py:/app/exporter.py:ro
|
||||||
|
ports:
|
||||||
|
- "127.0.0.1:9105:9105"
|
||||||
|
{% endif %}
|
||||||
|
|
||||||
vmagent:
|
vmagent:
|
||||||
image: {{ vmagent_image }}
|
image: {{ vmagent_image }}
|
||||||
|
|||||||
@@ -0,0 +1,6 @@
|
|||||||
|
OPNSENSE_API_URL={{ opnsense_api_url }}
|
||||||
|
OPNSENSE_API_KEY={{ opnsense_api_key }}
|
||||||
|
OPNSENSE_API_SECRET={{ opnsense_api_secret }}
|
||||||
|
OPNSENSE_VERIFY_TLS=false
|
||||||
|
OPNSENSE_EXPORTER_PORT=9105
|
||||||
|
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
- job_name: opnsense
|
||||||
|
scrape_interval: 15s
|
||||||
|
static_configs:
|
||||||
|
- targets: ["opnsense-exporter:9105"]
|
||||||
|
labels:
|
||||||
|
instance: "opnsense"
|
||||||
|
cluster: "{{ vmagent_cluster }}"
|
||||||
|
host: "opnsense"
|
||||||
|
|
||||||
Reference in new issue
Block a user