Compare commits

...
Author SHA1 Message Date
karpenok c064d518d1 refactor(agent_vault): move Gitea service registration to core-secrets playbook
ansible-lint / lint (push) Successful in 3s
ansible-deploy / deploy (core-secrets, ) (push) Successful in 21s
ansible-deploy / plan (push) Successful in 2s
2026-11-17 16:18:23 +07:00
karpenok 195830da21 fix(agent_vault): use token header prefix for Gitea API
ansible-deploy / plan (push) Successful in 2s
ansible-deploy / deploy (push) Skipped
ansible-lint / lint (push) Successful in 3s
2026-11-17 16:12:21 +07:00
karpenok 07e98ce7aa feat(agent_vault): register Grafana and Gitea API services
ansible-deploy / plan (push) Successful in 2s
ansible-lint / lint (push) Successful in 3s
ansible-deploy / deploy (core-mon, ) (push) Successful in 31s
2026-11-17 16:10:06 +07:00
karpenok 53626e9271 fix(monitoring): split gateway availability and default route timelines
ansible-deploy / plan (push) Successful in 2s
ansible-lint / lint (push) Successful in 3s
ansible-deploy / deploy (core-mon, ) (push) Successful in 22s
2026-11-17 16:03:11 +07:00
karpenok 376006e809 feat(monitoring): add OPNsense CPU, disk, load, uptime metrics and alerts
ansible-deploy / plan (push) Successful in 2s
ansible-lint / lint (push) Successful in 3s
ansible-deploy / deploy (core-mon, ) (push) Successful in 33s
2026-11-17 16:00:19 +07:00
karpenok 28ae06834b feat(monitoring): add OPNsense multi-WAN exporter, alerts, and Grafana dashboard
ansible-lint / lint (push) Successful in 3s
ansible-deploy / plan (push) Successful in 1s
ansible-deploy / deploy (core-mon, ) (push) Successful in 43s
2026-11-17 15:53:22 +07:00
13 changed files with 884 additions and 0 deletions

No files matched your search

+3
View File
@@ -89,6 +89,9 @@ case "$machine" in
fi fi
eval "$(infisical export --domain="${INFISICAL_DOMAIN}" --token="${TOKEN}" --env=prod --projectId="${INFISICAL_PROJECT_ID}" --path="/services/remnawave" --format=dotenv-export --silent 2>/dev/null || true)" eval "$(infisical export --domain="${INFISICAL_DOMAIN}" --token="${TOKEN}" --env=prod --projectId="${INFISICAL_PROJECT_ID}" --path="/services/remnawave" --format=dotenv-export --silent 2>/dev/null || true)"
export XRAY_CHECKER_SUBSCRIPTION_URL="${XRAY_CHECKER_SUBSCRIPTION_URL:-}" export XRAY_CHECKER_SUBSCRIPTION_URL="${XRAY_CHECKER_SUBSCRIPTION_URL:-}"
eval "$(infisical export --domain="${INFISICAL_DOMAIN}" --token="${TOKEN}" --env=prod --projectId="${INFISICAL_PROJECT_ID}" --path="/services/opnsense" --format=dotenv-export --silent 2>/dev/null || true)"
export OPNSENSE_API_KEY="${OPNSENSE_API_KEY:-}"
export OPNSENSE_API_SECRET="${OPNSENSE_API_SECRET:-}"
;; ;;
rus-central) rus-central)
input_email="${INPUT_LETSENCRYPT_EMAIL:-}" input_email="${INPUT_LETSENCRYPT_EMAIL:-}"
+16
View File
@@ -5,3 +5,19 @@
roles: roles:
- role: core-mon/monitoring - role: core-mon/monitoring
tags: [monitoring] tags: [monitoring]
post_tasks:
- name: Зарегистрировать Grafana в Agent Vault
ansible.builtin.include_role:
name: agent_vault_service
vars:
agent_vault_service_name: grafana
agent_vault_service_description: >-
Grafana (graf.gunihome.ru): чтение метрик VictoriaMetrics через
datasource proxy/ds query, дашбордов и алертов. Read-only доступ (Viewer).
agent_vault_service_host: "graf.gunihome.ru:443"
agent_vault_service_paths: ["/api"]
agent_vault_service_methods: [GET, POST]
agent_vault_service_credential_type: bearer
agent_vault_secret_path: /services/grafana
agent_vault_secret_key: GRAFANA_API_TOKEN
tags: [agent_vault, monitoring]
+19
View File
@@ -9,3 +9,22 @@
tags: [vaultwarden] tags: [vaultwarden]
- role: core-secrets/cadvisor - role: core-secrets/cadvisor
tags: [cadvisor] tags: [cadvisor]
post_tasks:
- name: Зарегистрировать Gitea в Agent Vault
ansible.builtin.include_role:
name: agent_vault_service
apply:
tags: [agent_vault, gitea]
vars:
agent_vault_service_name: gitea
agent_vault_service_description: >-
Gitea (gitea.gunihome.ru): чтение репозиториев, коммитов и логов
CI/Actions. Read-only доступ (GET) через токен.
agent_vault_service_host: "gitea.gunihome.ru:443"
agent_vault_service_paths: ["/api"]
agent_vault_service_methods: [GET]
agent_vault_service_credential_type: bearer
agent_vault_service_header_prefix: token
agent_vault_secret_path: /ci-cd
agent_vault_secret_key: TF_STATE_TOKEN
tags: [agent_vault, gitea]
@@ -19,6 +19,11 @@ vmagent_central_scrape_skip:
cadvisor_image: gcr.io/cadvisor/cadvisor:v0.49.2 cadvisor_image: gcr.io/cadvisor/cadvisor:v0.49.2
xray_checker_image: kutovoys/xray-checker:v1.3.1 xray_checker_image: kutovoys/xray-checker:v1.3.1
xray_checker_subscription_url: "{{ lookup('env', 'XRAY_CHECKER_SUBSCRIPTION_URL', default='') }}" xray_checker_subscription_url: "{{ lookup('env', 'XRAY_CHECKER_SUBSCRIPTION_URL', default='') }}"
opnsense_exporter_image: "python:3.12-alpine"
opnsense_api_url: "https://192.168.1.1:4433"
opnsense_api_key: "{{ lookup('env', 'OPNSENSE_API_KEY', default='') }}"
opnsense_api_secret: "{{ lookup('env', 'OPNSENSE_API_SECRET', default='') }}"
opnsense_exporter_enabled: "{{ (opnsense_api_key | length > 0) and (opnsense_api_secret | length > 0) }}"
vmagent_cluster: homelab vmagent_cluster: homelab
# cAdvisor этих хостов слушает :9101. core-mon, core-net и ger-nod-02 пишут его своим агентом. # cAdvisor этих хостов слушает :9101. core-mon, core-net и ger-nod-02 пишут его своим агентом.
monitoring_cadvisor_port: 9101 monitoring_cadvisor_port: 9101
@@ -0,0 +1,329 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
"id": null,
"links": [],
"panels": [
{
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
"fieldConfig": {
"defaults": {
"mappings": [
{ "options": { "0": { "color": "red", "index": 1, "text": "Offline" }, "1": { "color": "green", "index": 0, "text": "Online" } }, "type": "value" }
],
"thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] }
},
"overrides": []
},
"gridPos": { "h": 4, "w": 4, "x": 0, "y": 0 },
"id": 1,
"options": { "colorMode": "background", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" },
"targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_gateway_status{gateway=\"NORCOM_DHCP\"}", "refId": "A" }],
"title": "NorCom (Primary)",
"type": "stat"
},
{
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
"fieldConfig": {
"defaults": {
"mappings": [
{ "options": { "0": { "color": "red", "index": 1, "text": "Offline" }, "1": { "color": "green", "index": 0, "text": "Online" } }, "type": "value" }
],
"thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] }
},
"overrides": []
},
"gridPos": { "h": 4, "w": 4, "x": 4, "y": 0 },
"id": 2,
"options": { "colorMode": "background", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" },
"targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_gateway_status{gateway=\"WAN1\"}", "refId": "A" }],
"title": "MTC (WAN1)",
"type": "stat"
},
{
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
"fieldConfig": {
"defaults": {
"mappings": [
{ "options": { "0": { "color": "blue", "index": 1, "text": "Offline (sched)" }, "1": { "color": "green", "index": 0, "text": "Online" } }, "type": "value" }
],
"thresholds": { "mode": "absolute", "steps": [{ "color": "blue", "value": null }, { "color": "green", "value": 1 }] }
},
"overrides": []
},
"gridPos": { "h": 4, "w": 4, "x": 8, "y": 0 },
"id": 3,
"options": { "colorMode": "background", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" },
"targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_gateway_status{gateway=\"WAN_0\"}", "refId": "A" }],
"title": "Iterika (WAN_0)",
"type": "stat"
},
{
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
"fieldConfig": {
"defaults": {
"color": { "mode": "fixed", "fixedColor": "super-light-blue" }
},
"overrides": []
},
"gridPos": { "h": 4, "w": 4, "x": 12, "y": 0 },
"id": 4,
"options": { "colorMode": "value", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "name" },
"targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_gateway_default == 1", "legendFormat": "{{gateway}} ({{descr}})", "refId": "A" }],
"title": "Active Default GW",
"type": "stat"
},
{
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
"fieldConfig": {
"defaults": {
"unit": "short",
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "orange", "value": 50000 }, { "color": "red", "value": 200000 }] }
},
"overrides": []
},
"gridPos": { "h": 4, "w": 4, "x": 16, "y": 0 },
"id": 5,
"options": { "colorMode": "value", "graphMode": "area", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" },
"targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_pf_states_current", "refId": "A" }],
"title": "pf Active States",
"type": "stat"
},
{
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
"fieldConfig": {
"defaults": {
"unit": "percent",
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "orange", "value": 75 }, { "color": "red", "value": 90 }] }
},
"overrides": []
},
"gridPos": { "h": 4, "w": 4, "x": 20, "y": 0 },
"id": 6,
"options": { "colorMode": "value", "graphMode": "area", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" },
"targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "(opnsense_memory_used_bytes / opnsense_memory_total_bytes) * 100", "refId": "A" }],
"title": "RAM Usage",
"type": "stat"
},
{
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
"fieldConfig": {
"defaults": {
"mappings": [
{ "options": { "0": { "color": "red", "index": 1, "text": "Offline" }, "1": { "color": "green", "index": 0, "text": "Online" } }, "type": "value" }
]
},
"overrides": [
{
"matcher": { "id": "byName", "options": "WAN_0 (Iterika)" },
"properties": [
{
"id": "mappings",
"value": [
{ "options": { "0": { "color": "blue", "index": 1, "text": "Offline (sched)" }, "1": { "color": "green", "index": 0, "text": "Online" } }, "type": "value" }
]
}
]
}
]
},
"gridPos": { "h": 5, "w": 12, "x": 0, "y": 4 },
"id": 7,
"options": { "alignValue": "left", "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "mergeValues": true, "rowHeight": 0.8, "showValue": "auto" },
"targets": [
{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_gateway_status{gateway=~\"NORCOM_DHCP|WAN1|WAN_0\"}", "legendFormat": "{{gateway}} ({{descr}})", "refId": "A" }
],
"title": "Gateway Availability (dpinger)",
"type": "state-timeline"
},
{
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
"fieldConfig": {
"defaults": {
"mappings": [
{ "options": { "0": { "color": "semi-dark-purple", "index": 1, "text": "Standby" }, "1": { "color": "green", "index": 0, "text": "Active" } }, "type": "value" }
]
},
"overrides": []
},
"gridPos": { "h": 5, "w": 12, "x": 12, "y": 4 },
"id": 18,
"options": { "alignValue": "left", "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "mergeValues": true, "rowHeight": 0.8, "showValue": "auto" },
"targets": [
{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_gateway_default{gateway=~\"NORCOM_DHCP|WAN1|WAN_0\"}", "legendFormat": "{{gateway}} ({{descr}})", "refId": "A" }
],
"title": "Active Default Route (Failover)",
"type": "state-timeline"
},
{
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
"fieldConfig": {
"defaults": { "unit": "ms", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10 } },
"overrides": []
},
"gridPos": { "h": 7, "w": 12, "x": 0, "y": 9 },
"id": 8,
"options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": ["mean", "max", "lastNotNull"] }, "tooltip": { "mode": "multi" } },
"targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_gateway_delay_seconds{gateway=~\"NORCOM_DHCP|WAN1|WAN_0\"} * 1000", "legendFormat": "{{gateway}} ({{descr}} -> {{monitor}})", "refId": "A" }],
"title": "Gateway Latency (RTT, dpinger)",
"type": "timeseries"
},
{
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
"fieldConfig": {
"defaults": { "unit": "percent", "min": 0, "max": 100, "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 15 } },
"overrides": []
},
"gridPos": { "h": 7, "w": 12, "x": 12, "y": 9 },
"id": 9,
"options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": ["mean", "max", "lastNotNull"] }, "tooltip": { "mode": "multi" } },
"targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_gateway_loss_ratio{gateway=~\"NORCOM_DHCP|WAN1|WAN_0\"} * 100", "legendFormat": "{{gateway}} ({{descr}})", "refId": "A" }],
"title": "Gateway Packet Loss (%)",
"type": "timeseries"
},
{
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
"fieldConfig": {
"defaults": { "unit": "bps", "custom": { "drawStyle": "line", "lineWidth": 1.5, "fillOpacity": 15 } },
"overrides": []
},
"gridPos": { "h": 7, "w": 12, "x": 0, "y": 16 },
"id": 10,
"options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": ["mean", "max", "lastNotNull"] }, "tooltip": { "mode": "multi" } },
"targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "rate(opnsense_interface_rx_bytes_total{device=~\"igc0|igc1|igc2|proxytun2socks0|vtnet0\"}[1m]) * 8", "legendFormat": "RX {{descr}} ({{device}})", "refId": "A" }],
"title": "Interface Download (RX)",
"type": "timeseries"
},
{
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
"fieldConfig": {
"defaults": { "unit": "bps", "custom": { "drawStyle": "line", "lineWidth": 1.5, "fillOpacity": 15 } },
"overrides": []
},
"gridPos": { "h": 7, "w": 12, "x": 12, "y": 16 },
"id": 11,
"options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": ["mean", "max", "lastNotNull"] }, "tooltip": { "mode": "multi" } },
"targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "rate(opnsense_interface_tx_bytes_total{device=~\"igc0|igc1|igc2|proxytun2socks0|vtnet0\"}[1m]) * 8", "legendFormat": "TX {{descr}} ({{device}})", "refId": "A" }],
"title": "Interface Upload (TX)",
"type": "timeseries"
},
{
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
"fieldConfig": {
"defaults": { "unit": "pps", "custom": { "drawStyle": "line", "lineWidth": 1.5, "fillOpacity": 10 } },
"overrides": []
},
"gridPos": { "h": 6, "w": 12, "x": 0, "y": 23 },
"id": 12,
"options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi" } },
"targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "rate(opnsense_interface_rx_errors_total{device=~\"igc0|igc1|igc2|vtnet0\"}[5m]) + rate(opnsense_interface_tx_errors_total{device=~\"igc0|igc1|igc2|vtnet0\"}[5m]) + rate(opnsense_interface_dropped_packets_total{device=~\"igc0|igc1|igc2|vtnet0\"}[5m])", "legendFormat": "{{descr}} ({{device}})", "refId": "A" }],
"title": "Interface Errors & Drops",
"type": "timeseries"
},
{
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
"fieldConfig": {
"defaults": {
"mappings": [
{ "options": { "0": { "color": "red", "index": 1, "text": "Stopped" }, "1": { "color": "green", "index": 0, "text": "Running" } }, "type": "value" }
],
"thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] }
},
"overrides": []
},
"gridPos": { "h": 6, "w": 12, "x": 12, "y": 23 },
"id": 13,
"options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "value_and_name" },
"targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_service_running{name=~\"unbound|xray|netbird|dhcpd|pf|node_exporter\"}", "legendFormat": "{{name}}", "refId": "A" }],
"title": "OPNsense Services",
"type": "stat"
},
{
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
"fieldConfig": {
"defaults": { "unit": "percent", "min": 0, "max": 100, "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 15 } },
"overrides": []
},
"gridPos": { "h": 6, "w": 8, "x": 0, "y": 29 },
"id": 14,
"options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi" } },
"targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_cpu_usage_percent", "legendFormat": "CPU %", "refId": "A" }],
"title": "CPU Usage (%)",
"type": "timeseries"
},
{
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
"fieldConfig": {
"defaults": { "unit": "short", "custom": { "drawStyle": "line", "lineWidth": 1.5, "fillOpacity": 10 } },
"overrides": []
},
"gridPos": { "h": 6, "w": 8, "x": 8, "y": 29 },
"id": 15,
"options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi" } },
"targets": [
{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_load1", "legendFormat": "1m", "refId": "A" },
{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_load5", "legendFormat": "5m", "refId": "B" },
{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_load15", "legendFormat": "15m", "refId": "C" }
],
"title": "Load Average",
"type": "timeseries"
},
{
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
"fieldConfig": {
"defaults": {
"unit": "percent",
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "orange", "value": 80 }, { "color": "red", "value": 95 }] }
},
"overrides": []
},
"gridPos": { "h": 6, "w": 4, "x": 16, "y": 29 },
"id": 16,
"options": { "colorMode": "value", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" },
"targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_disk_used_percent{mountpoint=\"/\"}", "refId": "A" }],
"title": "Disk Used (/)",
"type": "stat"
},
{
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
"fieldConfig": {
"defaults": { "unit": "s" },
"overrides": []
},
"gridPos": { "h": 6, "w": 4, "x": 20, "y": 29 },
"id": 17,
"options": { "colorMode": "value", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" },
"targets": [{ "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "expr": "opnsense_uptime_seconds", "refId": "A" }],
"title": "Uptime",
"type": "stat"
}
],
"refresh": "15s",
"schemaVersion": 39,
"tags": ["opnsense", "wan", "network"],
"templating": { "list": [] },
"time": { "from": "now-6h", "to": "now" },
"timepicker": {},
"timezone": "",
"title": "OPNsense — Multi-WAN & Router",
"uid": "opnsense-wan",
"version": 1
}
@@ -0,0 +1,338 @@
#!/usr/bin/env python3
"""Lightweight Prometheus exporter for OPNsense gateways, interfaces, pf, and system metrics."""
import base64
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
import json
import os
import re
import ssl
import sys
import urllib.error
import urllib.request
API_URL = os.environ.get("OPNSENSE_API_URL", "https://192.168.1.1:4433").rstrip("/")
API_KEY = os.environ.get("OPNSENSE_API_KEY", "")
API_SECRET = os.environ.get("OPNSENSE_API_SECRET", "")
VERIFY_TLS = os.environ.get("OPNSENSE_VERIFY_TLS", "false").lower() in ("1", "true", "yes")
LISTEN_PORT = int(os.environ.get("OPNSENSE_EXPORTER_PORT", "9105"))
TIMEOUT = float(os.environ.get("OPNSENSE_API_TIMEOUT", "5"))
def _ssl_context() -> ssl.SSLContext:
ctx = ssl.create_default_context()
if not VERIFY_TLS:
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
return ctx
def _auth_header() -> str:
raw = f"{API_KEY}:{API_SECRET}".encode("utf-8")
return "Basic " + base64.b64encode(raw).decode("ascii")
def _api_get(path: str):
req = urllib.request.Request(
f"{API_URL}{path}",
headers={
"Authorization": _auth_header(),
"Accept": "application/json",
},
method="GET",
)
with urllib.request.urlopen(req, context=_ssl_context(), timeout=TIMEOUT) as resp:
return json.loads(resp.read().decode("utf-8"))
def _esc(val: str) -> str:
return str(val).replace("\\", "\\\\").replace('"', '\\"').replace("\n", "\\n")
def _labels(mapping: dict[str, str]) -> str:
parts = [f'{k}="{_esc(v)}"' for k, v in mapping.items()]
return "{" + ",".join(parts) + "}"
def _parse_ms(text: str):
m = re.match(r"^\s*([0-9]+(?:\.[0-9]+)?)\s*ms\s*$", str(text or ""))
return float(m.group(1)) / 1000.0 if m else None
def _parse_pct(text: str):
m = re.match(r"^\s*([0-9]+(?:\.[0-9]+)?)\s*%\s*$", str(text or ""))
return float(m.group(1)) / 100.0 if m else None
def _parse_uptime(text: str):
# Examples: "3 days, 16:05:44", "16:05:44", "1 day, 02:10:00"
m = re.match(r"^(?:(\d+)\s+days?,\s*)?(\d+):(\d+):(\d+)$", str(text or "").strip())
if not m:
return None
days = int(m.group(1) or 0)
hours, mins, secs = int(m.group(2)), int(m.group(3)), int(m.group(4))
return days * 86400 + hours * 3600 + mins * 60 + secs
def collect_metrics() -> str:
lines: list[str] = []
gateways_data = _api_get("/api/routing/settings/searchGateway").get("rows", [])
routes_data = _api_get("/api/diagnostics/interface/getRoutes")
ifaces_data = _api_get("/api/interfaces/overview/interfacesInfo").get("rows", [])
ifstats_data = _api_get("/api/diagnostics/interface/getInterfaceStatistics").get("statistics", {})
pf_data = _api_get("/api/diagnostics/firewall/pfStatistics/info").get("info", {})
sys_res = _api_get("/api/diagnostics/system/systemResources")
sys_time = _api_get("/api/diagnostics/system/systemTime")
sys_disk = _api_get("/api/diagnostics/system/systemDisk").get("devices", [])
sys_act = _api_get("/api/diagnostics/activity/getActivity").get("headers", [])
services_data = _api_get("/api/core/service/search").get("rows", [])
# Active IPv4 default route from kernel routing table
active_default_gw = None
active_default_if = None
for r in routes_data:
if r.get("proto") == "ipv4" and r.get("destination") == "default":
active_default_gw = r.get("gateway")
active_default_if = r.get("netif")
break
# Gateways (dpinger)
lines.extend([
"# HELP opnsense_gateway_status Gateway online status (1=Online, 0=Offline)",
"# TYPE opnsense_gateway_status gauge",
"# HELP opnsense_gateway_configured_default Configured default gateway flag",
"# TYPE opnsense_gateway_configured_default gauge",
"# HELP opnsense_gateway_default Active kernel default route flag (1=active default)",
"# TYPE opnsense_gateway_default gauge",
"# HELP opnsense_gateway_delay_seconds Gateway monitor RTT in seconds",
"# TYPE opnsense_gateway_delay_seconds gauge",
"# HELP opnsense_gateway_stddev_seconds Gateway monitor RTT stddev in seconds",
"# TYPE opnsense_gateway_stddev_seconds gauge",
"# HELP opnsense_gateway_loss_ratio Gateway monitor packet loss ratio (0.0-1.0)",
"# TYPE opnsense_gateway_loss_ratio gauge",
])
for gw in gateways_data:
name = gw.get("name", "")
descr = gw.get("interface_descr") or gw.get("descr") or name
iface = gw.get("interface", "")
device = gw.get("if", "")
monitor = gw.get("monitor", "")
gw_addr = gw.get("gateway", "")
lbl = _labels({
"gateway": name,
"descr": descr,
"interface": iface,
"device": device,
"monitor": monitor,
})
online = 1 if gw.get("status") == "Online" else 0
lines.append(f"opnsense_gateway_status{lbl} {online}")
lines.append(f"opnsense_gateway_configured_default{lbl} {1 if gw.get('defaultgw') else 0}")
is_active_default = 1 if (
active_default_gw and active_default_if
and gw_addr == active_default_gw and device == active_default_if
) else 0
lines.append(f"opnsense_gateway_default{lbl} {is_active_default}")
loss = _parse_pct(gw.get("loss"))
if loss is not None:
lines.append(f"opnsense_gateway_loss_ratio{lbl} {loss}")
if online:
delay = _parse_ms(gw.get("delay"))
stddev = _parse_ms(gw.get("stddev"))
if delay is not None:
lines.append(f"opnsense_gateway_delay_seconds{lbl} {delay}")
if stddev is not None:
lines.append(f"opnsense_gateway_stddev_seconds{lbl} {stddev}")
# Interfaces status & traffic counters
device_meta: dict[str, tuple[str, str]] = {}
lines.extend([
"# HELP opnsense_interface_up Interface operational status (1=up, 0=down)",
"# TYPE opnsense_interface_up gauge",
])
for iface in ifaces_data:
ident = iface.get("identifier") or ""
device = iface.get("device") or ""
descr = iface.get("description") or ident or device
if not ident or not device:
continue
device_meta[device] = (ident, descr)
up = 1 if iface.get("status") == "up" else 0
lbl = _labels({"interface": ident, "device": device, "descr": descr})
lines.append(f"opnsense_interface_up{lbl} {up}")
lines.extend([
"# HELP opnsense_interface_rx_bytes_total Received bytes on interface",
"# TYPE opnsense_interface_rx_bytes_total counter",
"# HELP opnsense_interface_tx_bytes_total Transmitted bytes on interface",
"# TYPE opnsense_interface_tx_bytes_total counter",
"# HELP opnsense_interface_rx_packets_total Received packets on interface",
"# TYPE opnsense_interface_rx_packets_total counter",
"# HELP opnsense_interface_tx_packets_total Transmitted packets on interface",
"# TYPE opnsense_interface_tx_packets_total counter",
"# HELP opnsense_interface_rx_errors_total Receive errors on interface",
"# TYPE opnsense_interface_rx_errors_total counter",
"# HELP opnsense_interface_tx_errors_total Transmit errors on interface",
"# TYPE opnsense_interface_tx_errors_total counter",
"# HELP opnsense_interface_dropped_packets_total Dropped packets on interface",
"# TYPE opnsense_interface_dropped_packets_total counter",
])
seen_devices: set[str] = set()
for stat in ifstats_data.values():
device = stat.get("name")
if not device or device not in device_meta or device in seen_devices:
continue
if stat.get("received-bytes") is None:
continue
seen_devices.add(device)
ident, descr = device_meta[device]
lbl = _labels({"interface": ident, "device": device, "descr": descr})
lines.append(f"opnsense_interface_rx_bytes_total{lbl} {stat['received-bytes']}")
lines.append(f"opnsense_interface_tx_bytes_total{lbl} {stat['sent-bytes']}")
lines.append(f"opnsense_interface_rx_packets_total{lbl} {stat['received-packets']}")
lines.append(f"opnsense_interface_tx_packets_total{lbl} {stat['sent-packets']}")
lines.append(f"opnsense_interface_rx_errors_total{lbl} {stat.get('received-errors') or 0}")
lines.append(f"opnsense_interface_tx_errors_total{lbl} {stat.get('send-errors') or 0}")
lines.append(f"opnsense_interface_dropped_packets_total{lbl} {stat.get('dropped-packets') or 0}")
# pf state table
state_table = pf_data.get("state-table", {})
pf_current = state_table.get("current-entries", {}).get("total")
pf_inserts = state_table.get("inserts", {}).get("total")
pf_removals = state_table.get("removals", {}).get("total")
if pf_current is not None:
lines.extend([
"# HELP opnsense_pf_states_current Current active pf states",
"# TYPE opnsense_pf_states_current gauge",
f"opnsense_pf_states_current {pf_current}",
])
if pf_inserts is not None and pf_removals is not None:
lines.extend([
"# HELP opnsense_pf_state_inserts_total Total pf state insertions",
"# TYPE opnsense_pf_state_inserts_total counter",
f"opnsense_pf_state_inserts_total {pf_inserts}",
"# HELP opnsense_pf_state_removals_total Total pf state removals",
"# TYPE opnsense_pf_state_removals_total counter",
f"opnsense_pf_state_removals_total {pf_removals}",
])
# System memory, load, CPU, disk & uptime
mem = sys_res.get("memory", {})
if mem.get("total") is not None and mem.get("used") is not None:
lines.extend([
"# HELP opnsense_memory_total_bytes Total physical memory in bytes",
"# TYPE opnsense_memory_total_bytes gauge",
f"opnsense_memory_total_bytes {mem['total']}",
"# HELP opnsense_memory_used_bytes Used memory in bytes",
"# TYPE opnsense_memory_used_bytes gauge",
f"opnsense_memory_used_bytes {mem['used']}",
])
load_parts = [p.strip() for p in str(sys_time.get("loadavg", "")).split(",") if p.strip()]
if len(load_parts) == 3:
lines.extend([
"# HELP opnsense_load1 1-minute system load average",
"# TYPE opnsense_load1 gauge",
f"opnsense_load1 {float(load_parts[0])}",
"# HELP opnsense_load5 5-minute system load average",
"# TYPE opnsense_load5 gauge",
f"opnsense_load5 {float(load_parts[1])}",
"# HELP opnsense_load15 15-minute system load average",
"# TYPE opnsense_load15 gauge",
f"opnsense_load15 {float(load_parts[2])}",
])
uptime_sec = _parse_uptime(sys_time.get("uptime"))
if uptime_sec is not None:
lines.extend([
"# HELP opnsense_uptime_seconds System uptime in seconds",
"# TYPE opnsense_uptime_seconds gauge",
f"opnsense_uptime_seconds {uptime_sec}",
])
for h in sys_act:
m = re.search(r"([0-9]+(?:\.[0-9]+)?)%\s*idle", str(h))
if m:
cpu_used = round(max(0.0, 100.0 - float(m.group(1))), 2)
lines.extend([
"# HELP opnsense_cpu_usage_percent Total CPU usage percentage",
"# TYPE opnsense_cpu_usage_percent gauge",
f"opnsense_cpu_usage_percent {cpu_used}",
])
break
lines.extend([
"# HELP opnsense_disk_used_percent Disk space used percentage",
"# TYPE opnsense_disk_used_percent gauge",
])
for dev in sys_disk:
mp = dev.get("mountpoint")
pct = dev.get("used_pct")
if mp and pct is not None:
lbl = _labels({
"mountpoint": mp,
"device": dev.get("device", ""),
"fstype": dev.get("type", ""),
})
lines.append(f"opnsense_disk_used_percent{lbl} {pct}")
# Services
lines.extend([
"# HELP opnsense_service_running OPNsense service status (1=running, 0=stopped)",
"# TYPE opnsense_service_running gauge",
])
for svc in services_data:
lbl = _labels({
"service_id": svc.get("id", ""),
"name": svc.get("name", ""),
})
lines.append(f"opnsense_service_running{lbl} {1 if svc.get('running') else 0}")
return "\n".join(lines) + "\n"
class MetricsHandler(BaseHTTPRequestHandler):
def do_GET(self):
if self.path == "/health":
self.send_response(200)
self.send_header("Content-Type", "text/plain")
self.end_headers()
self.wfile.write(b"ok\n")
return
if self.path != "/metrics":
self.send_response(404)
self.end_headers()
return
try:
body = collect_metrics().encode("utf-8")
self.send_response(200)
self.send_header("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
self.end_headers()
self.wfile.write(body)
except (urllib.error.URLError, TimeoutError, json.JSONDecodeError, KeyError) as exc:
msg = f"# error collecting OPNsense metrics: {exc}\n".encode("utf-8")
self.send_response(503)
self.send_header("Content-Type", "text/plain; charset=utf-8")
self.end_headers()
self.wfile.write(msg)
def log_message(self, format, *args):
return
def main():
if not API_KEY or not API_SECRET:
print("OPNSENSE_API_KEY and OPNSENSE_API_SECRET must be set", file=sys.stderr)
sys.exit(1)
server = ThreadingHTTPServer(("0.0.0.0", LISTEN_PORT), MetricsHandler)
server.serve_forever()
if __name__ == "__main__":
main()
@@ -18,6 +18,13 @@
when: xray_checker_env is defined and xray_checker_env is changed when: xray_checker_env is defined and xray_checker_env is changed
changed_when: true changed_when: true
- name: Пересоздать экспортер OPNsense при обновлении скрипта или окружения
ansible.builtin.command: docker compose -f {{ monitoring_dir }}/compose.yaml up -d --force-recreate --no-deps opnsense-exporter
when: >
(opnsense_exporter_env is defined and opnsense_exporter_env is changed) or
(opnsense_exporter_script is defined and opnsense_exporter_script is changed)
changed_when: true
- name: Дождаться VictoriaMetrics - name: Дождаться VictoriaMetrics
ansible.builtin.uri: ansible.builtin.uri:
url: "http://{{ ansible_host }}:8428/health" url: "http://{{ ansible_host }}:8428/health"
@@ -21,3 +21,4 @@
- { path: "{{ monitoring_dir }}/vmagent", owner: "root", group: "root", mode: "0755" } - { path: "{{ monitoring_dir }}/vmagent", owner: "root", group: "root", mode: "0755" }
- { path: "{{ monitoring_dir }}/vmagent/conf.d", owner: "root", group: "root", mode: "0755" } - { path: "{{ monitoring_dir }}/vmagent/conf.d", owner: "root", group: "root", mode: "0755" }
- { path: "{{ monitoring_dir }}/vmalert", owner: "root", group: "root", mode: "0755" } - { path: "{{ monitoring_dir }}/vmalert", owner: "root", group: "root", mode: "0755" }
- { path: "{{ monitoring_dir }}/opnsense-exporter", owner: "root", group: "root", mode: "0755" }
@@ -96,3 +96,42 @@
mode: "0644" mode: "0644"
when: xray_checker_subscription_url | length > 0 when: xray_checker_subscription_url | length > 0
- name: Убрать scrape OPNsense без API-ключей
ansible.builtin.file:
path: "{{ item }}"
state: absent
loop:
- "{{ monitoring_dir }}/vmagent/conf.d/opnsense.yml"
- "{{ monitoring_dir }}/opnsense-exporter.env"
when: not (opnsense_exporter_enabled | bool)
- name: Скрипт экспортера OPNsense
ansible.builtin.copy:
src: opnsense-exporter/exporter.py
dest: "{{ monitoring_dir }}/opnsense-exporter/exporter.py"
owner: root
group: root
mode: "0755"
register: opnsense_exporter_script
when: opnsense_exporter_enabled | bool
- name: Файл окружения экспортера OPNsense
ansible.builtin.template:
src: opnsense-exporter.env.j2
dest: "{{ monitoring_dir }}/opnsense-exporter.env"
owner: root
group: root
mode: "0600"
no_log: true
register: opnsense_exporter_env
when: opnsense_exporter_enabled | bool
- name: Scrape OPNsense
ansible.builtin.template:
src: vmagent-opnsense.yml.j2
dest: "{{ monitoring_dir }}/vmagent/conf.d/opnsense.yml"
owner: root
group: root
mode: "0644"
when: opnsense_exporter_enabled | bool
@@ -218,3 +218,101 @@ groups:
annotations: annotations:
summary: "Прокси {{ $labels.name }} не проходит проверку" summary: "Прокси {{ $labels.name }} не проходит проверку"
description: "xray_proxy_status {{ $labels.name }} ({{ $labels.address }}) не равен 1." description: "xray_proxy_status {{ $labels.name }} ({{ $labels.address }}) не равен 1."
[% if opnsense_exporter_enabled | bool %]
- alert: OpnsenseExporterDown
expr: up{job="opnsense"} == 0
for: 2m
labels:
severity: critical
annotations:
summary: "Экспортер OPNsense недоступен"
description: "Scrape opnsense ({{ $labels.instance }}) возвращает up=0 более 2 минут."
- alert: OpnsenseMetricsStale
expr: |
(
time() - tlast_over_time(up{job="opnsense"}[30m]) > 180
)
or absent_over_time(up{job="opnsense",instance="opnsense"}[30m])
for: 1m
labels:
severity: critical
annotations:
summary: "Нет свежих метрик OPNsense"
description: "opnsense-exporter не присылает up дольше 3 минут."
# Мониторим отвал только постоянных каналов (NorCom и MTC); WAN_0 (Iterika) работает по расписанию.
- alert: OpnsenseGatewayDown
expr: opnsense_gateway_status{gateway=~"NORCOM_DHCP|WAN1"} == 0
for: 2m
labels:
severity: critical
annotations:
summary: "Провайдер OPNsense {{ $labels.gateway }} ({{ $labels.descr }}) недоступен"
description: "Шлюз {{ $labels.gateway }} (монитор {{ $labels.monitor }}) в статусе Offline более 2 минут."
- alert: OpnsenseGatewayPacketLoss
expr: opnsense_gateway_loss_ratio{gateway=~"NORCOM_DHCP|WAN1"} > 0.1
for: 3m
labels:
severity: warning
annotations:
summary: "Потери пакетов на {{ $labels.gateway }} ({{ $labels.descr }})"
description: "Потери на шлюзе {{ $labels.gateway }} превышают 10% более 3 минут."
- alert: OpnsenseGatewayHighLatency
expr: opnsense_gateway_delay_seconds{gateway=~"NORCOM_DHCP|WAN1"} > 0.25
for: 5m
labels:
severity: warning
annotations:
summary: "Высокая задержка на {{ $labels.gateway }} ({{ $labels.descr }})"
description: "RTT до {{ $labels.monitor }} через {{ $labels.gateway }} превышает 250 ms более 5 минут."
# Уведомление при переключении default-маршрута на резерв (включая WAN_0 / Iterika).
- alert: OpnsenseFailoverActive
expr: opnsense_gateway_default{gateway!="NORCOM_DHCP"} == 1
for: 30s
labels:
severity: warning
annotations:
summary: "OPNsense переключился на резерв: {{ $labels.gateway }} ({{ $labels.descr }})"
description: "Активный default-маршрут ушёл с NORCOM_DHCP на {{ $labels.gateway }} ({{ $labels.descr }}, {{ $labels.device }})."
- alert: OpnsenseDiskSpaceLow
expr: opnsense_disk_used_percent{mountpoint="/"} > 85 and opnsense_disk_used_percent{mountpoint="/"} <= 95
for: 5m
labels:
severity: warning
annotations:
summary: "Мало свободного места на OPNsense (/)"
description: "Корневой раздел / на OPNsense заполнен более чем на 85%."
- alert: OpnsenseDiskSpaceCritical
expr: opnsense_disk_used_percent{mountpoint="/"} > 95
for: 5m
labels:
severity: critical
annotations:
summary: "Заканчивается место на OPNsense (/)"
description: "Корневой раздел / на OPNsense заполнен более чем на 95%."
- alert: OpnsenseHighCpuLoad
expr: opnsense_cpu_usage_percent > 90
for: 10m
labels:
severity: warning
annotations:
summary: "Высокая нагрузка CPU на OPNsense"
description: "Утилизация CPU на OPNsense превышает 90% более 10 минут."
- alert: OpnsenseHighMemoryUsage
expr: (opnsense_memory_used_bytes / opnsense_memory_total_bytes) * 100 > 90
for: 10m
labels:
severity: warning
annotations:
summary: "Мало оперативной памяти на OPNsense"
description: "Использование RAM на OPNsense превышает 90% более 10 минут."
[% endif %]
@@ -103,6 +103,20 @@ services:
ports: ports:
- "127.0.0.1:2112:2112" - "127.0.0.1:2112:2112"
{% endif %} {% endif %}
{% if opnsense_exporter_enabled | bool %}
opnsense-exporter:
image: {{ opnsense_exporter_image }}
container_name: monitoring-opnsense-exporter
restart: unless-stopped
command: ["python3", "/app/exporter.py"]
env_file:
- ./opnsense-exporter.env
volumes:
- ./opnsense-exporter/exporter.py:/app/exporter.py:ro
ports:
- "127.0.0.1:9105:9105"
{% endif %}
vmagent: vmagent:
image: {{ vmagent_image }} image: {{ vmagent_image }}
@@ -0,0 +1,6 @@
OPNSENSE_API_URL={{ opnsense_api_url }}
OPNSENSE_API_KEY={{ opnsense_api_key }}
OPNSENSE_API_SECRET={{ opnsense_api_secret }}
OPNSENSE_VERIFY_TLS=false
OPNSENSE_EXPORTER_PORT=9105
@@ -0,0 +1,9 @@
- job_name: opnsense
scrape_interval: 15s
static_configs:
- targets: ["opnsense-exporter:9105"]
labels:
instance: "opnsense"
cluster: "{{ vmagent_cluster }}"
host: "opnsense"