fix(monitoring): distinguish OPNsense Packetloss from Offline and inhibit duplicate gateway alerts
ansible-deploy / plan (push) Successful in 2s
ansible-lint / lint (push) Successful in 3s
ansible-deploy / deploy (core-mon, ) (push) Successful in 32s

This commit is contained in:
karpenok committed 2026-11-19 23:46:55 +07:00
1 parent e623faff2b
commit 044c992edd
6 files changed
+45 -15

No files matched your search

@@ -127,7 +127,9 @@ def collect_metrics() -> str:
"device": device,
"monitor": monitor,
})
online = 1 if gw.get("status") == "Online" else 0
status_str = str(gw.get("status") or "").strip()
force_down = str(gw.get("force_down") or "0").strip() == "1"
online = 0 if (force_down or status_str in ("Offline", "force_down", "")) else 1
lines.append(f"opnsense_gateway_status{lbl} {online}")
lines.append(f"opnsense_gateway_configured_default{lbl} {1 if gw.get('defaultgw') else 0}")
@@ -141,13 +143,12 @@ def collect_metrics() -> str:
if loss is not None:
lines.append(f"opnsense_gateway_loss_ratio{lbl} {loss}")
if online:
delay = _parse_ms(gw.get("delay"))
stddev = _parse_ms(gw.get("stddev"))
if delay is not None:
lines.append(f"opnsense_gateway_delay_seconds{lbl} {delay}")
if stddev is not None:
lines.append(f"opnsense_gateway_stddev_seconds{lbl} {stddev}")
delay = _parse_ms(gw.get("delay"))
stddev = _parse_ms(gw.get("stddev"))
if online and delay is not None and (loss is None or loss < 1.0):
lines.append(f"opnsense_gateway_delay_seconds{lbl} {delay}")
if online and stddev is not None and (loss is None or loss < 1.0):
lines.append(f"opnsense_gateway_stddev_seconds{lbl} {stddev}")
# Interfaces status & traffic counters
device_meta: dict[str, tuple[str, str]] = {}
@@ -13,6 +13,16 @@
register: compose_up
changed_when: "'Started' in compose_up.stderr or 'Created' in compose_up.stderr or 'Recreated' in compose_up.stderr or 'Pulled' in compose_up.stderr"
- name: Пересоздать Alertmanager при обновлении конфигурации
ansible.builtin.command: docker compose -f {{ monitoring_dir }}/compose.yaml up -d --force-recreate --no-deps alertmanager
when: alertmanager_config is defined and alertmanager_config is changed
changed_when: true
- name: Пересоздать alertmanager-ntfy при обновлении конфигурации
ansible.builtin.command: docker compose -f {{ monitoring_dir }}/compose.yaml up -d --force-recreate --no-deps alertmanager-ntfy
when: alertmanager_ntfy_config is defined and alertmanager_ntfy_config is changed
changed_when: true
- name: Пересоздать Xray Checker, чтобы подхватить метод проверки
ansible.builtin.command: docker compose -f {{ monitoring_dir }}/compose.yaml up -d --force-recreate --no-deps xray-checker
when: xray_checker_env is defined and xray_checker_env is changed
@@ -41,6 +41,7 @@
owner: root
group: root
mode: "0644"
register: alertmanager_config
- name: Установить конфигурацию alertmanager-ntfy
ansible.builtin.template:
@@ -49,6 +50,7 @@
owner: root
group: root
mode: "0644"
register: alertmanager_ntfy_config
- name: Конфиг vmagent на core-mon
ansible.builtin.template:
@@ -2,12 +2,21 @@ global:
resolve_timeout: 5m
route:
group_by: ['alertname', 'instance', 'severity']
group_by: ['alertname', 'instance', 'gateway', 'severity']
group_wait: 30s
group_interval: 5m
repeat_interval: 4h
receiver: 'ntfy'
inhibit_rules:
- source_matchers:
- alertname="OpnsenseGatewayDown"
- severity="critical"
target_matchers:
- alertname=~"OpnsenseGatewayPacketLoss|OpnsenseGatewayHighLatency"
- severity="warning"
equal: ['instance', 'gateway']
receivers:
- name: 'ntfy'
webhook_configs:
@@ -245,24 +245,30 @@ groups:
# Мониторим отвал только постоянных каналов (NorCom и MTC); WAN_0 (Iterika) работает по расписанию.
- alert: OpnsenseGatewayDown
expr: opnsense_gateway_status{gateway=~"NORCOM_DHCP|WAN1"} == 0
for: 2m
for: 3m
labels:
severity: critical
annotations:
summary: "Провайдер OPNsense {{ $labels.gateway }} ({{ $labels.descr }}) недоступен"
description: "Шлюз {{ $labels.gateway }} (монитор {{ $labels.monitor }}) в статусе Offline более 2 минут."
description: "Шлюз {{ $labels.gateway }} (монитор {{ $labels.monitor }}) в статусе Offline более 3 минут."
- alert: OpnsenseGatewayPacketLoss
expr: opnsense_gateway_loss_ratio{gateway=~"NORCOM_DHCP|WAN1"} > 0.1
for: 3m
expr: |
opnsense_gateway_loss_ratio{gateway=~"NORCOM_DHCP|WAN1"} > 0.1
and
opnsense_gateway_status{gateway=~"NORCOM_DHCP|WAN1"} == 1
for: 5m
labels:
severity: warning
annotations:
summary: "Потери пакетов на {{ $labels.gateway }} ({{ $labels.descr }})"
description: "Потери на шлюзе {{ $labels.gateway }} превышают 10% более 3 минут."
description: "Потери на шлюзе {{ $labels.gateway }} превышают 10% более 5 минут."
- alert: OpnsenseGatewayHighLatency
expr: opnsense_gateway_delay_seconds{gateway=~"NORCOM_DHCP|WAN1"} > 0.25
expr: |
opnsense_gateway_delay_seconds{gateway=~"NORCOM_DHCP|WAN1"} > 0.25
and
opnsense_gateway_status{gateway=~"NORCOM_DHCP|WAN1"} == 1
for: 5m
labels:
severity: warning
@@ -157,6 +157,8 @@ services:
restart: unless-stopped
command:
- -datasource.url=http://vmsingle:8428
- -remoteRead.url=http://vmsingle:8428
- -remoteWrite.url=http://vmsingle:8428
- -notifier.url=http://alertmanager:9093
- -rule=/etc/vmalert/alerts.yml
- -evaluationInterval=15s