fix(monitoring): distinguish OPNsense Packetloss from Offline and inhibit duplicate gateway alerts
This commit is contained in:
1 parent
e623faff2b
commit
044c992edd
6 files changed
+41
-11
No files matched your search
@@ -127,7 +127,9 @@ def collect_metrics() -> str:
|
||||
"device": device,
|
||||
"monitor": monitor,
|
||||
})
|
||||
online = 1 if gw.get("status") == "Online" else 0
|
||||
status_str = str(gw.get("status") or "").strip()
|
||||
force_down = str(gw.get("force_down") or "0").strip() == "1"
|
||||
online = 0 if (force_down or status_str in ("Offline", "force_down", "")) else 1
|
||||
lines.append(f"opnsense_gateway_status{lbl} {online}")
|
||||
lines.append(f"opnsense_gateway_configured_default{lbl} {1 if gw.get('defaultgw') else 0}")
|
||||
|
||||
@@ -141,12 +143,11 @@ def collect_metrics() -> str:
|
||||
if loss is not None:
|
||||
lines.append(f"opnsense_gateway_loss_ratio{lbl} {loss}")
|
||||
|
||||
if online:
|
||||
delay = _parse_ms(gw.get("delay"))
|
||||
stddev = _parse_ms(gw.get("stddev"))
|
||||
if delay is not None:
|
||||
if online and delay is not None and (loss is None or loss < 1.0):
|
||||
lines.append(f"opnsense_gateway_delay_seconds{lbl} {delay}")
|
||||
if stddev is not None:
|
||||
if online and stddev is not None and (loss is None or loss < 1.0):
|
||||
lines.append(f"opnsense_gateway_stddev_seconds{lbl} {stddev}")
|
||||
|
||||
# Interfaces status & traffic counters
|
||||
|
||||
@@ -13,6 +13,16 @@
|
||||
register: compose_up
|
||||
changed_when: "'Started' in compose_up.stderr or 'Created' in compose_up.stderr or 'Recreated' in compose_up.stderr or 'Pulled' in compose_up.stderr"
|
||||
|
||||
- name: Пересоздать Alertmanager при обновлении конфигурации
|
||||
ansible.builtin.command: docker compose -f {{ monitoring_dir }}/compose.yaml up -d --force-recreate --no-deps alertmanager
|
||||
when: alertmanager_config is defined and alertmanager_config is changed
|
||||
changed_when: true
|
||||
|
||||
- name: Пересоздать alertmanager-ntfy при обновлении конфигурации
|
||||
ansible.builtin.command: docker compose -f {{ monitoring_dir }}/compose.yaml up -d --force-recreate --no-deps alertmanager-ntfy
|
||||
when: alertmanager_ntfy_config is defined and alertmanager_ntfy_config is changed
|
||||
changed_when: true
|
||||
|
||||
- name: Пересоздать Xray Checker, чтобы подхватить метод проверки
|
||||
ansible.builtin.command: docker compose -f {{ monitoring_dir }}/compose.yaml up -d --force-recreate --no-deps xray-checker
|
||||
when: xray_checker_env is defined and xray_checker_env is changed
|
||||
|
||||
@@ -41,6 +41,7 @@
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0644"
|
||||
register: alertmanager_config
|
||||
|
||||
- name: Установить конфигурацию alertmanager-ntfy
|
||||
ansible.builtin.template:
|
||||
@@ -49,6 +50,7 @@
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0644"
|
||||
register: alertmanager_ntfy_config
|
||||
|
||||
- name: Конфиг vmagent на core-mon
|
||||
ansible.builtin.template:
|
||||
|
||||
@@ -2,12 +2,21 @@ global:
|
||||
resolve_timeout: 5m
|
||||
|
||||
route:
|
||||
group_by: ['alertname', 'instance', 'severity']
|
||||
group_by: ['alertname', 'instance', 'gateway', 'severity']
|
||||
group_wait: 30s
|
||||
group_interval: 5m
|
||||
repeat_interval: 4h
|
||||
receiver: 'ntfy'
|
||||
|
||||
inhibit_rules:
|
||||
- source_matchers:
|
||||
- alertname="OpnsenseGatewayDown"
|
||||
- severity="critical"
|
||||
target_matchers:
|
||||
- alertname=~"OpnsenseGatewayPacketLoss|OpnsenseGatewayHighLatency"
|
||||
- severity="warning"
|
||||
equal: ['instance', 'gateway']
|
||||
|
||||
receivers:
|
||||
- name: 'ntfy'
|
||||
webhook_configs:
|
||||
|
||||
@@ -245,24 +245,30 @@ groups:
|
||||
# Мониторим отвал только постоянных каналов (NorCom и MTC); WAN_0 (Iterika) работает по расписанию.
|
||||
- alert: OpnsenseGatewayDown
|
||||
expr: opnsense_gateway_status{gateway=~"NORCOM_DHCP|WAN1"} == 0
|
||||
for: 2m
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Провайдер OPNsense {{ $labels.gateway }} ({{ $labels.descr }}) недоступен"
|
||||
description: "Шлюз {{ $labels.gateway }} (монитор {{ $labels.monitor }}) в статусе Offline более 2 минут."
|
||||
description: "Шлюз {{ $labels.gateway }} (монитор {{ $labels.monitor }}) в статусе Offline более 3 минут."
|
||||
|
||||
- alert: OpnsenseGatewayPacketLoss
|
||||
expr: opnsense_gateway_loss_ratio{gateway=~"NORCOM_DHCP|WAN1"} > 0.1
|
||||
for: 3m
|
||||
expr: |
|
||||
opnsense_gateway_loss_ratio{gateway=~"NORCOM_DHCP|WAN1"} > 0.1
|
||||
and
|
||||
opnsense_gateway_status{gateway=~"NORCOM_DHCP|WAN1"} == 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Потери пакетов на {{ $labels.gateway }} ({{ $labels.descr }})"
|
||||
description: "Потери на шлюзе {{ $labels.gateway }} превышают 10% более 3 минут."
|
||||
description: "Потери на шлюзе {{ $labels.gateway }} превышают 10% более 5 минут."
|
||||
|
||||
- alert: OpnsenseGatewayHighLatency
|
||||
expr: opnsense_gateway_delay_seconds{gateway=~"NORCOM_DHCP|WAN1"} > 0.25
|
||||
expr: |
|
||||
opnsense_gateway_delay_seconds{gateway=~"NORCOM_DHCP|WAN1"} > 0.25
|
||||
and
|
||||
opnsense_gateway_status{gateway=~"NORCOM_DHCP|WAN1"} == 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
|
||||
@@ -157,6 +157,8 @@ services:
|
||||
restart: unless-stopped
|
||||
command:
|
||||
- -datasource.url=http://vmsingle:8428
|
||||
- -remoteRead.url=http://vmsingle:8428
|
||||
- -remoteWrite.url=http://vmsingle:8428
|
||||
- -notifier.url=http://alertmanager:9093
|
||||
- -rule=/etc/vmalert/alerts.yml
|
||||
- -evaluationInterval=15s
|
||||
|
||||
Reference in new issue
Block a user