fix(monitoring): distinguish OPNsense Packetloss from Offline and inhibit duplicate gateway alerts
This commit is contained in:
1 parent
e623faff2b
commit
044c992edd
6 files changed
+41
-11
No files matched your search
@@ -127,7 +127,9 @@ def collect_metrics() -> str:
|
|||||||
"device": device,
|
"device": device,
|
||||||
"monitor": monitor,
|
"monitor": monitor,
|
||||||
})
|
})
|
||||||
online = 1 if gw.get("status") == "Online" else 0
|
status_str = str(gw.get("status") or "").strip()
|
||||||
|
force_down = str(gw.get("force_down") or "0").strip() == "1"
|
||||||
|
online = 0 if (force_down or status_str in ("Offline", "force_down", "")) else 1
|
||||||
lines.append(f"opnsense_gateway_status{lbl} {online}")
|
lines.append(f"opnsense_gateway_status{lbl} {online}")
|
||||||
lines.append(f"opnsense_gateway_configured_default{lbl} {1 if gw.get('defaultgw') else 0}")
|
lines.append(f"opnsense_gateway_configured_default{lbl} {1 if gw.get('defaultgw') else 0}")
|
||||||
|
|
||||||
@@ -141,12 +143,11 @@ def collect_metrics() -> str:
|
|||||||
if loss is not None:
|
if loss is not None:
|
||||||
lines.append(f"opnsense_gateway_loss_ratio{lbl} {loss}")
|
lines.append(f"opnsense_gateway_loss_ratio{lbl} {loss}")
|
||||||
|
|
||||||
if online:
|
|
||||||
delay = _parse_ms(gw.get("delay"))
|
delay = _parse_ms(gw.get("delay"))
|
||||||
stddev = _parse_ms(gw.get("stddev"))
|
stddev = _parse_ms(gw.get("stddev"))
|
||||||
if delay is not None:
|
if online and delay is not None and (loss is None or loss < 1.0):
|
||||||
lines.append(f"opnsense_gateway_delay_seconds{lbl} {delay}")
|
lines.append(f"opnsense_gateway_delay_seconds{lbl} {delay}")
|
||||||
if stddev is not None:
|
if online and stddev is not None and (loss is None or loss < 1.0):
|
||||||
lines.append(f"opnsense_gateway_stddev_seconds{lbl} {stddev}")
|
lines.append(f"opnsense_gateway_stddev_seconds{lbl} {stddev}")
|
||||||
|
|
||||||
# Interfaces status & traffic counters
|
# Interfaces status & traffic counters
|
||||||
|
|||||||
@@ -13,6 +13,16 @@
|
|||||||
register: compose_up
|
register: compose_up
|
||||||
changed_when: "'Started' in compose_up.stderr or 'Created' in compose_up.stderr or 'Recreated' in compose_up.stderr or 'Pulled' in compose_up.stderr"
|
changed_when: "'Started' in compose_up.stderr or 'Created' in compose_up.stderr or 'Recreated' in compose_up.stderr or 'Pulled' in compose_up.stderr"
|
||||||
|
|
||||||
|
- name: Пересоздать Alertmanager при обновлении конфигурации
|
||||||
|
ansible.builtin.command: docker compose -f {{ monitoring_dir }}/compose.yaml up -d --force-recreate --no-deps alertmanager
|
||||||
|
when: alertmanager_config is defined and alertmanager_config is changed
|
||||||
|
changed_when: true
|
||||||
|
|
||||||
|
- name: Пересоздать alertmanager-ntfy при обновлении конфигурации
|
||||||
|
ansible.builtin.command: docker compose -f {{ monitoring_dir }}/compose.yaml up -d --force-recreate --no-deps alertmanager-ntfy
|
||||||
|
when: alertmanager_ntfy_config is defined and alertmanager_ntfy_config is changed
|
||||||
|
changed_when: true
|
||||||
|
|
||||||
- name: Пересоздать Xray Checker, чтобы подхватить метод проверки
|
- name: Пересоздать Xray Checker, чтобы подхватить метод проверки
|
||||||
ansible.builtin.command: docker compose -f {{ monitoring_dir }}/compose.yaml up -d --force-recreate --no-deps xray-checker
|
ansible.builtin.command: docker compose -f {{ monitoring_dir }}/compose.yaml up -d --force-recreate --no-deps xray-checker
|
||||||
when: xray_checker_env is defined and xray_checker_env is changed
|
when: xray_checker_env is defined and xray_checker_env is changed
|
||||||
|
|||||||
@@ -41,6 +41,7 @@
|
|||||||
owner: root
|
owner: root
|
||||||
group: root
|
group: root
|
||||||
mode: "0644"
|
mode: "0644"
|
||||||
|
register: alertmanager_config
|
||||||
|
|
||||||
- name: Установить конфигурацию alertmanager-ntfy
|
- name: Установить конфигурацию alertmanager-ntfy
|
||||||
ansible.builtin.template:
|
ansible.builtin.template:
|
||||||
@@ -49,6 +50,7 @@
|
|||||||
owner: root
|
owner: root
|
||||||
group: root
|
group: root
|
||||||
mode: "0644"
|
mode: "0644"
|
||||||
|
register: alertmanager_ntfy_config
|
||||||
|
|
||||||
- name: Конфиг vmagent на core-mon
|
- name: Конфиг vmagent на core-mon
|
||||||
ansible.builtin.template:
|
ansible.builtin.template:
|
||||||
|
|||||||
@@ -2,12 +2,21 @@ global:
|
|||||||
resolve_timeout: 5m
|
resolve_timeout: 5m
|
||||||
|
|
||||||
route:
|
route:
|
||||||
group_by: ['alertname', 'instance', 'severity']
|
group_by: ['alertname', 'instance', 'gateway', 'severity']
|
||||||
group_wait: 30s
|
group_wait: 30s
|
||||||
group_interval: 5m
|
group_interval: 5m
|
||||||
repeat_interval: 4h
|
repeat_interval: 4h
|
||||||
receiver: 'ntfy'
|
receiver: 'ntfy'
|
||||||
|
|
||||||
|
inhibit_rules:
|
||||||
|
- source_matchers:
|
||||||
|
- alertname="OpnsenseGatewayDown"
|
||||||
|
- severity="critical"
|
||||||
|
target_matchers:
|
||||||
|
- alertname=~"OpnsenseGatewayPacketLoss|OpnsenseGatewayHighLatency"
|
||||||
|
- severity="warning"
|
||||||
|
equal: ['instance', 'gateway']
|
||||||
|
|
||||||
receivers:
|
receivers:
|
||||||
- name: 'ntfy'
|
- name: 'ntfy'
|
||||||
webhook_configs:
|
webhook_configs:
|
||||||
|
|||||||
@@ -245,24 +245,30 @@ groups:
|
|||||||
# Мониторим отвал только постоянных каналов (NorCom и MTC); WAN_0 (Iterika) работает по расписанию.
|
# Мониторим отвал только постоянных каналов (NorCom и MTC); WAN_0 (Iterika) работает по расписанию.
|
||||||
- alert: OpnsenseGatewayDown
|
- alert: OpnsenseGatewayDown
|
||||||
expr: opnsense_gateway_status{gateway=~"NORCOM_DHCP|WAN1"} == 0
|
expr: opnsense_gateway_status{gateway=~"NORCOM_DHCP|WAN1"} == 0
|
||||||
for: 2m
|
for: 3m
|
||||||
labels:
|
labels:
|
||||||
severity: critical
|
severity: critical
|
||||||
annotations:
|
annotations:
|
||||||
summary: "Провайдер OPNsense {{ $labels.gateway }} ({{ $labels.descr }}) недоступен"
|
summary: "Провайдер OPNsense {{ $labels.gateway }} ({{ $labels.descr }}) недоступен"
|
||||||
description: "Шлюз {{ $labels.gateway }} (монитор {{ $labels.monitor }}) в статусе Offline более 2 минут."
|
description: "Шлюз {{ $labels.gateway }} (монитор {{ $labels.monitor }}) в статусе Offline более 3 минут."
|
||||||
|
|
||||||
- alert: OpnsenseGatewayPacketLoss
|
- alert: OpnsenseGatewayPacketLoss
|
||||||
expr: opnsense_gateway_loss_ratio{gateway=~"NORCOM_DHCP|WAN1"} > 0.1
|
expr: |
|
||||||
for: 3m
|
opnsense_gateway_loss_ratio{gateway=~"NORCOM_DHCP|WAN1"} > 0.1
|
||||||
|
and
|
||||||
|
opnsense_gateway_status{gateway=~"NORCOM_DHCP|WAN1"} == 1
|
||||||
|
for: 5m
|
||||||
labels:
|
labels:
|
||||||
severity: warning
|
severity: warning
|
||||||
annotations:
|
annotations:
|
||||||
summary: "Потери пакетов на {{ $labels.gateway }} ({{ $labels.descr }})"
|
summary: "Потери пакетов на {{ $labels.gateway }} ({{ $labels.descr }})"
|
||||||
description: "Потери на шлюзе {{ $labels.gateway }} превышают 10% более 3 минут."
|
description: "Потери на шлюзе {{ $labels.gateway }} превышают 10% более 5 минут."
|
||||||
|
|
||||||
- alert: OpnsenseGatewayHighLatency
|
- alert: OpnsenseGatewayHighLatency
|
||||||
expr: opnsense_gateway_delay_seconds{gateway=~"NORCOM_DHCP|WAN1"} > 0.25
|
expr: |
|
||||||
|
opnsense_gateway_delay_seconds{gateway=~"NORCOM_DHCP|WAN1"} > 0.25
|
||||||
|
and
|
||||||
|
opnsense_gateway_status{gateway=~"NORCOM_DHCP|WAN1"} == 1
|
||||||
for: 5m
|
for: 5m
|
||||||
labels:
|
labels:
|
||||||
severity: warning
|
severity: warning
|
||||||
|
|||||||
@@ -157,6 +157,8 @@ services:
|
|||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
command:
|
command:
|
||||||
- -datasource.url=http://vmsingle:8428
|
- -datasource.url=http://vmsingle:8428
|
||||||
|
- -remoteRead.url=http://vmsingle:8428
|
||||||
|
- -remoteWrite.url=http://vmsingle:8428
|
||||||
- -notifier.url=http://alertmanager:9093
|
- -notifier.url=http://alertmanager:9093
|
||||||
- -rule=/etc/vmalert/alerts.yml
|
- -rule=/etc/vmalert/alerts.yml
|
||||||
- -evaluationInterval=15s
|
- -evaluationInterval=15s
|
||||||
|
|||||||
Reference in new issue
Block a user