From 044c992edd148f8344d104b4f64fd7507abe8560 Mon Sep 17 00:00:00 2001 From: karpenok Date: Thu, 19 Nov 2026 23:46:55 +0700 Subject: [PATCH] fix(monitoring): distinguish OPNsense Packetloss from Offline and inhibit duplicate gateway alerts --- .../files/opnsense-exporter/exporter.py | 17 +++++++++-------- .../core-mon/monitoring/tasks/compose.yml | 10 ++++++++++ ansible/roles/core-mon/monitoring/tasks/vm.yml | 2 ++ .../monitoring/templates/alertmanager.yml.j2 | 11 ++++++++++- .../monitoring/templates/alerts.yml.j2 | 18 ++++++++++++------ .../monitoring/templates/compose.yaml.j2 | 2 ++ 6 files changed, 45 insertions(+), 15 deletions(-) diff --git a/ansible/roles/core-mon/monitoring/files/opnsense-exporter/exporter.py b/ansible/roles/core-mon/monitoring/files/opnsense-exporter/exporter.py index e12953e..b956c6f 100644 --- a/ansible/roles/core-mon/monitoring/files/opnsense-exporter/exporter.py +++ b/ansible/roles/core-mon/monitoring/files/opnsense-exporter/exporter.py @@ -127,7 +127,9 @@ def collect_metrics() -> str: "device": device, "monitor": monitor, }) - online = 1 if gw.get("status") == "Online" else 0 + status_str = str(gw.get("status") or "").strip() + force_down = str(gw.get("force_down") or "0").strip() == "1" + online = 0 if (force_down or status_str in ("Offline", "force_down", "")) else 1 lines.append(f"opnsense_gateway_status{lbl} {online}") lines.append(f"opnsense_gateway_configured_default{lbl} {1 if gw.get('defaultgw') else 0}") @@ -141,13 +143,12 @@ def collect_metrics() -> str: if loss is not None: lines.append(f"opnsense_gateway_loss_ratio{lbl} {loss}") - if online: - delay = _parse_ms(gw.get("delay")) - stddev = _parse_ms(gw.get("stddev")) - if delay is not None: - lines.append(f"opnsense_gateway_delay_seconds{lbl} {delay}") - if stddev is not None: - lines.append(f"opnsense_gateway_stddev_seconds{lbl} {stddev}") + delay = _parse_ms(gw.get("delay")) + stddev = _parse_ms(gw.get("stddev")) + if online and delay is not None and (loss is None or loss < 1.0): + lines.append(f"opnsense_gateway_delay_seconds{lbl} {delay}") + if online and stddev is not None and (loss is None or loss < 1.0): + lines.append(f"opnsense_gateway_stddev_seconds{lbl} {stddev}") # Interfaces status & traffic counters device_meta: dict[str, tuple[str, str]] = {} diff --git a/ansible/roles/core-mon/monitoring/tasks/compose.yml b/ansible/roles/core-mon/monitoring/tasks/compose.yml index 9100871..45fa26a 100644 --- a/ansible/roles/core-mon/monitoring/tasks/compose.yml +++ b/ansible/roles/core-mon/monitoring/tasks/compose.yml @@ -13,6 +13,16 @@ register: compose_up changed_when: "'Started' in compose_up.stderr or 'Created' in compose_up.stderr or 'Recreated' in compose_up.stderr or 'Pulled' in compose_up.stderr" +- name: Пересоздать Alertmanager при обновлении конфигурации + ansible.builtin.command: docker compose -f {{ monitoring_dir }}/compose.yaml up -d --force-recreate --no-deps alertmanager + when: alertmanager_config is defined and alertmanager_config is changed + changed_when: true + +- name: Пересоздать alertmanager-ntfy при обновлении конфигурации + ansible.builtin.command: docker compose -f {{ monitoring_dir }}/compose.yaml up -d --force-recreate --no-deps alertmanager-ntfy + when: alertmanager_ntfy_config is defined and alertmanager_ntfy_config is changed + changed_when: true + - name: Пересоздать Xray Checker, чтобы подхватить метод проверки ansible.builtin.command: docker compose -f {{ monitoring_dir }}/compose.yaml up -d --force-recreate --no-deps xray-checker when: xray_checker_env is defined and xray_checker_env is changed diff --git a/ansible/roles/core-mon/monitoring/tasks/vm.yml b/ansible/roles/core-mon/monitoring/tasks/vm.yml index 04f9c47..3fc4fd6 100644 --- a/ansible/roles/core-mon/monitoring/tasks/vm.yml +++ b/ansible/roles/core-mon/monitoring/tasks/vm.yml @@ -41,6 +41,7 @@ owner: root group: root mode: "0644" + register: alertmanager_config - name: Установить конфигурацию alertmanager-ntfy ansible.builtin.template: @@ -49,6 +50,7 @@ owner: root group: root mode: "0644" + register: alertmanager_ntfy_config - name: Конфиг vmagent на core-mon ansible.builtin.template: diff --git a/ansible/roles/core-mon/monitoring/templates/alertmanager.yml.j2 b/ansible/roles/core-mon/monitoring/templates/alertmanager.yml.j2 index 2959853..da8c7f1 100644 --- a/ansible/roles/core-mon/monitoring/templates/alertmanager.yml.j2 +++ b/ansible/roles/core-mon/monitoring/templates/alertmanager.yml.j2 @@ -2,12 +2,21 @@ global: resolve_timeout: 5m route: - group_by: ['alertname', 'instance', 'severity'] + group_by: ['alertname', 'instance', 'gateway', 'severity'] group_wait: 30s group_interval: 5m repeat_interval: 4h receiver: 'ntfy' +inhibit_rules: + - source_matchers: + - alertname="OpnsenseGatewayDown" + - severity="critical" + target_matchers: + - alertname=~"OpnsenseGatewayPacketLoss|OpnsenseGatewayHighLatency" + - severity="warning" + equal: ['instance', 'gateway'] + receivers: - name: 'ntfy' webhook_configs: diff --git a/ansible/roles/core-mon/monitoring/templates/alerts.yml.j2 b/ansible/roles/core-mon/monitoring/templates/alerts.yml.j2 index e5ac5c3..bf43f61 100644 --- a/ansible/roles/core-mon/monitoring/templates/alerts.yml.j2 +++ b/ansible/roles/core-mon/monitoring/templates/alerts.yml.j2 @@ -245,24 +245,30 @@ groups: # Мониторим отвал только постоянных каналов (NorCom и MTC); WAN_0 (Iterika) работает по расписанию. - alert: OpnsenseGatewayDown expr: opnsense_gateway_status{gateway=~"NORCOM_DHCP|WAN1"} == 0 - for: 2m + for: 3m labels: severity: critical annotations: summary: "Провайдер OPNsense {{ $labels.gateway }} ({{ $labels.descr }}) недоступен" - description: "Шлюз {{ $labels.gateway }} (монитор {{ $labels.monitor }}) в статусе Offline более 2 минут." + description: "Шлюз {{ $labels.gateway }} (монитор {{ $labels.monitor }}) в статусе Offline более 3 минут." - alert: OpnsenseGatewayPacketLoss - expr: opnsense_gateway_loss_ratio{gateway=~"NORCOM_DHCP|WAN1"} > 0.1 - for: 3m + expr: | + opnsense_gateway_loss_ratio{gateway=~"NORCOM_DHCP|WAN1"} > 0.1 + and + opnsense_gateway_status{gateway=~"NORCOM_DHCP|WAN1"} == 1 + for: 5m labels: severity: warning annotations: summary: "Потери пакетов на {{ $labels.gateway }} ({{ $labels.descr }})" - description: "Потери на шлюзе {{ $labels.gateway }} превышают 10% более 3 минут." + description: "Потери на шлюзе {{ $labels.gateway }} превышают 10% более 5 минут." - alert: OpnsenseGatewayHighLatency - expr: opnsense_gateway_delay_seconds{gateway=~"NORCOM_DHCP|WAN1"} > 0.25 + expr: | + opnsense_gateway_delay_seconds{gateway=~"NORCOM_DHCP|WAN1"} > 0.25 + and + opnsense_gateway_status{gateway=~"NORCOM_DHCP|WAN1"} == 1 for: 5m labels: severity: warning diff --git a/ansible/roles/core-mon/monitoring/templates/compose.yaml.j2 b/ansible/roles/core-mon/monitoring/templates/compose.yaml.j2 index 7c3890c..e82e9c8 100644 --- a/ansible/roles/core-mon/monitoring/templates/compose.yaml.j2 +++ b/ansible/roles/core-mon/monitoring/templates/compose.yaml.j2 @@ -157,6 +157,8 @@ services: restart: unless-stopped command: - -datasource.url=http://vmsingle:8428 + - -remoteRead.url=http://vmsingle:8428 + - -remoteWrite.url=http://vmsingle:8428 - -notifier.url=http://alertmanager:9093 - -rule=/etc/vmalert/alerts.yml - -evaluationInterval=15s