From 57aa12b9c5ab505c7985d8300b81c413c28852cf Mon Sep 17 00:00:00 2001 From: karpenok Date: Thu, 12 Nov 2026 17:54:25 +0700 Subject: [PATCH] =?UTF-8?q?feat(monitoring):=20=D0=B7=D0=B0=D0=BC=D0=B5?= =?UTF-8?q?=D0=BD=D0=B8=D1=82=D1=8C=20Prometheus=20=D0=BD=D0=B0=20vmalert?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Алерты считаются по VictoriaMetrics и уходят в тот же Alertmanager, второй сбор метрик больше не нужен. Co-authored-by: Cursor --- BOOTSTRAP.md | 6 +-- ansible/inventories/prod/hosts.yml | 6 ++- ansible/playbooks/core-mon.yml | 2 +- .../core-mon/monitoring/defaults/main.yml | 9 +++-- .../core-mon/monitoring/files/alerts.yml | 2 +- .../roles/core-mon/monitoring/tasks/main.yml | 34 +++++++--------- .../monitoring/templates/compose.yaml.j2 | 39 ++++++++++--------- .../templates/grafana-datasources.yml.j2 | 6 +-- .../monitoring/templates/prometheus.yml.j2 | 27 ------------- .../templates/vmagent-nodeexporter.yml.j2 | 7 ++++ 10 files changed, 59 insertions(+), 79 deletions(-) delete mode 100644 ansible/roles/core-mon/monitoring/templates/prometheus.yml.j2 diff --git a/BOOTSTRAP.md b/BOOTSTRAP.md index c3f46c9..005d5c1 100644 --- a/BOOTSTRAP.md +++ b/BOOTSTRAP.md @@ -8,7 +8,7 @@ |---|---|---|---| | `core-net` | `playbooks/core-net.yml` | `roles/core-net/traefik`; `roles/core-net/netbird`; `roles/core-net/remnawave`; `roles/core-net/vmagent`; `roles/core-net/authentik` *(опц., по умолчанию выкл.)* | `/opt/traefik`; клиент NetBird; `/opt/remnawave`; `/opt/vmagent`; `/opt/authentik` *(если включён)* | | `core-secrets` | `playbooks/core-secrets.yml` | `roles/core-secrets/infisical`, `roles/core-secrets/vaultwarden` | `/opt/infisical`, `/opt/vaultwarden` | -| `core-mon` | `playbooks/core-mon.yml` | `roles/core-mon/monitoring` | `/opt/monitoring` (Prometheus и VictoriaMetrics) | +| `core-mon` | `playbooks/core-mon.yml` | `roles/core-mon/monitoring` | `/opt/monitoring` (VictoriaMetrics, vmalert, Grafana) | | `rus-central` | `playbooks/rus-central.yml` | `roles/rus-central/netbird` | `/opt/netbird` | | `ger-nod-02` | `playbooks/ger-nod-02.yml` | `roles/ger-nod-02/netbird`; `roles/ger-nod-02/remnanode`; `roles/ger-nod-02/vmagent` | клиент NetBird; `/opt/remnanode`; `/opt/vmagent` | | `opnsense` | `playbooks/opnsense.yml` | `roles/opnsense/netbird` | REST API (`192.168.1.1`) | @@ -34,7 +34,7 @@ Terraform в этом репозитории описывает только у | **CT 109** | `pve` | `192.168.1.217` | `gitea` | Gitea (`:3000`), Postgres (`:5432`), Gitea Actions Runner (`dind`) | | **CT 901** | `netcore` | `192.168.1.102` | `core-secrets` | Infisical (`:8080`), Postgres, Redis, Vaultwarden (`:8000`) | | **CT 902** | `netcore` | `192.168.1.104` | `core-net` | Traefik (`:80`, `:443`); Remnawave Panel (`127.0.0.1:3000`) + Subscription (`127.0.0.1:3010`); клиент NetBird (нужен `/dev/net/tun`); Authentik (`127.0.0.1:9000`) — опционально, по умолчанию выкл. | -| **CT 903** | `netcore` | `192.168.1.108` | `core-mon` | Prometheus (`:9090`), VictoriaMetrics (`:8428`), Alertmanager (`:9093`), Grafana (`:3000`), Ntfy (`:80`) | +| **CT 903** | `netcore` | `192.168.1.108` | `core-mon` | VictoriaMetrics (`:8428`), vmalert (`127.0.0.1:8880`), Alertmanager (`:9093`), Grafana (`:3000`), Ntfy (`:80`) | ### 1.3. Внешние узлы | Имя | IP | Роль | Стек | @@ -163,7 +163,7 @@ API Endpoint (внутренний LAN): `http://192.168.1.102:8080/api` 1. После успешного `ansible-core-net` (чтобы в Infisical был `REMNANODE_SECRET_KEY`) запустить **`ansible-ger-nod-02`**. 2. Первый прогон: как у rus-central — `ansible_password` из `/VPS` → `PASS_GER_NOD_02`; роль `host` кладёт ключ и отключает пароль. Роль `remnanode` держит A-запись `yali.gunis.top` в Cloudflare, выпускает LE по **HTTP-01** (порт 80, без TXT `_acme-challenge` в зоне), поднимает remnanode + nginx SelfSteal. 3. В панели нода должна стать зелёной; браузер на `https://yali.gunis.top` — маскировочный сайт. -4. Клиент NetBird ходит на `https://central.gunihome.ru` ключом `remnawave-peers`. Пока peer панели не виден, `NODE_PORT` 2222 остаётся открыт для WAN egress панели. Когда peer `core-net` есть, 2222 разрешён только с его NetBird IP. Публичный `:9100` закрыт: метрики ноды уходят vmagent-ом в VictoriaMetrics на `192.168.1.108:8428` через маршрут `homelab-lan`. В Prometheus эта нода больше не входит. +4. Клиент NetBird ходит на `https://central.gunihome.ru` ключом `remnawave-peers`. Пока peer панели не виден, `NODE_PORT` 2222 остаётся открыт для WAN egress панели. Когда peer `core-net` есть, 2222 разрешён только с его NetBird IP. Публичный `:9100` закрыт: метрики ноды уходят vmagent-ом в VictoriaMetrics на `192.168.1.108:8428` через маршрут `homelab-lan`. 5. На `core-net` (unprivileged LXC) `/dev/net/tun` сам не появляется. Токен Terraform получает 403: проброс устройства разрешён только `root@pam`. На netcore: `pct set 902 -dev0 /dev/net/tun,mode=0666 && pct reboot 902`, затем ещё раз `ansible-core-net`. До этого панель остаётся на публичном адресе ноды. ### Шаг 5. NetBird (rus-central) и связка с OPNsense diff --git a/ansible/inventories/prod/hosts.yml b/ansible/inventories/prod/hosts.yml index 2baf51f..92ba4d1 100644 --- a/ansible/inventories/prod/hosts.yml +++ b/ansible/inventories/prod/hosts.yml @@ -1,6 +1,8 @@ # Машины. Состав каждой машины — в playbooks/<хост>.yml. -# Scrape Prometheus (node-exporter :9100): все записи ниже, кроме monitoring_scrape_exclude_hosts -# и хостов с monitoring_scrape_enabled: false (ger-nod-02 отдаёт метрики через vmagent). +# Scrape node-exporter (:9100) центральным vmagent: все записи ниже, +# кроме monitoring_scrape_exclude_hosts, хостов с monitoring_scrape_enabled: false +# и хостов из vmagent_central_scrape_skip (у них свой vmagent). +# ger-nod-02 отдаёт метрики своим vmagent. all: hosts: core-net: diff --git a/ansible/playbooks/core-mon.yml b/ansible/playbooks/core-mon.yml index 54a1c07..8f063cc 100644 --- a/ansible/playbooks/core-mon.yml +++ b/ansible/playbooks/core-mon.yml @@ -1,5 +1,5 @@ --- -# Playbook for core-mon (Monitoring stack: Prometheus, Alertmanager, Grafana, Ntfy) +# Playbook for core-mon (Monitoring stack: VictoriaMetrics, vmalert, Alertmanager, Grafana, Ntfy) - name: Развертывание стека мониторинга на core-mon hosts: core-mon roles: diff --git a/ansible/roles/core-mon/monitoring/defaults/main.yml b/ansible/roles/core-mon/monitoring/defaults/main.yml index cdcce80..e69dcbf 100644 --- a/ansible/roles/core-mon/monitoring/defaults/main.yml +++ b/ansible/roles/core-mon/monitoring/defaults/main.yml @@ -1,6 +1,6 @@ --- monitoring_dir: /opt/monitoring -prometheus_retention: "30d" +victoriametrics_retention: "30d" # Цели node-exporter: все хосты из inventory, кроме исключений. # Отключить scrape на хосте: host_vars monitoring_scrape_enabled: false @@ -8,11 +8,14 @@ prometheus_retention: "30d" monitoring_scrape_exclude_hosts: - opnsense monitoring_node_exporter_port: 9100 -monitoring_scrape_job_name: node-exporter -monitoring_env_label: prod victoriametrics_image: victoriametrics/victoria-metrics:v1.153.0 +vmalert_image: victoriametrics/vmalert:v1.153.0 vmagent_image: victoriametrics/vmagent:v1.153.0 +# Свой vmagent уже пишет node-exporter этих хостов. Повторный scrape даст двойные сэмплы. +vmagent_central_scrape_skip: + - core-mon + - core-net cadvisor_image: gcr.io/cadvisor/cadvisor:v0.49.2 xray_checker_image: kutovoys/xray-checker:v1.3.1 xray_checker_subscription_url: "{{ lookup('env', 'XRAY_CHECKER_SUBSCRIPTION_URL', default='') }}" diff --git a/ansible/roles/core-mon/monitoring/files/alerts.yml b/ansible/roles/core-mon/monitoring/files/alerts.yml index 8c1593d..c0eae0d 100644 --- a/ansible/roles/core-mon/monitoring/files/alerts.yml +++ b/ansible/roles/core-mon/monitoring/files/alerts.yml @@ -2,7 +2,7 @@ groups: - name: homelab_alerts rules: - alert: HostDown - expr: up == 0 + expr: up{job="integrations/node_exporter"} == 0 for: 2m labels: severity: critical diff --git a/ansible/roles/core-mon/monitoring/tasks/main.yml b/ansible/roles/core-mon/monitoring/tasks/main.yml index 477ce0b..23c9a32 100644 --- a/ansible/roles/core-mon/monitoring/tasks/main.yml +++ b/ansible/roles/core-mon/monitoring/tasks/main.yml @@ -8,8 +8,6 @@ mode: "{{ item.mode }}" loop: - { path: "{{ monitoring_dir }}", owner: "root", group: "root", mode: "0755" } - - { path: "{{ monitoring_dir }}/prometheus", owner: "root", group: "root", mode: "0755" } - - { path: "{{ monitoring_dir }}/prometheus_data", owner: "65534", group: "65534", mode: "0777" } - { path: "{{ monitoring_dir }}/alertmanager", owner: "root", group: "root", mode: "0755" } - { path: "{{ monitoring_dir }}/alertmanager_data", owner: "65534", group: "65534", mode: "0777" } - { path: "{{ monitoring_dir }}/grafana", owner: "root", group: "root", mode: "0755" } @@ -22,6 +20,7 @@ - { path: "{{ monitoring_dir }}/victoria-metrics-data", owner: "65534", group: "65534", mode: "0775" } - { path: "{{ monitoring_dir }}/vmagent", owner: "root", group: "root", mode: "0755" } - { path: "{{ monitoring_dir }}/vmagent/conf.d", owner: "root", group: "root", mode: "0755" } + - { path: "{{ monitoring_dir }}/vmalert", owner: "root", group: "root", mode: "0755" } - name: Сбросить список целей scrape node-exporter ansible.builtin.set_fact: @@ -35,13 +34,12 @@ name: "{{ item }}" address: "{{ hostvars[item].monitoring_scrape_address | default(hostvars[item].ansible_host) }}" port: "{{ hostvars[item].monitoring_node_exporter_port | default(monitoring_node_exporter_port) }}" - env: "{{ hostvars[item].monitoring_env | default(monitoring_env_label) }}" loop: "{{ groups['all'] | sort }}" when: - item not in monitoring_scrape_exclude_hosts - hostvars[item].monitoring_scrape_enabled | default(true) | bool -- name: Проверить, что в Prometheus есть хотя бы одна цель node-exporter +- name: Проверить, что есть хотя бы одна цель node-exporter ansible.builtin.assert: that: - monitoring_scrape_targets | length > 0 @@ -51,23 +49,13 @@ ansible.builtin.debug: msg: "{{ monitoring_scrape_targets | map(attribute='name') | list | sort }}" -- name: Установить конфигурацию Prometheus - ansible.builtin.template: - src: prometheus.yml.j2 - dest: "{{ monitoring_dir }}/prometheus/prometheus.yml" - owner: root - group: root - mode: "0644" - register: prometheus_config - -- name: Установить правила алертов Prometheus +- name: Установить правила алертов vmalert ansible.builtin.copy: src: alerts.yml - dest: "{{ monitoring_dir }}/prometheus/alerts.yml" + dest: "{{ monitoring_dir }}/vmalert/alerts.yml" owner: root group: root mode: "0644" - register: prometheus_alerts - name: Установить конфигурацию Alertmanager ansible.builtin.template: @@ -177,11 +165,6 @@ when: xray_checker_env is changed changed_when: true -- name: Пересоздать Prometheus, чтобы подхватить конфиг - ansible.builtin.command: docker compose -f {{ monitoring_dir }}/compose.yaml up -d --force-recreate --no-deps prometheus - when: prometheus_config is changed or prometheus_alerts is changed or monitoring_compose is changed - changed_when: true - - name: Дождаться готовности Ntfy ansible.builtin.uri: url: "http://127.0.0.1:80/v1/health" @@ -217,6 +200,15 @@ delay: 2 until: victoriametrics_health.status == 200 +- name: Дождаться vmalert + ansible.builtin.uri: + url: "http://127.0.0.1:8880/health" + status_code: 200 + register: vmalert_health + retries: 20 + delay: 2 + until: vmalert_health.status == 200 + - name: Перезапустить Grafana, чтобы подхватить datasource ansible.builtin.command: docker compose -f {{ monitoring_dir }}/compose.yaml restart grafana when: grafana_datasources is changed or grafana_dashboards is changed or grafana_dashboard_provider is changed diff --git a/ansible/roles/core-mon/monitoring/templates/compose.yaml.j2 b/ansible/roles/core-mon/monitoring/templates/compose.yaml.j2 index ff96675..a0b5cf4 100644 --- a/ansible/roles/core-mon/monitoring/templates/compose.yaml.j2 +++ b/ansible/roles/core-mon/monitoring/templates/compose.yaml.j2 @@ -1,23 +1,6 @@ name: monitoring services: - prometheus: - image: prom/prometheus:v3.2.1 - container_name: monitoring-prometheus - restart: unless-stopped - command: - - "--config.file=/etc/prometheus/prometheus.yml" - - "--storage.tsdb.path=/prometheus" - - "--storage.tsdb.retention.time={{ prometheus_retention }}" - - "--web.enable-lifecycle" - volumes: - - ./prometheus:/etc/prometheus:ro - - ./prometheus_data:/prometheus - ports: - - "9090:9090" - depends_on: - - alertmanager - alertmanager: image: prom/alertmanager:v0.28.1 container_name: monitoring-alertmanager @@ -76,7 +59,7 @@ services: ports: - "3000:3000" depends_on: - - prometheus + - vmsingle vmsingle: image: {{ victoriametrics_image }} @@ -85,6 +68,7 @@ services: restart: unless-stopped command: - --storageDataPath=/victoria-metrics-data + - --retentionPeriod={{ victoriametrics_retention }} volumes: - ./victoria-metrics-data:/victoria-metrics-data ports: @@ -138,4 +122,23 @@ services: - vmsingle - cadvisor + vmalert: + image: {{ vmalert_image }} + container_name: monitoring-vmalert + restart: unless-stopped + command: + - -datasource.url=http://vmsingle:8428 + - -notifier.url=http://alertmanager:9093 + - -rule=/etc/vmalert/alerts.yml + - -evaluationInterval=15s + - -configCheckInterval=30s + - -httpListenAddr=0.0.0.0:8880 + volumes: + - ./vmalert:/etc/vmalert:ro + ports: + - "127.0.0.1:8880:8880" + depends_on: + - vmsingle + - alertmanager + diff --git a/ansible/roles/core-mon/monitoring/templates/grafana-datasources.yml.j2 b/ansible/roles/core-mon/monitoring/templates/grafana-datasources.yml.j2 index 2d16332..69bdca8 100644 --- a/ansible/roles/core-mon/monitoring/templates/grafana-datasources.yml.j2 +++ b/ansible/roles/core-mon/monitoring/templates/grafana-datasources.yml.j2 @@ -5,8 +5,8 @@ datasources: uid: prometheus type: prometheus access: proxy - url: http://prometheus:9090 - isDefault: true + url: http://vmsingle:8428 + isDefault: false jsonData: timeInterval: 15s @@ -15,7 +15,7 @@ datasources: type: prometheus access: proxy url: http://vmsingle:8428 - isDefault: false + isDefault: true jsonData: timeInterval: 15s diff --git a/ansible/roles/core-mon/monitoring/templates/prometheus.yml.j2 b/ansible/roles/core-mon/monitoring/templates/prometheus.yml.j2 deleted file mode 100644 index 381b11e..0000000 --- a/ansible/roles/core-mon/monitoring/templates/prometheus.yml.j2 +++ /dev/null @@ -1,27 +0,0 @@ -global: - scrape_interval: 15s - evaluation_interval: 15s - -rule_files: - - "/etc/prometheus/alerts.yml" - -alerting: - alertmanagers: - - static_configs: - - targets: - - "alertmanager:9093" - -scrape_configs: - - job_name: "prometheus" - static_configs: - - targets: ["localhost:9090"] - - - job_name: "{{ monitoring_scrape_job_name }}" - static_configs: -{% for target in monitoring_scrape_targets %} - - targets: - - "{{ target.address }}:{{ target.port }}" - labels: - host: "{{ target.name }}" - env: "{{ target.env }}" -{% endfor %} diff --git a/ansible/roles/core-mon/monitoring/templates/vmagent-nodeexporter.yml.j2 b/ansible/roles/core-mon/monitoring/templates/vmagent-nodeexporter.yml.j2 index 2001b92..3830267 100644 --- a/ansible/roles/core-mon/monitoring/templates/vmagent-nodeexporter.yml.j2 +++ b/ansible/roles/core-mon/monitoring/templates/vmagent-nodeexporter.yml.j2 @@ -5,3 +5,10 @@ labels: instance: "{{ inventory_hostname }}" cluster: "{{ vmagent_cluster }}" +{% for target in monitoring_scrape_targets if target.name not in vmagent_central_scrape_skip %} + - targets: ["{{ target.address }}:{{ target.port }}"] + labels: + instance: "{{ target.name }}" + cluster: "{{ vmagent_cluster }}" + host: "{{ target.name }}" +{% endfor %}