feat(monitoring): заменить Prometheus на vmalert
ansible-core-mon / deploy (push) Successful in 37s
ansible-core-net / deploy (push) Successful in 56s
ansible-core-secrets / deploy (push) Successful in 16s
ansible-ger-nod-02 / deploy (push) Successful in 1m22s
ansible-opnsense / deploy (push) Successful in 6s
ansible-rus-central / deploy (push) Successful in 39s

Алерты считаются по VictoriaMetrics и уходят в тот же Alertmanager, второй сбор метрик больше не нужен.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
karpenokandCursor committed 2026-11-12 17:54:25 +07:00
1 parent 28b8bcf708
commit 57aa12b9c5
10 files changed
+59 -79

No files matched your search

+4 -2
View File
@@ -1,6 +1,8 @@
# Машины. Состав каждой машины — в playbooks/<хост>.yml.
# Scrape Prometheus (node-exporter :9100): все записи ниже, кроме monitoring_scrape_exclude_hosts
# и хостов с monitoring_scrape_enabled: false (ger-nod-02 отдаёт метрики через vmagent).
# Scrape node-exporter (:9100) центральным vmagent: все записи ниже,
# кроме monitoring_scrape_exclude_hosts, хостов с monitoring_scrape_enabled: false
# и хостов из vmagent_central_scrape_skip (у них свой vmagent).
# ger-nod-02 отдаёт метрики своим vmagent.
all:
hosts:
core-net:
+1 -1
View File
@@ -1,5 +1,5 @@
---
# Playbook for core-mon (Monitoring stack: Prometheus, Alertmanager, Grafana, Ntfy)
# Playbook for core-mon (Monitoring stack: VictoriaMetrics, vmalert, Alertmanager, Grafana, Ntfy)
- name: Развертывание стека мониторинга на core-mon
hosts: core-mon
roles:
@@ -1,6 +1,6 @@
---
monitoring_dir: /opt/monitoring
prometheus_retention: "30d"
victoriametrics_retention: "30d"
# Цели node-exporter: все хосты из inventory, кроме исключений.
# Отключить scrape на хосте: host_vars monitoring_scrape_enabled: false
@@ -8,11 +8,14 @@ prometheus_retention: "30d"
monitoring_scrape_exclude_hosts:
- opnsense
monitoring_node_exporter_port: 9100
monitoring_scrape_job_name: node-exporter
monitoring_env_label: prod
victoriametrics_image: victoriametrics/victoria-metrics:v1.153.0
vmalert_image: victoriametrics/vmalert:v1.153.0
vmagent_image: victoriametrics/vmagent:v1.153.0
# Свой vmagent уже пишет node-exporter этих хостов. Повторный scrape даст двойные сэмплы.
vmagent_central_scrape_skip:
- core-mon
- core-net
cadvisor_image: gcr.io/cadvisor/cadvisor:v0.49.2
xray_checker_image: kutovoys/xray-checker:v1.3.1
xray_checker_subscription_url: "{{ lookup('env', 'XRAY_CHECKER_SUBSCRIPTION_URL', default='') }}"
@@ -2,7 +2,7 @@ groups:
- name: homelab_alerts
rules:
- alert: HostDown
expr: up == 0
expr: up{job="integrations/node_exporter"} == 0
for: 2m
labels:
severity: critical
@@ -8,8 +8,6 @@
mode: "{{ item.mode }}"
loop:
- { path: "{{ monitoring_dir }}", owner: "root", group: "root", mode: "0755" }
- { path: "{{ monitoring_dir }}/prometheus", owner: "root", group: "root", mode: "0755" }
- { path: "{{ monitoring_dir }}/prometheus_data", owner: "65534", group: "65534", mode: "0777" }
- { path: "{{ monitoring_dir }}/alertmanager", owner: "root", group: "root", mode: "0755" }
- { path: "{{ monitoring_dir }}/alertmanager_data", owner: "65534", group: "65534", mode: "0777" }
- { path: "{{ monitoring_dir }}/grafana", owner: "root", group: "root", mode: "0755" }
@@ -22,6 +20,7 @@
- { path: "{{ monitoring_dir }}/victoria-metrics-data", owner: "65534", group: "65534", mode: "0775" }
- { path: "{{ monitoring_dir }}/vmagent", owner: "root", group: "root", mode: "0755" }
- { path: "{{ monitoring_dir }}/vmagent/conf.d", owner: "root", group: "root", mode: "0755" }
- { path: "{{ monitoring_dir }}/vmalert", owner: "root", group: "root", mode: "0755" }
- name: Сбросить список целей scrape node-exporter
ansible.builtin.set_fact:
@@ -35,13 +34,12 @@
name: "{{ item }}"
address: "{{ hostvars[item].monitoring_scrape_address | default(hostvars[item].ansible_host) }}"
port: "{{ hostvars[item].monitoring_node_exporter_port | default(monitoring_node_exporter_port) }}"
env: "{{ hostvars[item].monitoring_env | default(monitoring_env_label) }}"
loop: "{{ groups['all'] | sort }}"
when:
- item not in monitoring_scrape_exclude_hosts
- hostvars[item].monitoring_scrape_enabled | default(true) | bool
- name: Проверить, что в Prometheus есть хотя бы одна цель node-exporter
- name: Проверить, что есть хотя бы одна цель node-exporter
ansible.builtin.assert:
that:
- monitoring_scrape_targets | length > 0
@@ -51,23 +49,13 @@
ansible.builtin.debug:
msg: "{{ monitoring_scrape_targets | map(attribute='name') | list | sort }}"
- name: Установить конфигурацию Prometheus
ansible.builtin.template:
src: prometheus.yml.j2
dest: "{{ monitoring_dir }}/prometheus/prometheus.yml"
owner: root
group: root
mode: "0644"
register: prometheus_config
- name: Установить правила алертов Prometheus
- name: Установить правила алертов vmalert
ansible.builtin.copy:
src: alerts.yml
dest: "{{ monitoring_dir }}/prometheus/alerts.yml"
dest: "{{ monitoring_dir }}/vmalert/alerts.yml"
owner: root
group: root
mode: "0644"
register: prometheus_alerts
- name: Установить конфигурацию Alertmanager
ansible.builtin.template:
@@ -177,11 +165,6 @@
when: xray_checker_env is changed
changed_when: true
- name: Пересоздать Prometheus, чтобы подхватить конфиг
ansible.builtin.command: docker compose -f {{ monitoring_dir }}/compose.yaml up -d --force-recreate --no-deps prometheus
when: prometheus_config is changed or prometheus_alerts is changed or monitoring_compose is changed
changed_when: true
- name: Дождаться готовности Ntfy
ansible.builtin.uri:
url: "http://127.0.0.1:80/v1/health"
@@ -217,6 +200,15 @@
delay: 2
until: victoriametrics_health.status == 200
- name: Дождаться vmalert
ansible.builtin.uri:
url: "http://127.0.0.1:8880/health"
status_code: 200
register: vmalert_health
retries: 20
delay: 2
until: vmalert_health.status == 200
- name: Перезапустить Grafana, чтобы подхватить datasource
ansible.builtin.command: docker compose -f {{ monitoring_dir }}/compose.yaml restart grafana
when: grafana_datasources is changed or grafana_dashboards is changed or grafana_dashboard_provider is changed
@@ -1,23 +1,6 @@
name: monitoring
services:
prometheus:
image: prom/prometheus:v3.2.1
container_name: monitoring-prometheus
restart: unless-stopped
command:
- "--config.file=/etc/prometheus/prometheus.yml"
- "--storage.tsdb.path=/prometheus"
- "--storage.tsdb.retention.time={{ prometheus_retention }}"
- "--web.enable-lifecycle"
volumes:
- ./prometheus:/etc/prometheus:ro
- ./prometheus_data:/prometheus
ports:
- "9090:9090"
depends_on:
- alertmanager
alertmanager:
image: prom/alertmanager:v0.28.1
container_name: monitoring-alertmanager
@@ -76,7 +59,7 @@ services:
ports:
- "3000:3000"
depends_on:
- prometheus
- vmsingle
vmsingle:
image: {{ victoriametrics_image }}
@@ -85,6 +68,7 @@ services:
restart: unless-stopped
command:
- --storageDataPath=/victoria-metrics-data
- --retentionPeriod={{ victoriametrics_retention }}
volumes:
- ./victoria-metrics-data:/victoria-metrics-data
ports:
@@ -138,4 +122,23 @@ services:
- vmsingle
- cadvisor
vmalert:
image: {{ vmalert_image }}
container_name: monitoring-vmalert
restart: unless-stopped
command:
- -datasource.url=http://vmsingle:8428
- -notifier.url=http://alertmanager:9093
- -rule=/etc/vmalert/alerts.yml
- -evaluationInterval=15s
- -configCheckInterval=30s
- -httpListenAddr=0.0.0.0:8880
volumes:
- ./vmalert:/etc/vmalert:ro
ports:
- "127.0.0.1:8880:8880"
depends_on:
- vmsingle
- alertmanager
@@ -5,8 +5,8 @@ datasources:
uid: prometheus
type: prometheus
access: proxy
url: http://prometheus:9090
isDefault: true
url: http://vmsingle:8428
isDefault: false
jsonData:
timeInterval: 15s
@@ -15,7 +15,7 @@ datasources:
type: prometheus
access: proxy
url: http://vmsingle:8428
isDefault: false
isDefault: true
jsonData:
timeInterval: 15s
@@ -1,27 +0,0 @@
global:
scrape_interval: 15s
evaluation_interval: 15s
rule_files:
- "/etc/prometheus/alerts.yml"
alerting:
alertmanagers:
- static_configs:
- targets:
- "alertmanager:9093"
scrape_configs:
- job_name: "prometheus"
static_configs:
- targets: ["localhost:9090"]
- job_name: "{{ monitoring_scrape_job_name }}"
static_configs:
{% for target in monitoring_scrape_targets %}
- targets:
- "{{ target.address }}:{{ target.port }}"
labels:
host: "{{ target.name }}"
env: "{{ target.env }}"
{% endfor %}
@@ -5,3 +5,10 @@
labels:
instance: "{{ inventory_hostname }}"
cluster: "{{ vmagent_cluster }}"
{% for target in monitoring_scrape_targets if target.name not in vmagent_central_scrape_skip %}
- targets: ["{{ target.address }}:{{ target.port }}"]
labels:
instance: "{{ target.name }}"
cluster: "{{ vmagent_cluster }}"
host: "{{ target.name }}"
{% endfor %}