From 408dfd09c338662e5b18603f1af2a2194f4cd544 Mon Sep 17 00:00:00 2001 From: karpenok Date: Thu, 12 Nov 2026 12:50:23 +0700 Subject: [PATCH] =?UTF-8?q?feat(monitoring):=20=D1=86=D0=B5=D0=BB=D0=B8=20?= =?UTF-8?q?Prometheus=20=D0=B8=D0=B7=20inventory=20=D0=B8=20=D1=85=D0=BE?= =?UTF-8?q?=D1=81=D1=82=D1=8B=20=D0=B2=20Grafana=20=D1=87=D0=B5=D1=80?= =?UTF-8?q?=D0=B5=D0=B7=20up?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Список scrape строится из inventory, дашборд показывает все цели, не только с успешным node_exporter. Co-authored-by: Cursor --- ansible/inventories/prod/hosts.yml | 1 + .../core-mon/monitoring/defaults/main.yml | 9 ++++++ .../files/dashboards/node-exporter-full.json | 12 +++---- .../core-mon/monitoring/handlers/main.yml | 26 ++++++++++++++++ .../roles/core-mon/monitoring/tasks/main.yml | 31 +++++++++++++++++++ .../monitoring/templates/prometheus.yml.j2 | 26 ++++------------ 6 files changed, 79 insertions(+), 26 deletions(-) create mode 100644 ansible/roles/core-mon/monitoring/handlers/main.yml diff --git a/ansible/inventories/prod/hosts.yml b/ansible/inventories/prod/hosts.yml index 593e867..188c336 100644 --- a/ansible/inventories/prod/hosts.yml +++ b/ansible/inventories/prod/hosts.yml @@ -1,4 +1,5 @@ # Машины. Состав каждой машины — в playbooks/<хост>.yml. +# Scrape Prometheus (node-exporter :9100): все записи ниже, кроме monitoring_scrape_exclude_hosts в group_vars/роли monitoring. all: hosts: core-net: diff --git a/ansible/roles/core-mon/monitoring/defaults/main.yml b/ansible/roles/core-mon/monitoring/defaults/main.yml index 73fc00d..e51a7ba 100644 --- a/ansible/roles/core-mon/monitoring/defaults/main.yml +++ b/ansible/roles/core-mon/monitoring/defaults/main.yml @@ -2,6 +2,15 @@ monitoring_dir: /opt/monitoring prometheus_retention: "30d" +# Цели node-exporter: все хосты из inventory, кроме исключений. +# Отключить scrape на хосте: host_vars monitoring_scrape_enabled: false +# Другой адрес/порт: monitoring_scrape_address, monitoring_node_exporter_port +monitoring_scrape_exclude_hosts: + - opnsense +monitoring_node_exporter_port: 9100 +monitoring_scrape_job_name: node-exporter +monitoring_env_label: prod + grafana_admin_user: "{{ lookup('ansible.builtin.env', 'ADMIN_USER', default='karpenok') }}" grafana_admin_password: "{{ lookup('ansible.builtin.env', 'ADMIN_PASSWORD', default='') }}" grafana_admin_email: "{{ lookup('ansible.builtin.env', 'MAIL_ADMIN_USER', default=grafana_admin_user ~ '@gunihome.ru') }}" diff --git a/ansible/roles/core-mon/monitoring/files/dashboards/node-exporter-full.json b/ansible/roles/core-mon/monitoring/files/dashboards/node-exporter-full.json index 80c15d9..e4809ff 100644 --- a/ansible/roles/core-mon/monitoring/files/dashboards/node-exporter-full.json +++ b/ansible/roles/core-mon/monitoring/files/dashboards/node-exporter-full.json @@ -15491,7 +15491,7 @@ "name": "job", "options": [], "query": { - "query": "label_values(node_uname_info, job)", + "query": "label_values(up, job)", "refId": "Prometheus-job-Variable-Query" }, "refresh": 1, @@ -15505,13 +15505,13 @@ "type": "prometheus", "uid": "prometheus" }, - "definition": "label_values(node_uname_info{job=\"$job\"}, host)", + "definition": "label_values(up{job=\"$job\"}, host)", "includeAll": false, "label": "Host", "name": "nodename", "options": [], "query": { - "query": "label_values(node_uname_info{job=\"$job\"}, host)", + "query": "label_values(up{job=\"$job\"}, host)", "refId": "Prometheus-nodename-Variable-Query" }, "refresh": 1, @@ -15525,13 +15525,13 @@ "type": "prometheus", "uid": "prometheus" }, - "definition": "label_values(node_uname_info{job=\"$job\", host=\"$nodename\"}, instance)", + "definition": "label_values(up{job=\"$job\", host=\"$nodename\"}, instance)", "includeAll": false, "label": "Instance", "name": "node", "options": [], "query": { - "query": "label_values(node_uname_info{job=\"$job\", host=\"$nodename\"}, instance)", + "query": "label_values(up{job=\"$job\", host=\"$nodename\"}, instance)", "refId": "Prometheus-node-Variable-Query" }, "refresh": 1, @@ -15549,7 +15549,7 @@ "timezone": "browser", "title": "Node Exporter Full", "uid": "rYdddlPWk", - "version": 105, + "version": 106, "weekStart": "", "gnetId": 1860 } \ No newline at end of file diff --git a/ansible/roles/core-mon/monitoring/handlers/main.yml b/ansible/roles/core-mon/monitoring/handlers/main.yml new file mode 100644 index 0000000..e70227d --- /dev/null +++ b/ansible/roles/core-mon/monitoring/handlers/main.yml @@ -0,0 +1,26 @@ +--- +- name: Reload Prometheus configuration + ansible.builtin.uri: + url: http://127.0.0.1:9090/-/reload + method: POST + status_code: 200 + register: prometheus_reload + until: prometheus_reload.status == 200 + retries: 10 + delay: 3 + when: prometheus_config_reload | default(true) | bool + ignore_errors: true + +- name: Restart Prometheus after failed reload + ansible.builtin.command: docker compose -f {{ monitoring_dir }}/compose.yaml restart prometheus + args: + chdir: "{{ monitoring_dir }}" + when: + - prometheus_config_reload | default(true) | bool + - prometheus_reload is defined + - prometheus_reload is failed or (prometheus_reload.status | default(0) != 200) + +- name: Restart Grafana + ansible.builtin.command: docker compose -f {{ monitoring_dir }}/compose.yaml restart grafana + args: + chdir: "{{ monitoring_dir }}" diff --git a/ansible/roles/core-mon/monitoring/tasks/main.yml b/ansible/roles/core-mon/monitoring/tasks/main.yml index ca58ec4..cd19b6d 100644 --- a/ansible/roles/core-mon/monitoring/tasks/main.yml +++ b/ansible/roles/core-mon/monitoring/tasks/main.yml @@ -20,6 +20,34 @@ - { path: "{{ monitoring_dir }}/ntfy_data", owner: "root", group: "root", mode: "0777" } - { path: "{{ monitoring_dir }}/alertmanager-ntfy", owner: "root", group: "root", mode: "0755" } +- name: Сбросить список целей scrape node-exporter + ansible.builtin.set_fact: + monitoring_scrape_targets: [] + +- name: Собрать цели scrape node-exporter из inventory + ansible.builtin.set_fact: + monitoring_scrape_targets: "{{ monitoring_scrape_targets + [target] }}" + vars: + target: + name: "{{ item }}" + address: "{{ hostvars[item].monitoring_scrape_address | default(hostvars[item].ansible_host) }}" + port: "{{ hostvars[item].monitoring_node_exporter_port | default(monitoring_node_exporter_port) }}" + env: "{{ hostvars[item].monitoring_env | default(monitoring_env_label) }}" + loop: "{{ groups['all'] | sort }}" + when: + - item not in monitoring_scrape_exclude_hosts + - hostvars[item].monitoring_scrape_enabled | default(true) | bool + +- name: Проверить, что в Prometheus есть хотя бы одна цель node-exporter + ansible.builtin.assert: + that: + - monitoring_scrape_targets | length > 0 + fail_msg: "Нет целей scrape: проверьте inventory и monitoring_scrape_exclude_hosts." + +- name: Показать цели scrape node-exporter + ansible.builtin.debug: + msg: "{{ monitoring_scrape_targets | map(attribute='name') | list | sort }}" + - name: Установить конфигурацию Prometheus ansible.builtin.template: src: prometheus.yml.j2 @@ -27,6 +55,7 @@ owner: root group: root mode: "0644" + notify: Reload Prometheus configuration - name: Установить правила алертов Prometheus ansible.builtin.copy: @@ -35,6 +64,7 @@ owner: root group: root mode: "0644" + notify: Reload Prometheus configuration - name: Установить конфигурацию Alertmanager ansible.builtin.template: @@ -75,6 +105,7 @@ owner: root group: root mode: "0644" + notify: Restart Grafana - name: Установить compose.yaml для стека мониторинга ansible.builtin.template: diff --git a/ansible/roles/core-mon/monitoring/templates/prometheus.yml.j2 b/ansible/roles/core-mon/monitoring/templates/prometheus.yml.j2 index 0227e65..381b11e 100644 --- a/ansible/roles/core-mon/monitoring/templates/prometheus.yml.j2 +++ b/ansible/roles/core-mon/monitoring/templates/prometheus.yml.j2 @@ -16,26 +16,12 @@ scrape_configs: static_configs: - targets: ["localhost:9090"] - - job_name: "node-exporter" + - job_name: "{{ monitoring_scrape_job_name }}" static_configs: +{% for target in monitoring_scrape_targets %} - targets: - - "192.168.1.108:9100" + - "{{ target.address }}:{{ target.port }}" labels: - host: "core-mon" - env: "prod" - - targets: - - "192.168.1.104:9100" - labels: - host: "core-net" - env: "prod" - - targets: - - "192.168.1.102:9100" - labels: - host: "core-secrets" - env: "prod" - - targets: - - "45.154.185.208:9100" - labels: - host: "rus-central" - env: "prod" - + host: "{{ target.name }}" + env: "{{ target.env }}" +{% endfor %}