feat(monitoring): цели Prometheus из inventory и хосты в Grafana через up
ansible-core-mon / deploy (push) Successful in 26s
ansible-core-net / deploy (push) Successful in 43s
ansible-core-secrets / deploy (push) Successful in 15s
ansible-ger-nod-02 / deploy (push) Successful in 55s
ansible-opnsense / deploy (push) Successful in 5s
ansible-rus-central / deploy (push) Successful in 59s
ansible-core-mon / deploy (push) Successful in 26s
ansible-core-net / deploy (push) Successful in 43s
ansible-core-secrets / deploy (push) Successful in 15s
ansible-ger-nod-02 / deploy (push) Successful in 55s
ansible-opnsense / deploy (push) Successful in 5s
ansible-rus-central / deploy (push) Successful in 59s
Список scrape строится из inventory, дашборд показывает все цели, не только с успешным node_exporter. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
4316f2c68e
commit
408dfd09c3
6 files changed
+79
-26
No files matched your search
@@ -1,4 +1,5 @@
|
|||||||
# Машины. Состав каждой машины — в playbooks/<хост>.yml.
|
# Машины. Состав каждой машины — в playbooks/<хост>.yml.
|
||||||
|
# Scrape Prometheus (node-exporter :9100): все записи ниже, кроме monitoring_scrape_exclude_hosts в group_vars/роли monitoring.
|
||||||
all:
|
all:
|
||||||
hosts:
|
hosts:
|
||||||
core-net:
|
core-net:
|
||||||
|
|||||||
@@ -2,6 +2,15 @@
|
|||||||
monitoring_dir: /opt/monitoring
|
monitoring_dir: /opt/monitoring
|
||||||
prometheus_retention: "30d"
|
prometheus_retention: "30d"
|
||||||
|
|
||||||
|
# Цели node-exporter: все хосты из inventory, кроме исключений.
|
||||||
|
# Отключить scrape на хосте: host_vars monitoring_scrape_enabled: false
|
||||||
|
# Другой адрес/порт: monitoring_scrape_address, monitoring_node_exporter_port
|
||||||
|
monitoring_scrape_exclude_hosts:
|
||||||
|
- opnsense
|
||||||
|
monitoring_node_exporter_port: 9100
|
||||||
|
monitoring_scrape_job_name: node-exporter
|
||||||
|
monitoring_env_label: prod
|
||||||
|
|
||||||
grafana_admin_user: "{{ lookup('ansible.builtin.env', 'ADMIN_USER', default='karpenok') }}"
|
grafana_admin_user: "{{ lookup('ansible.builtin.env', 'ADMIN_USER', default='karpenok') }}"
|
||||||
grafana_admin_password: "{{ lookup('ansible.builtin.env', 'ADMIN_PASSWORD', default='') }}"
|
grafana_admin_password: "{{ lookup('ansible.builtin.env', 'ADMIN_PASSWORD', default='') }}"
|
||||||
grafana_admin_email: "{{ lookup('ansible.builtin.env', 'MAIL_ADMIN_USER', default=grafana_admin_user ~ '@gunihome.ru') }}"
|
grafana_admin_email: "{{ lookup('ansible.builtin.env', 'MAIL_ADMIN_USER', default=grafana_admin_user ~ '@gunihome.ru') }}"
|
||||||
|
|||||||
@@ -15491,7 +15491,7 @@
|
|||||||
"name": "job",
|
"name": "job",
|
||||||
"options": [],
|
"options": [],
|
||||||
"query": {
|
"query": {
|
||||||
"query": "label_values(node_uname_info, job)",
|
"query": "label_values(up, job)",
|
||||||
"refId": "Prometheus-job-Variable-Query"
|
"refId": "Prometheus-job-Variable-Query"
|
||||||
},
|
},
|
||||||
"refresh": 1,
|
"refresh": 1,
|
||||||
@@ -15505,13 +15505,13 @@
|
|||||||
"type": "prometheus",
|
"type": "prometheus",
|
||||||
"uid": "prometheus"
|
"uid": "prometheus"
|
||||||
},
|
},
|
||||||
"definition": "label_values(node_uname_info{job=\"$job\"}, host)",
|
"definition": "label_values(up{job=\"$job\"}, host)",
|
||||||
"includeAll": false,
|
"includeAll": false,
|
||||||
"label": "Host",
|
"label": "Host",
|
||||||
"name": "nodename",
|
"name": "nodename",
|
||||||
"options": [],
|
"options": [],
|
||||||
"query": {
|
"query": {
|
||||||
"query": "label_values(node_uname_info{job=\"$job\"}, host)",
|
"query": "label_values(up{job=\"$job\"}, host)",
|
||||||
"refId": "Prometheus-nodename-Variable-Query"
|
"refId": "Prometheus-nodename-Variable-Query"
|
||||||
},
|
},
|
||||||
"refresh": 1,
|
"refresh": 1,
|
||||||
@@ -15525,13 +15525,13 @@
|
|||||||
"type": "prometheus",
|
"type": "prometheus",
|
||||||
"uid": "prometheus"
|
"uid": "prometheus"
|
||||||
},
|
},
|
||||||
"definition": "label_values(node_uname_info{job=\"$job\", host=\"$nodename\"}, instance)",
|
"definition": "label_values(up{job=\"$job\", host=\"$nodename\"}, instance)",
|
||||||
"includeAll": false,
|
"includeAll": false,
|
||||||
"label": "Instance",
|
"label": "Instance",
|
||||||
"name": "node",
|
"name": "node",
|
||||||
"options": [],
|
"options": [],
|
||||||
"query": {
|
"query": {
|
||||||
"query": "label_values(node_uname_info{job=\"$job\", host=\"$nodename\"}, instance)",
|
"query": "label_values(up{job=\"$job\", host=\"$nodename\"}, instance)",
|
||||||
"refId": "Prometheus-node-Variable-Query"
|
"refId": "Prometheus-node-Variable-Query"
|
||||||
},
|
},
|
||||||
"refresh": 1,
|
"refresh": 1,
|
||||||
@@ -15549,7 +15549,7 @@
|
|||||||
"timezone": "browser",
|
"timezone": "browser",
|
||||||
"title": "Node Exporter Full",
|
"title": "Node Exporter Full",
|
||||||
"uid": "rYdddlPWk",
|
"uid": "rYdddlPWk",
|
||||||
"version": 105,
|
"version": 106,
|
||||||
"weekStart": "",
|
"weekStart": "",
|
||||||
"gnetId": 1860
|
"gnetId": 1860
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,26 @@
|
|||||||
|
---
|
||||||
|
- name: Reload Prometheus configuration
|
||||||
|
ansible.builtin.uri:
|
||||||
|
url: http://127.0.0.1:9090/-/reload
|
||||||
|
method: POST
|
||||||
|
status_code: 200
|
||||||
|
register: prometheus_reload
|
||||||
|
until: prometheus_reload.status == 200
|
||||||
|
retries: 10
|
||||||
|
delay: 3
|
||||||
|
when: prometheus_config_reload | default(true) | bool
|
||||||
|
ignore_errors: true
|
||||||
|
|
||||||
|
- name: Restart Prometheus after failed reload
|
||||||
|
ansible.builtin.command: docker compose -f {{ monitoring_dir }}/compose.yaml restart prometheus
|
||||||
|
args:
|
||||||
|
chdir: "{{ monitoring_dir }}"
|
||||||
|
when:
|
||||||
|
- prometheus_config_reload | default(true) | bool
|
||||||
|
- prometheus_reload is defined
|
||||||
|
- prometheus_reload is failed or (prometheus_reload.status | default(0) != 200)
|
||||||
|
|
||||||
|
- name: Restart Grafana
|
||||||
|
ansible.builtin.command: docker compose -f {{ monitoring_dir }}/compose.yaml restart grafana
|
||||||
|
args:
|
||||||
|
chdir: "{{ monitoring_dir }}"
|
||||||
@@ -20,6 +20,34 @@
|
|||||||
- { path: "{{ monitoring_dir }}/ntfy_data", owner: "root", group: "root", mode: "0777" }
|
- { path: "{{ monitoring_dir }}/ntfy_data", owner: "root", group: "root", mode: "0777" }
|
||||||
- { path: "{{ monitoring_dir }}/alertmanager-ntfy", owner: "root", group: "root", mode: "0755" }
|
- { path: "{{ monitoring_dir }}/alertmanager-ntfy", owner: "root", group: "root", mode: "0755" }
|
||||||
|
|
||||||
|
- name: Сбросить список целей scrape node-exporter
|
||||||
|
ansible.builtin.set_fact:
|
||||||
|
monitoring_scrape_targets: []
|
||||||
|
|
||||||
|
- name: Собрать цели scrape node-exporter из inventory
|
||||||
|
ansible.builtin.set_fact:
|
||||||
|
monitoring_scrape_targets: "{{ monitoring_scrape_targets + [target] }}"
|
||||||
|
vars:
|
||||||
|
target:
|
||||||
|
name: "{{ item }}"
|
||||||
|
address: "{{ hostvars[item].monitoring_scrape_address | default(hostvars[item].ansible_host) }}"
|
||||||
|
port: "{{ hostvars[item].monitoring_node_exporter_port | default(monitoring_node_exporter_port) }}"
|
||||||
|
env: "{{ hostvars[item].monitoring_env | default(monitoring_env_label) }}"
|
||||||
|
loop: "{{ groups['all'] | sort }}"
|
||||||
|
when:
|
||||||
|
- item not in monitoring_scrape_exclude_hosts
|
||||||
|
- hostvars[item].monitoring_scrape_enabled | default(true) | bool
|
||||||
|
|
||||||
|
- name: Проверить, что в Prometheus есть хотя бы одна цель node-exporter
|
||||||
|
ansible.builtin.assert:
|
||||||
|
that:
|
||||||
|
- monitoring_scrape_targets | length > 0
|
||||||
|
fail_msg: "Нет целей scrape: проверьте inventory и monitoring_scrape_exclude_hosts."
|
||||||
|
|
||||||
|
- name: Показать цели scrape node-exporter
|
||||||
|
ansible.builtin.debug:
|
||||||
|
msg: "{{ monitoring_scrape_targets | map(attribute='name') | list | sort }}"
|
||||||
|
|
||||||
- name: Установить конфигурацию Prometheus
|
- name: Установить конфигурацию Prometheus
|
||||||
ansible.builtin.template:
|
ansible.builtin.template:
|
||||||
src: prometheus.yml.j2
|
src: prometheus.yml.j2
|
||||||
@@ -27,6 +55,7 @@
|
|||||||
owner: root
|
owner: root
|
||||||
group: root
|
group: root
|
||||||
mode: "0644"
|
mode: "0644"
|
||||||
|
notify: Reload Prometheus configuration
|
||||||
|
|
||||||
- name: Установить правила алертов Prometheus
|
- name: Установить правила алертов Prometheus
|
||||||
ansible.builtin.copy:
|
ansible.builtin.copy:
|
||||||
@@ -35,6 +64,7 @@
|
|||||||
owner: root
|
owner: root
|
||||||
group: root
|
group: root
|
||||||
mode: "0644"
|
mode: "0644"
|
||||||
|
notify: Reload Prometheus configuration
|
||||||
|
|
||||||
- name: Установить конфигурацию Alertmanager
|
- name: Установить конфигурацию Alertmanager
|
||||||
ansible.builtin.template:
|
ansible.builtin.template:
|
||||||
@@ -75,6 +105,7 @@
|
|||||||
owner: root
|
owner: root
|
||||||
group: root
|
group: root
|
||||||
mode: "0644"
|
mode: "0644"
|
||||||
|
notify: Restart Grafana
|
||||||
|
|
||||||
- name: Установить compose.yaml для стека мониторинга
|
- name: Установить compose.yaml для стека мониторинга
|
||||||
ansible.builtin.template:
|
ansible.builtin.template:
|
||||||
|
|||||||
@@ -16,26 +16,12 @@ scrape_configs:
|
|||||||
static_configs:
|
static_configs:
|
||||||
- targets: ["localhost:9090"]
|
- targets: ["localhost:9090"]
|
||||||
|
|
||||||
- job_name: "node-exporter"
|
- job_name: "{{ monitoring_scrape_job_name }}"
|
||||||
static_configs:
|
static_configs:
|
||||||
|
{% for target in monitoring_scrape_targets %}
|
||||||
- targets:
|
- targets:
|
||||||
- "192.168.1.108:9100"
|
- "{{ target.address }}:{{ target.port }}"
|
||||||
labels:
|
labels:
|
||||||
host: "core-mon"
|
host: "{{ target.name }}"
|
||||||
env: "prod"
|
env: "{{ target.env }}"
|
||||||
- targets:
|
{% endfor %}
|
||||||
- "192.168.1.104:9100"
|
|
||||||
labels:
|
|
||||||
host: "core-net"
|
|
||||||
env: "prod"
|
|
||||||
- targets:
|
|
||||||
- "192.168.1.102:9100"
|
|
||||||
labels:
|
|
||||||
host: "core-secrets"
|
|
||||||
env: "prod"
|
|
||||||
- targets:
|
|
||||||
- "45.154.185.208:9100"
|
|
||||||
labels:
|
|
||||||
host: "rus-central"
|
|
||||||
env: "prod"
|
|
||||||
|
|
||||||
Reference in new issue
Block a user