fix(monitoring): убрать Node Exporter Full и собирать cAdvisor со всех Docker-хостов
Старый дашборд оставался в каталоге Grafana, потому что копирование его не удаляет. core-secrets и rus-central не попадали в Cadvisor exporter: экспортёра на них не было. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
57aa12b9c5
commit
93958f93e2
14 files changed
+98
-15558
No files matched your search
@@ -8,6 +8,7 @@ on:
|
|||||||
- 'ansible/ansible.cfg'
|
- 'ansible/ansible.cfg'
|
||||||
- 'ansible/ci/**'
|
- 'ansible/ci/**'
|
||||||
- 'ansible/roles/core-secrets/**'
|
- 'ansible/roles/core-secrets/**'
|
||||||
|
- 'ansible/roles/cadvisor/**'
|
||||||
- 'ansible/roles/host/**'
|
- 'ansible/roles/host/**'
|
||||||
- 'ansible/playbooks/core-secrets.yml'
|
- 'ansible/playbooks/core-secrets.yml'
|
||||||
- 'ansible/playbooks/host.yml'
|
- 'ansible/playbooks/host.yml'
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ on:
|
|||||||
- 'ansible/ansible.cfg'
|
- 'ansible/ansible.cfg'
|
||||||
- 'ansible/ci/**'
|
- 'ansible/ci/**'
|
||||||
- 'ansible/roles/rus-central/**'
|
- 'ansible/roles/rus-central/**'
|
||||||
|
- 'ansible/roles/cadvisor/**'
|
||||||
- 'ansible/roles/host/**'
|
- 'ansible/roles/host/**'
|
||||||
- 'ansible/playbooks/rus-central.yml'
|
- 'ansible/playbooks/rus-central.yml'
|
||||||
- 'ansible/playbooks/host.yml'
|
- 'ansible/playbooks/host.yml'
|
||||||
|
|||||||
+2
-2
@@ -7,9 +7,9 @@
|
|||||||
| Машина | Плейбук | Роли | Каталоги |
|
| Машина | Плейбук | Роли | Каталоги |
|
||||||
|---|---|---|---|
|
|---|---|---|---|
|
||||||
| `core-net` | `playbooks/core-net.yml` | `roles/core-net/traefik`; `roles/core-net/netbird`; `roles/core-net/remnawave`; `roles/core-net/vmagent`; `roles/core-net/authentik` *(опц., по умолчанию выкл.)* | `/opt/traefik`; клиент NetBird; `/opt/remnawave`; `/opt/vmagent`; `/opt/authentik` *(если включён)* |
|
| `core-net` | `playbooks/core-net.yml` | `roles/core-net/traefik`; `roles/core-net/netbird`; `roles/core-net/remnawave`; `roles/core-net/vmagent`; `roles/core-net/authentik` *(опц., по умолчанию выкл.)* | `/opt/traefik`; клиент NetBird; `/opt/remnawave`; `/opt/vmagent`; `/opt/authentik` *(если включён)* |
|
||||||
| `core-secrets` | `playbooks/core-secrets.yml` | `roles/core-secrets/infisical`, `roles/core-secrets/vaultwarden` | `/opt/infisical`, `/opt/vaultwarden` |
|
| `core-secrets` | `playbooks/core-secrets.yml` | `roles/core-secrets/infisical`, `roles/core-secrets/vaultwarden`, `roles/core-secrets/cadvisor` | `/opt/infisical`, `/opt/vaultwarden`, `/opt/cadvisor` |
|
||||||
| `core-mon` | `playbooks/core-mon.yml` | `roles/core-mon/monitoring` | `/opt/monitoring` (VictoriaMetrics, vmalert, Grafana) |
|
| `core-mon` | `playbooks/core-mon.yml` | `roles/core-mon/monitoring` | `/opt/monitoring` (VictoriaMetrics, vmalert, Grafana) |
|
||||||
| `rus-central` | `playbooks/rus-central.yml` | `roles/rus-central/netbird` | `/opt/netbird` |
|
| `rus-central` | `playbooks/rus-central.yml` | `roles/rus-central/netbird`, `roles/rus-central/cadvisor` | `/opt/netbird`, `/opt/cadvisor` |
|
||||||
| `ger-nod-02` | `playbooks/ger-nod-02.yml` | `roles/ger-nod-02/netbird`; `roles/ger-nod-02/remnanode`; `roles/ger-nod-02/vmagent` | клиент NetBird; `/opt/remnanode`; `/opt/vmagent` |
|
| `ger-nod-02` | `playbooks/ger-nod-02.yml` | `roles/ger-nod-02/netbird`; `roles/ger-nod-02/remnanode`; `roles/ger-nod-02/vmagent` | клиент NetBird; `/opt/remnanode`; `/opt/vmagent` |
|
||||||
| `opnsense` | `playbooks/opnsense.yml` | `roles/opnsense/netbird` | REST API (`192.168.1.1`) |
|
| `opnsense` | `playbooks/opnsense.yml` | `roles/opnsense/netbird` | REST API (`192.168.1.1`) |
|
||||||
|
|
||||||
|
|||||||
@@ -7,3 +7,5 @@
|
|||||||
tags: [infisical]
|
tags: [infisical]
|
||||||
- role: core-secrets/vaultwarden
|
- role: core-secrets/vaultwarden
|
||||||
tags: [vaultwarden]
|
tags: [vaultwarden]
|
||||||
|
- role: core-secrets/cadvisor
|
||||||
|
tags: [cadvisor]
|
||||||
@@ -5,3 +5,5 @@
|
|||||||
roles:
|
roles:
|
||||||
- role: rus-central/netbird
|
- role: rus-central/netbird
|
||||||
tags: [netbird]
|
tags: [netbird]
|
||||||
|
- role: rus-central/cadvisor
|
||||||
|
tags: [cadvisor]
|
||||||
@@ -0,0 +1,4 @@
|
|||||||
|
---
|
||||||
|
cadvisor_dir: /opt/cadvisor
|
||||||
|
cadvisor_image: gcr.io/cadvisor/cadvisor:v0.49.2
|
||||||
|
cadvisor_port: 9101
|
||||||
@@ -0,0 +1,40 @@
|
|||||||
|
---
|
||||||
|
- name: Каталог cAdvisor
|
||||||
|
ansible.builtin.file:
|
||||||
|
path: "{{ cadvisor_dir }}"
|
||||||
|
state: directory
|
||||||
|
owner: root
|
||||||
|
group: root
|
||||||
|
mode: "0755"
|
||||||
|
|
||||||
|
- name: Compose cAdvisor
|
||||||
|
ansible.builtin.template:
|
||||||
|
src: compose.yaml.j2
|
||||||
|
dest: "{{ cadvisor_dir }}/compose.yaml"
|
||||||
|
owner: root
|
||||||
|
group: root
|
||||||
|
mode: "0644"
|
||||||
|
|
||||||
|
- name: Разрешить порт cAdvisor в firewall
|
||||||
|
ansible.builtin.command: ufw allow {{ cadvisor_port }}/tcp
|
||||||
|
register: ufw_cadvisor
|
||||||
|
changed_when: "'added' in (ufw_cadvisor.stdout | lower)"
|
||||||
|
when: ansible_virtualization_type | default('') != 'lxc'
|
||||||
|
|
||||||
|
- name: Запустить cAdvisor
|
||||||
|
ansible.builtin.command: docker compose -f {{ cadvisor_dir }}/compose.yaml up -d --remove-orphans
|
||||||
|
register: cadvisor_up
|
||||||
|
changed_when: >-
|
||||||
|
'Started' in cadvisor_up.stderr or
|
||||||
|
'Created' in cadvisor_up.stderr or
|
||||||
|
'Recreated' in cadvisor_up.stderr or
|
||||||
|
'Pulled' in cadvisor_up.stderr
|
||||||
|
|
||||||
|
- name: Дождаться cAdvisor
|
||||||
|
ansible.builtin.uri:
|
||||||
|
url: "http://127.0.0.1:{{ cadvisor_port }}/healthz"
|
||||||
|
status_code: 200
|
||||||
|
register: cadvisor_health
|
||||||
|
retries: 15
|
||||||
|
delay: 2
|
||||||
|
until: cadvisor_health.status == 200
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
name: cadvisor
|
||||||
|
|
||||||
|
services:
|
||||||
|
cadvisor:
|
||||||
|
image: {{ cadvisor_image }}
|
||||||
|
container_name: cadvisor
|
||||||
|
restart: unless-stopped
|
||||||
|
privileged: true
|
||||||
|
command:
|
||||||
|
- --listen_ip=0.0.0.0
|
||||||
|
- --port={{ cadvisor_port }}
|
||||||
|
- --docker_only=true
|
||||||
|
volumes:
|
||||||
|
- /:/rootfs:ro
|
||||||
|
- /var/run:/var/run:ro
|
||||||
|
- /sys:/sys:ro
|
||||||
|
- /var/lib/docker:/var/lib/docker:ro
|
||||||
|
- /dev/disk:/dev/disk:ro
|
||||||
|
ports:
|
||||||
|
- "{{ cadvisor_port }}:{{ cadvisor_port }}"
|
||||||
@@ -20,6 +20,11 @@ cadvisor_image: gcr.io/cadvisor/cadvisor:v0.49.2
|
|||||||
xray_checker_image: kutovoys/xray-checker:v1.3.1
|
xray_checker_image: kutovoys/xray-checker:v1.3.1
|
||||||
xray_checker_subscription_url: "{{ lookup('env', 'XRAY_CHECKER_SUBSCRIPTION_URL', default='') }}"
|
xray_checker_subscription_url: "{{ lookup('env', 'XRAY_CHECKER_SUBSCRIPTION_URL', default='') }}"
|
||||||
vmagent_cluster: homelab
|
vmagent_cluster: homelab
|
||||||
|
# cAdvisor этих хостов слушает :9101. core-mon, core-net и ger-nod-02 пишут его своим агентом.
|
||||||
|
monitoring_cadvisor_port: 9101
|
||||||
|
monitoring_cadvisor_scrape_hosts:
|
||||||
|
- core-secrets
|
||||||
|
- rus-central
|
||||||
|
|
||||||
grafana_admin_user: "{{ lookup('ansible.builtin.env', 'ADMIN_USER', default='karpenok') }}"
|
grafana_admin_user: "{{ lookup('ansible.builtin.env', 'ADMIN_USER', default='karpenok') }}"
|
||||||
grafana_admin_password: "{{ lookup('ansible.builtin.env', 'ADMIN_PASSWORD', default='') }}"
|
grafana_admin_password: "{{ lookup('ansible.builtin.env', 'ADMIN_PASSWORD', default='') }}"
|
||||||
|
|||||||
File diff suppressed because it is too large.
Load diff
@@ -100,6 +100,12 @@
|
|||||||
mode: "0644"
|
mode: "0644"
|
||||||
register: grafana_dashboards
|
register: grafana_dashboards
|
||||||
|
|
||||||
|
- name: Удалить старый дашборд Node Exporter Full
|
||||||
|
ansible.builtin.file:
|
||||||
|
path: "{{ monitoring_dir }}/grafana/provisioning/dashboards/node-exporter-full.json"
|
||||||
|
state: absent
|
||||||
|
register: grafana_old_dashboard
|
||||||
|
|
||||||
- name: Конфиг vmagent на core-mon
|
- name: Конфиг vmagent на core-mon
|
||||||
ansible.builtin.template:
|
ansible.builtin.template:
|
||||||
src: "{{ item.src }}"
|
src: "{{ item.src }}"
|
||||||
@@ -211,7 +217,7 @@
|
|||||||
|
|
||||||
- name: Перезапустить Grafana, чтобы подхватить datasource
|
- name: Перезапустить Grafana, чтобы подхватить datasource
|
||||||
ansible.builtin.command: docker compose -f {{ monitoring_dir }}/compose.yaml restart grafana
|
ansible.builtin.command: docker compose -f {{ monitoring_dir }}/compose.yaml restart grafana
|
||||||
when: grafana_datasources is changed or grafana_dashboards is changed or grafana_dashboard_provider is changed
|
when: grafana_datasources is changed or grafana_dashboards is changed or grafana_dashboard_provider is changed or grafana_old_dashboard is changed
|
||||||
changed_when: true
|
changed_when: true
|
||||||
|
|
||||||
- name: Дождаться готовности Grafana
|
- name: Дождаться готовности Grafana
|
||||||
|
|||||||
@@ -5,3 +5,9 @@
|
|||||||
labels:
|
labels:
|
||||||
instance: "{{ inventory_hostname }}"
|
instance: "{{ inventory_hostname }}"
|
||||||
cluster: "{{ vmagent_cluster }}"
|
cluster: "{{ vmagent_cluster }}"
|
||||||
|
{% for host in monitoring_cadvisor_scrape_hosts %}
|
||||||
|
- targets: ["{{ hostvars[host].ansible_host }}:{{ monitoring_cadvisor_port }}"]
|
||||||
|
labels:
|
||||||
|
instance: "{{ host }}"
|
||||||
|
cluster: "{{ vmagent_cluster }}"
|
||||||
|
{% endfor %}
|
||||||
@@ -0,0 +1,4 @@
|
|||||||
|
---
|
||||||
|
- name: cAdvisor секретов
|
||||||
|
ansible.builtin.include_role:
|
||||||
|
name: cadvisor
|
||||||
@@ -0,0 +1,4 @@
|
|||||||
|
---
|
||||||
|
- name: cAdvisor шлюза
|
||||||
|
ansible.builtin.include_role:
|
||||||
|
name: cadvisor
|
||||||
Reference in new issue
Block a user