feat(remnawave,monitoring): full node/host cleanup on sync and dynamic alerts.yml.j2
ansible-deploy / plan (push) Successful in 2s
ansible-lint / lint (push) Successful in 3s
ansible-deploy / deploy (core-mon, ) (push) Successful in 17s
ansible-deploy / deploy (core-net, ) (push) Successful in 20s

This commit is contained in:
karpenok committed 2026-11-15 23:00:55 +07:00
1 parent b2524d1d1e
commit ae9beee5ca
4 files changed
+44 -30

No files matched your search

@@ -0,0 +1,220 @@
#jinja2: variable_start_string:'[[', variable_end_string:']]', block_start_string:'[%', block_end_string:'%]'
groups:
- name: homelab_alerts
rules:
- alert: HostDown
expr: up{job="integrations/node_exporter"} == 0
for: 2m
labels:
severity: critical
annotations:
summary: "Сервис или узел {{ $labels.instance }} недоступен"
description: "Экспортер {{ $labels.instance }} (job={{ $labels.job }}) не отвечает более 2 минут."
# Pull-цель пишет up=0, push-хост (core-mon, core-net, ger-nod-02) просто перестаёт слать ряды.
# tlast ловит паузу до 30 минут, absent держит алерт, пока ряд не вернётся.
- alert: HostMetricsStale
expr: |
(
time() - tlast_over_time(up{job="integrations/node_exporter"}[30m]) > 180
)
[% for h in (['core-mon', 'core-net', 'core-secrets', 'rus-central'] + (groups['exit_nodes'] | default([]))) | unique | sort %]
or absent_over_time(up{job="integrations/node_exporter",instance="[[ h ]]"}[30m])
[% endfor %]
for: 1m
labels:
severity: critical
annotations:
summary: "Нет свежих метрик узла {{ $labels.instance }}"
description: "node_exporter {{ $labels.instance }} не присылает up дольше 3 минут."
- alert: HostDiskSpaceLow
expr: |
(
node_filesystem_avail_bytes{job="integrations/node_exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs|rootfs",mountpoint!~"/boot(/efi)?"}
/
node_filesystem_size_bytes{job="integrations/node_exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs|rootfs",mountpoint!~"/boot(/efi)?"}
) * 100 < 15
and
(
node_filesystem_avail_bytes{job="integrations/node_exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs|rootfs",mountpoint!~"/boot(/efi)?"}
/
node_filesystem_size_bytes{job="integrations/node_exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs|rootfs",mountpoint!~"/boot(/efi)?"}
) * 100 >= 5
for: 5m
labels:
severity: warning
annotations:
summary: "Мало свободного места на {{ $labels.instance }} {{ $labels.mountpoint }}"
description: "На {{ $labels.mountpoint }} ({{ $labels.instance }}) свободно меньше 15%."
- alert: HostDiskSpaceCritical
expr: |
(
node_filesystem_avail_bytes{job="integrations/node_exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs|rootfs",mountpoint!~"/boot(/efi)?"}
/
node_filesystem_size_bytes{job="integrations/node_exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs|rootfs",mountpoint!~"/boot(/efi)?"}
) * 100 < 5
for: 5m
labels:
severity: critical
annotations:
summary: "Заканчивается место на {{ $labels.instance }} {{ $labels.mountpoint }}"
description: "На {{ $labels.mountpoint }} ({{ $labels.instance }}) свободно меньше 5%."
- alert: HostInodesLow
expr: |
(
node_filesystem_files_free{job="integrations/node_exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs|rootfs",mountpoint!~"/boot(/efi)?"}
/
node_filesystem_files{job="integrations/node_exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs|rootfs",mountpoint!~"/boot(/efi)?"}
) * 100 < 10
for: 10m
labels:
severity: warning
annotations:
summary: "Мало свободных inode на {{ $labels.instance }} {{ $labels.mountpoint }}"
description: "На {{ $labels.mountpoint }} ({{ $labels.instance }}) свободно меньше 10% inode."
- alert: HostFilesystemReadonly
expr: |
node_filesystem_readonly{job="integrations/node_exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs|rootfs",mountpoint!~"/boot(/efi)?"} == 1
for: 5m
labels:
severity: critical
annotations:
summary: "Файловая система {{ $labels.instance }} {{ $labels.mountpoint }} только для чтения"
description: "{{ $labels.mountpoint }} на {{ $labels.instance }} смонтирована readonly."
- alert: HostFilesystemDeviceError
expr: |
node_filesystem_device_error{job="integrations/node_exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs|rootfs",mountpoint!~"/boot(/efi)?"} == 1
for: 5m
labels:
severity: critical
annotations:
summary: "Ошибка устройства {{ $labels.instance }} {{ $labels.mountpoint }}"
description: "node_exporter видит ошибку устройства на {{ $labels.mountpoint }} ({{ $labels.instance }})."
- alert: HostOomKill
expr: increase(node_vmstat_oom_kill{job="integrations/node_exporter"}[5m]) > 0
for: 1m
labels:
severity: critical
annotations:
summary: "OOM killer на {{ $labels.instance }}"
description: "Ядро {{ $labels.instance }} завершило процесс из-за нехватки памяти."
- alert: HostHighCpuLoad
expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90
for: 10m
labels:
severity: warning
annotations:
summary: "Высокая нагрузка CPU на {{ $labels.instance }}"
description: "Утилизация CPU превышает 90% более 10 минут."
- alert: HostHighMemoryUsage
expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 90
for: 10m
labels:
severity: warning
annotations:
summary: "Мало оперативной памяти на {{ $labels.instance }}"
description: "Использование RAM превышает 90% более 10 минут."
- alert: CadvisorDown
expr: up{job="integrations/cAdvisor"} == 0
for: 2m
labels:
severity: critical
annotations:
summary: "cAdvisor {{ $labels.instance }} недоступен"
description: "Scrape cAdvisor на {{ $labels.instance }} возвращает up=0 более 2 минут."
- alert: CadvisorMetricsStale
expr: |
(
time() - tlast_over_time(up{job="integrations/cAdvisor"}[30m]) > 180
)
[% for h in (['core-mon', 'core-net', 'core-secrets', 'rus-central'] + (groups['exit_nodes'] | default([]))) | unique | sort %]
or absent_over_time(up{job="integrations/cAdvisor",instance="[[ h ]]"}[30m])
[% endfor %]
for: 1m
labels:
severity: critical
annotations:
summary: "Нет свежих метрик cAdvisor {{ $labels.instance }}"
description: "cAdvisor {{ $labels.instance }} не присылает up дольше 3 минут."
- alert: ContainerRestarting
expr: |
changes(container_start_time_seconds{container_label_com_docker_compose_service!=""}[15m]) > 2
for: 2m
labels:
severity: warning
annotations:
summary: "Контейнер {{ $labels.container_label_com_docker_compose_service }} перезапускается на {{ $labels.instance }}"
description: "Сервис Compose {{ $labels.container_label_com_docker_compose_service }} на {{ $labels.instance }} менял время старта больше двух раз за 15 минут."
- alert: RemnawaveDown
expr: up{job="remnawave"} == 0
for: 2m
labels:
severity: critical
annotations:
summary: "Панель Remnawave недоступна"
description: "Scrape remnawave ({{ $labels.instance }}) возвращает up=0 более 2 минут."
- alert: RemnawaveMetricsStale
expr: |
(
time() - tlast_over_time(up{job="remnawave"}[30m]) > 180
)
or absent_over_time(up{job="remnawave",instance="core-net"}[30m])
for: 1m
labels:
severity: critical
annotations:
summary: "Нет свежих метрик Remnawave"
description: "Панель Remnawave на {{ $labels.instance }} не присылает up дольше 3 минут."
- alert: RemnawaveNodeNotActive
expr: remnawave_node_status != 1
for: 2m
labels:
severity: critical
annotations:
summary: "Узел Remnawave не активен"
description: "remnawave_node_status узла {{ $labels.node_uuid }} на {{ $labels.instance }} не равен 1."
- alert: XrayCheckerDown
expr: up{job="xray_checker"} == 0
for: 2m
labels:
severity: critical
annotations:
summary: "xray-checker недоступен"
description: "Scrape xray_checker ({{ $labels.instance }}) возвращает up=0 более 2 минут."
- alert: XrayCheckerMetricsStale
expr: |
(
time() - tlast_over_time(up{job="xray_checker"}[30m]) > 180
)
or absent_over_time(up{job="xray_checker",instance="core-mon"}[30m])
for: 1m
labels:
severity: critical
annotations:
summary: "Нет свежих метрик xray-checker"
description: "xray-checker на {{ $labels.instance }} не присылает up дольше 3 минут."
- alert: XrayProxyDown
expr: xray_proxy_status != 1
for: 2m
labels:
severity: critical
annotations:
summary: "Прокси {{ $labels.name }} не проходит проверку"
description: "xray_proxy_status {{ $labels.name }} ({{ $labels.address }}) не равен 1."