feat(monitoring): закрыть пробелы в алертах хостов и сервисов
ansible-core-mon / deploy (push) Successful in 17s
ansible-core-mon / deploy (push) Successful in 17s
Пуш-хосты больше не пропадают молча, а диск, OOM, cAdvisor, Remnawave и xray-checker доходят до ntfy. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
a2b0c37522
commit
296c54f840
1 file changed
+188
-3
@@ -10,14 +10,101 @@ groups:
|
|||||||
summary: "Сервис или узел {{ $labels.instance }} недоступен"
|
summary: "Сервис или узел {{ $labels.instance }} недоступен"
|
||||||
description: "Экспортер {{ $labels.instance }} (job={{ $labels.job }}) не отвечает более 2 минут."
|
description: "Экспортер {{ $labels.instance }} (job={{ $labels.job }}) не отвечает более 2 минут."
|
||||||
|
|
||||||
|
# Pull-цель пишет up=0, push-хост (core-mon, core-net, ger-nod-02) просто перестаёт слать ряды.
|
||||||
|
# tlast ловит паузу до 30 минут, absent держит алерт, пока ряд не вернётся.
|
||||||
|
- alert: HostMetricsStale
|
||||||
|
expr: |
|
||||||
|
(
|
||||||
|
time() - tlast_over_time(up{job="integrations/node_exporter"}[30m]) > 180
|
||||||
|
)
|
||||||
|
or absent_over_time(up{job="integrations/node_exporter",instance="core-mon"}[30m])
|
||||||
|
or absent_over_time(up{job="integrations/node_exporter",instance="core-net"}[30m])
|
||||||
|
or absent_over_time(up{job="integrations/node_exporter",instance="core-secrets"}[30m])
|
||||||
|
or absent_over_time(up{job="integrations/node_exporter",instance="ger-nod-02"}[30m])
|
||||||
|
or absent_over_time(up{job="integrations/node_exporter",instance="rus-central"}[30m])
|
||||||
|
for: 1m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Нет свежих метрик узла {{ $labels.instance }}"
|
||||||
|
description: "node_exporter {{ $labels.instance }} не присылает up дольше 3 минут."
|
||||||
|
|
||||||
- alert: HostDiskSpaceLow
|
- alert: HostDiskSpaceLow
|
||||||
expr: (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100 < 15
|
expr: |
|
||||||
|
(
|
||||||
|
node_filesystem_avail_bytes{job="integrations/node_exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs|rootfs",mountpoint!~"/boot(/efi)?"}
|
||||||
|
/
|
||||||
|
node_filesystem_size_bytes{job="integrations/node_exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs|rootfs",mountpoint!~"/boot(/efi)?"}
|
||||||
|
) * 100 < 15
|
||||||
|
and
|
||||||
|
(
|
||||||
|
node_filesystem_avail_bytes{job="integrations/node_exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs|rootfs",mountpoint!~"/boot(/efi)?"}
|
||||||
|
/
|
||||||
|
node_filesystem_size_bytes{job="integrations/node_exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs|rootfs",mountpoint!~"/boot(/efi)?"}
|
||||||
|
) * 100 >= 5
|
||||||
for: 5m
|
for: 5m
|
||||||
labels:
|
labels:
|
||||||
severity: warning
|
severity: warning
|
||||||
annotations:
|
annotations:
|
||||||
summary: "Мало свободного места на {{ $labels.instance }}"
|
summary: "Мало свободного места на {{ $labels.instance }} {{ $labels.mountpoint }}"
|
||||||
description: "На корневом разделе осталось менее 15% свободного места."
|
description: "На {{ $labels.mountpoint }} ({{ $labels.instance }}) свободно меньше 15%."
|
||||||
|
|
||||||
|
- alert: HostDiskSpaceCritical
|
||||||
|
expr: |
|
||||||
|
(
|
||||||
|
node_filesystem_avail_bytes{job="integrations/node_exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs|rootfs",mountpoint!~"/boot(/efi)?"}
|
||||||
|
/
|
||||||
|
node_filesystem_size_bytes{job="integrations/node_exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs|rootfs",mountpoint!~"/boot(/efi)?"}
|
||||||
|
) * 100 < 5
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Заканчивается место на {{ $labels.instance }} {{ $labels.mountpoint }}"
|
||||||
|
description: "На {{ $labels.mountpoint }} ({{ $labels.instance }}) свободно меньше 5%."
|
||||||
|
|
||||||
|
- alert: HostInodesLow
|
||||||
|
expr: |
|
||||||
|
(
|
||||||
|
node_filesystem_files_free{job="integrations/node_exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs|rootfs",mountpoint!~"/boot(/efi)?"}
|
||||||
|
/
|
||||||
|
node_filesystem_files{job="integrations/node_exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs|rootfs",mountpoint!~"/boot(/efi)?"}
|
||||||
|
) * 100 < 10
|
||||||
|
for: 10m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Мало свободных inode на {{ $labels.instance }} {{ $labels.mountpoint }}"
|
||||||
|
description: "На {{ $labels.mountpoint }} ({{ $labels.instance }}) свободно меньше 10% inode."
|
||||||
|
|
||||||
|
- alert: HostFilesystemReadonly
|
||||||
|
expr: |
|
||||||
|
node_filesystem_readonly{job="integrations/node_exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs|rootfs",mountpoint!~"/boot(/efi)?"} == 1
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Файловая система {{ $labels.instance }} {{ $labels.mountpoint }} только для чтения"
|
||||||
|
description: "{{ $labels.mountpoint }} на {{ $labels.instance }} смонтирована readonly."
|
||||||
|
|
||||||
|
- alert: HostFilesystemDeviceError
|
||||||
|
expr: |
|
||||||
|
node_filesystem_device_error{job="integrations/node_exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs|rootfs",mountpoint!~"/boot(/efi)?"} == 1
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Ошибка устройства {{ $labels.instance }} {{ $labels.mountpoint }}"
|
||||||
|
description: "node_exporter видит ошибку устройства на {{ $labels.mountpoint }} ({{ $labels.instance }})."
|
||||||
|
|
||||||
|
- alert: HostOomKill
|
||||||
|
expr: increase(node_vmstat_oom_kill{job="integrations/node_exporter"}[5m]) > 0
|
||||||
|
for: 1m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "OOM killer на {{ $labels.instance }}"
|
||||||
|
description: "Ядро {{ $labels.instance }} завершило процесс из-за нехватки памяти."
|
||||||
|
|
||||||
- alert: HostHighCpuLoad
|
- alert: HostHighCpuLoad
|
||||||
expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90
|
expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90
|
||||||
@@ -36,3 +123,101 @@ groups:
|
|||||||
annotations:
|
annotations:
|
||||||
summary: "Мало оперативной памяти на {{ $labels.instance }}"
|
summary: "Мало оперативной памяти на {{ $labels.instance }}"
|
||||||
description: "Использование RAM превышает 90% более 10 минут."
|
description: "Использование RAM превышает 90% более 10 минут."
|
||||||
|
|
||||||
|
- alert: CadvisorDown
|
||||||
|
expr: up{job="integrations/cAdvisor"} == 0
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "cAdvisor {{ $labels.instance }} недоступен"
|
||||||
|
description: "Scrape cAdvisor на {{ $labels.instance }} возвращает up=0 более 2 минут."
|
||||||
|
|
||||||
|
- alert: CadvisorMetricsStale
|
||||||
|
expr: |
|
||||||
|
(
|
||||||
|
time() - tlast_over_time(up{job="integrations/cAdvisor"}[30m]) > 180
|
||||||
|
)
|
||||||
|
or absent_over_time(up{job="integrations/cAdvisor",instance="core-mon"}[30m])
|
||||||
|
or absent_over_time(up{job="integrations/cAdvisor",instance="core-net"}[30m])
|
||||||
|
or absent_over_time(up{job="integrations/cAdvisor",instance="core-secrets"}[30m])
|
||||||
|
or absent_over_time(up{job="integrations/cAdvisor",instance="ger-nod-02"}[30m])
|
||||||
|
or absent_over_time(up{job="integrations/cAdvisor",instance="rus-central"}[30m])
|
||||||
|
for: 1m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Нет свежих метрик cAdvisor {{ $labels.instance }}"
|
||||||
|
description: "cAdvisor {{ $labels.instance }} не присылает up дольше 3 минут."
|
||||||
|
|
||||||
|
- alert: ContainerRestarting
|
||||||
|
expr: |
|
||||||
|
changes(container_start_time_seconds{container_label_com_docker_compose_service!="",container!=""}[15m]) > 2
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Контейнер {{ $labels.container_label_com_docker_compose_service }} перезапускается на {{ $labels.instance }}"
|
||||||
|
description: "Сервис Compose {{ $labels.container_label_com_docker_compose_service }} на {{ $labels.instance }} менял время старта больше двух раз за 15 минут."
|
||||||
|
|
||||||
|
- alert: RemnawaveDown
|
||||||
|
expr: up{job="remnawave"} == 0
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Панель Remnawave недоступна"
|
||||||
|
description: "Scrape remnawave ({{ $labels.instance }}) возвращает up=0 более 2 минут."
|
||||||
|
|
||||||
|
- alert: RemnawaveMetricsStale
|
||||||
|
expr: |
|
||||||
|
(
|
||||||
|
time() - tlast_over_time(up{job="remnawave"}[30m]) > 180
|
||||||
|
)
|
||||||
|
or absent_over_time(up{job="remnawave",instance="core-net"}[30m])
|
||||||
|
for: 1m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Нет свежих метрик Remnawave"
|
||||||
|
description: "Панель Remnawave на {{ $labels.instance }} не присылает up дольше 3 минут."
|
||||||
|
|
||||||
|
- alert: RemnawaveNodeNotActive
|
||||||
|
expr: remnawave_node_status != 1
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Узел Remnawave не активен"
|
||||||
|
description: "remnawave_node_status узла {{ $labels.node_uuid }} на {{ $labels.instance }} не равен 1."
|
||||||
|
|
||||||
|
- alert: XrayCheckerDown
|
||||||
|
expr: up{job="xray_checker"} == 0
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "xray-checker недоступен"
|
||||||
|
description: "Scrape xray_checker ({{ $labels.instance }}) возвращает up=0 более 2 минут."
|
||||||
|
|
||||||
|
- alert: XrayCheckerMetricsStale
|
||||||
|
expr: |
|
||||||
|
(
|
||||||
|
time() - tlast_over_time(up{job="xray_checker"}[30m]) > 180
|
||||||
|
)
|
||||||
|
or absent_over_time(up{job="xray_checker",instance="core-mon"}[30m])
|
||||||
|
for: 1m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Нет свежих метрик xray-checker"
|
||||||
|
description: "xray-checker на {{ $labels.instance }} не присылает up дольше 3 минут."
|
||||||
|
|
||||||
|
- alert: XrayProxyDown
|
||||||
|
expr: xray_proxy_status != 1
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Прокси {{ $labels.name }} не проходит проверку"
|
||||||
|
description: "xray_proxy_status {{ $labels.name }} ({{ $labels.address }}) не равен 1."
|
||||||
Reference in new issue
Block a user