feat(monitoring): закрыть пробелы в алертах хостов и сервисов
ansible-core-mon / deploy (push) Successful in 17s
ansible-core-mon / deploy (push) Successful in 17s
Пуш-хосты больше не пропадают молча, а диск, OOM, cAdvisor, Remnawave и xray-checker доходят до ntfy. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
a2b0c37522
commit
296c54f840
1 file changed
+188
-3
@@ -10,14 +10,101 @@ groups:
|
||||
summary: "Сервис или узел {{ $labels.instance }} недоступен"
|
||||
description: "Экспортер {{ $labels.instance }} (job={{ $labels.job }}) не отвечает более 2 минут."
|
||||
|
||||
# Pull-цель пишет up=0, push-хост (core-mon, core-net, ger-nod-02) просто перестаёт слать ряды.
|
||||
# tlast ловит паузу до 30 минут, absent держит алерт, пока ряд не вернётся.
|
||||
- alert: HostMetricsStale
|
||||
expr: |
|
||||
(
|
||||
time() - tlast_over_time(up{job="integrations/node_exporter"}[30m]) > 180
|
||||
)
|
||||
or absent_over_time(up{job="integrations/node_exporter",instance="core-mon"}[30m])
|
||||
or absent_over_time(up{job="integrations/node_exporter",instance="core-net"}[30m])
|
||||
or absent_over_time(up{job="integrations/node_exporter",instance="core-secrets"}[30m])
|
||||
or absent_over_time(up{job="integrations/node_exporter",instance="ger-nod-02"}[30m])
|
||||
or absent_over_time(up{job="integrations/node_exporter",instance="rus-central"}[30m])
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Нет свежих метрик узла {{ $labels.instance }}"
|
||||
description: "node_exporter {{ $labels.instance }} не присылает up дольше 3 минут."
|
||||
|
||||
- alert: HostDiskSpaceLow
|
||||
expr: (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100 < 15
|
||||
expr: |
|
||||
(
|
||||
node_filesystem_avail_bytes{job="integrations/node_exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs|rootfs",mountpoint!~"/boot(/efi)?"}
|
||||
/
|
||||
node_filesystem_size_bytes{job="integrations/node_exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs|rootfs",mountpoint!~"/boot(/efi)?"}
|
||||
) * 100 < 15
|
||||
and
|
||||
(
|
||||
node_filesystem_avail_bytes{job="integrations/node_exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs|rootfs",mountpoint!~"/boot(/efi)?"}
|
||||
/
|
||||
node_filesystem_size_bytes{job="integrations/node_exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs|rootfs",mountpoint!~"/boot(/efi)?"}
|
||||
) * 100 >= 5
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Мало свободного места на {{ $labels.instance }}"
|
||||
description: "На корневом разделе осталось менее 15% свободного места."
|
||||
summary: "Мало свободного места на {{ $labels.instance }} {{ $labels.mountpoint }}"
|
||||
description: "На {{ $labels.mountpoint }} ({{ $labels.instance }}) свободно меньше 15%."
|
||||
|
||||
- alert: HostDiskSpaceCritical
|
||||
expr: |
|
||||
(
|
||||
node_filesystem_avail_bytes{job="integrations/node_exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs|rootfs",mountpoint!~"/boot(/efi)?"}
|
||||
/
|
||||
node_filesystem_size_bytes{job="integrations/node_exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs|rootfs",mountpoint!~"/boot(/efi)?"}
|
||||
) * 100 < 5
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Заканчивается место на {{ $labels.instance }} {{ $labels.mountpoint }}"
|
||||
description: "На {{ $labels.mountpoint }} ({{ $labels.instance }}) свободно меньше 5%."
|
||||
|
||||
- alert: HostInodesLow
|
||||
expr: |
|
||||
(
|
||||
node_filesystem_files_free{job="integrations/node_exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs|rootfs",mountpoint!~"/boot(/efi)?"}
|
||||
/
|
||||
node_filesystem_files{job="integrations/node_exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs|rootfs",mountpoint!~"/boot(/efi)?"}
|
||||
) * 100 < 10
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Мало свободных inode на {{ $labels.instance }} {{ $labels.mountpoint }}"
|
||||
description: "На {{ $labels.mountpoint }} ({{ $labels.instance }}) свободно меньше 10% inode."
|
||||
|
||||
- alert: HostFilesystemReadonly
|
||||
expr: |
|
||||
node_filesystem_readonly{job="integrations/node_exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs|rootfs",mountpoint!~"/boot(/efi)?"} == 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Файловая система {{ $labels.instance }} {{ $labels.mountpoint }} только для чтения"
|
||||
description: "{{ $labels.mountpoint }} на {{ $labels.instance }} смонтирована readonly."
|
||||
|
||||
- alert: HostFilesystemDeviceError
|
||||
expr: |
|
||||
node_filesystem_device_error{job="integrations/node_exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs|rootfs",mountpoint!~"/boot(/efi)?"} == 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Ошибка устройства {{ $labels.instance }} {{ $labels.mountpoint }}"
|
||||
description: "node_exporter видит ошибку устройства на {{ $labels.mountpoint }} ({{ $labels.instance }})."
|
||||
|
||||
- alert: HostOomKill
|
||||
expr: increase(node_vmstat_oom_kill{job="integrations/node_exporter"}[5m]) > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "OOM killer на {{ $labels.instance }}"
|
||||
description: "Ядро {{ $labels.instance }} завершило процесс из-за нехватки памяти."
|
||||
|
||||
- alert: HostHighCpuLoad
|
||||
expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90
|
||||
@@ -36,3 +123,101 @@ groups:
|
||||
annotations:
|
||||
summary: "Мало оперативной памяти на {{ $labels.instance }}"
|
||||
description: "Использование RAM превышает 90% более 10 минут."
|
||||
|
||||
- alert: CadvisorDown
|
||||
expr: up{job="integrations/cAdvisor"} == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "cAdvisor {{ $labels.instance }} недоступен"
|
||||
description: "Scrape cAdvisor на {{ $labels.instance }} возвращает up=0 более 2 минут."
|
||||
|
||||
- alert: CadvisorMetricsStale
|
||||
expr: |
|
||||
(
|
||||
time() - tlast_over_time(up{job="integrations/cAdvisor"}[30m]) > 180
|
||||
)
|
||||
or absent_over_time(up{job="integrations/cAdvisor",instance="core-mon"}[30m])
|
||||
or absent_over_time(up{job="integrations/cAdvisor",instance="core-net"}[30m])
|
||||
or absent_over_time(up{job="integrations/cAdvisor",instance="core-secrets"}[30m])
|
||||
or absent_over_time(up{job="integrations/cAdvisor",instance="ger-nod-02"}[30m])
|
||||
or absent_over_time(up{job="integrations/cAdvisor",instance="rus-central"}[30m])
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Нет свежих метрик cAdvisor {{ $labels.instance }}"
|
||||
description: "cAdvisor {{ $labels.instance }} не присылает up дольше 3 минут."
|
||||
|
||||
- alert: ContainerRestarting
|
||||
expr: |
|
||||
changes(container_start_time_seconds{container_label_com_docker_compose_service!="",container!=""}[15m]) > 2
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Контейнер {{ $labels.container_label_com_docker_compose_service }} перезапускается на {{ $labels.instance }}"
|
||||
description: "Сервис Compose {{ $labels.container_label_com_docker_compose_service }} на {{ $labels.instance }} менял время старта больше двух раз за 15 минут."
|
||||
|
||||
- alert: RemnawaveDown
|
||||
expr: up{job="remnawave"} == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Панель Remnawave недоступна"
|
||||
description: "Scrape remnawave ({{ $labels.instance }}) возвращает up=0 более 2 минут."
|
||||
|
||||
- alert: RemnawaveMetricsStale
|
||||
expr: |
|
||||
(
|
||||
time() - tlast_over_time(up{job="remnawave"}[30m]) > 180
|
||||
)
|
||||
or absent_over_time(up{job="remnawave",instance="core-net"}[30m])
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Нет свежих метрик Remnawave"
|
||||
description: "Панель Remnawave на {{ $labels.instance }} не присылает up дольше 3 минут."
|
||||
|
||||
- alert: RemnawaveNodeNotActive
|
||||
expr: remnawave_node_status != 1
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Узел Remnawave не активен"
|
||||
description: "remnawave_node_status узла {{ $labels.node_uuid }} на {{ $labels.instance }} не равен 1."
|
||||
|
||||
- alert: XrayCheckerDown
|
||||
expr: up{job="xray_checker"} == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "xray-checker недоступен"
|
||||
description: "Scrape xray_checker ({{ $labels.instance }}) возвращает up=0 более 2 минут."
|
||||
|
||||
- alert: XrayCheckerMetricsStale
|
||||
expr: |
|
||||
(
|
||||
time() - tlast_over_time(up{job="xray_checker"}[30m]) > 180
|
||||
)
|
||||
or absent_over_time(up{job="xray_checker",instance="core-mon"}[30m])
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Нет свежих метрик xray-checker"
|
||||
description: "xray-checker на {{ $labels.instance }} не присылает up дольше 3 минут."
|
||||
|
||||
- alert: XrayProxyDown
|
||||
expr: xray_proxy_status != 1
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Прокси {{ $labels.name }} не проходит проверку"
|
||||
description: "xray_proxy_status {{ $labels.name }} ({{ $labels.address }}) не равен 1."
|
||||
Reference in new issue
Block a user