diff --git a/ansible/roles/core-mon/monitoring/files/alerts.yml b/ansible/roles/core-mon/monitoring/files/alerts.yml index c0eae0d..591f69b 100644 --- a/ansible/roles/core-mon/monitoring/files/alerts.yml +++ b/ansible/roles/core-mon/monitoring/files/alerts.yml @@ -10,14 +10,101 @@ groups: summary: "Сервис или узел {{ $labels.instance }} недоступен" description: "Экспортер {{ $labels.instance }} (job={{ $labels.job }}) не отвечает более 2 минут." + # Pull-цель пишет up=0, push-хост (core-mon, core-net, ger-nod-02) просто перестаёт слать ряды. + # tlast ловит паузу до 30 минут, absent держит алерт, пока ряд не вернётся. + - alert: HostMetricsStale + expr: | + ( + time() - tlast_over_time(up{job="integrations/node_exporter"}[30m]) > 180 + ) + or absent_over_time(up{job="integrations/node_exporter",instance="core-mon"}[30m]) + or absent_over_time(up{job="integrations/node_exporter",instance="core-net"}[30m]) + or absent_over_time(up{job="integrations/node_exporter",instance="core-secrets"}[30m]) + or absent_over_time(up{job="integrations/node_exporter",instance="ger-nod-02"}[30m]) + or absent_over_time(up{job="integrations/node_exporter",instance="rus-central"}[30m]) + for: 1m + labels: + severity: critical + annotations: + summary: "Нет свежих метрик узла {{ $labels.instance }}" + description: "node_exporter {{ $labels.instance }} не присылает up дольше 3 минут." + - alert: HostDiskSpaceLow - expr: (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100 < 15 + expr: | + ( + node_filesystem_avail_bytes{job="integrations/node_exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs|rootfs",mountpoint!~"/boot(/efi)?"} + / + node_filesystem_size_bytes{job="integrations/node_exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs|rootfs",mountpoint!~"/boot(/efi)?"} + ) * 100 < 15 + and + ( + node_filesystem_avail_bytes{job="integrations/node_exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs|rootfs",mountpoint!~"/boot(/efi)?"} + / + node_filesystem_size_bytes{job="integrations/node_exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs|rootfs",mountpoint!~"/boot(/efi)?"} + ) * 100 >= 5 for: 5m labels: severity: warning annotations: - summary: "Мало свободного места на {{ $labels.instance }}" - description: "На корневом разделе осталось менее 15% свободного места." + summary: "Мало свободного места на {{ $labels.instance }} {{ $labels.mountpoint }}" + description: "На {{ $labels.mountpoint }} ({{ $labels.instance }}) свободно меньше 15%." + + - alert: HostDiskSpaceCritical + expr: | + ( + node_filesystem_avail_bytes{job="integrations/node_exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs|rootfs",mountpoint!~"/boot(/efi)?"} + / + node_filesystem_size_bytes{job="integrations/node_exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs|rootfs",mountpoint!~"/boot(/efi)?"} + ) * 100 < 5 + for: 5m + labels: + severity: critical + annotations: + summary: "Заканчивается место на {{ $labels.instance }} {{ $labels.mountpoint }}" + description: "На {{ $labels.mountpoint }} ({{ $labels.instance }}) свободно меньше 5%." + + - alert: HostInodesLow + expr: | + ( + node_filesystem_files_free{job="integrations/node_exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs|rootfs",mountpoint!~"/boot(/efi)?"} + / + node_filesystem_files{job="integrations/node_exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs|rootfs",mountpoint!~"/boot(/efi)?"} + ) * 100 < 10 + for: 10m + labels: + severity: warning + annotations: + summary: "Мало свободных inode на {{ $labels.instance }} {{ $labels.mountpoint }}" + description: "На {{ $labels.mountpoint }} ({{ $labels.instance }}) свободно меньше 10% inode." + + - alert: HostFilesystemReadonly + expr: | + node_filesystem_readonly{job="integrations/node_exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs|rootfs",mountpoint!~"/boot(/efi)?"} == 1 + for: 5m + labels: + severity: critical + annotations: + summary: "Файловая система {{ $labels.instance }} {{ $labels.mountpoint }} только для чтения" + description: "{{ $labels.mountpoint }} на {{ $labels.instance }} смонтирована readonly." + + - alert: HostFilesystemDeviceError + expr: | + node_filesystem_device_error{job="integrations/node_exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs|rootfs",mountpoint!~"/boot(/efi)?"} == 1 + for: 5m + labels: + severity: critical + annotations: + summary: "Ошибка устройства {{ $labels.instance }} {{ $labels.mountpoint }}" + description: "node_exporter видит ошибку устройства на {{ $labels.mountpoint }} ({{ $labels.instance }})." + + - alert: HostOomKill + expr: increase(node_vmstat_oom_kill{job="integrations/node_exporter"}[5m]) > 0 + for: 1m + labels: + severity: critical + annotations: + summary: "OOM killer на {{ $labels.instance }}" + description: "Ядро {{ $labels.instance }} завершило процесс из-за нехватки памяти." - alert: HostHighCpuLoad expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90 @@ -36,3 +123,101 @@ groups: annotations: summary: "Мало оперативной памяти на {{ $labels.instance }}" description: "Использование RAM превышает 90% более 10 минут." + + - alert: CadvisorDown + expr: up{job="integrations/cAdvisor"} == 0 + for: 2m + labels: + severity: critical + annotations: + summary: "cAdvisor {{ $labels.instance }} недоступен" + description: "Scrape cAdvisor на {{ $labels.instance }} возвращает up=0 более 2 минут." + + - alert: CadvisorMetricsStale + expr: | + ( + time() - tlast_over_time(up{job="integrations/cAdvisor"}[30m]) > 180 + ) + or absent_over_time(up{job="integrations/cAdvisor",instance="core-mon"}[30m]) + or absent_over_time(up{job="integrations/cAdvisor",instance="core-net"}[30m]) + or absent_over_time(up{job="integrations/cAdvisor",instance="core-secrets"}[30m]) + or absent_over_time(up{job="integrations/cAdvisor",instance="ger-nod-02"}[30m]) + or absent_over_time(up{job="integrations/cAdvisor",instance="rus-central"}[30m]) + for: 1m + labels: + severity: critical + annotations: + summary: "Нет свежих метрик cAdvisor {{ $labels.instance }}" + description: "cAdvisor {{ $labels.instance }} не присылает up дольше 3 минут." + + - alert: ContainerRestarting + expr: | + changes(container_start_time_seconds{container_label_com_docker_compose_service!="",container!=""}[15m]) > 2 + for: 2m + labels: + severity: warning + annotations: + summary: "Контейнер {{ $labels.container_label_com_docker_compose_service }} перезапускается на {{ $labels.instance }}" + description: "Сервис Compose {{ $labels.container_label_com_docker_compose_service }} на {{ $labels.instance }} менял время старта больше двух раз за 15 минут." + + - alert: RemnawaveDown + expr: up{job="remnawave"} == 0 + for: 2m + labels: + severity: critical + annotations: + summary: "Панель Remnawave недоступна" + description: "Scrape remnawave ({{ $labels.instance }}) возвращает up=0 более 2 минут." + + - alert: RemnawaveMetricsStale + expr: | + ( + time() - tlast_over_time(up{job="remnawave"}[30m]) > 180 + ) + or absent_over_time(up{job="remnawave",instance="core-net"}[30m]) + for: 1m + labels: + severity: critical + annotations: + summary: "Нет свежих метрик Remnawave" + description: "Панель Remnawave на {{ $labels.instance }} не присылает up дольше 3 минут." + + - alert: RemnawaveNodeNotActive + expr: remnawave_node_status != 1 + for: 2m + labels: + severity: critical + annotations: + summary: "Узел Remnawave не активен" + description: "remnawave_node_status узла {{ $labels.node_uuid }} на {{ $labels.instance }} не равен 1." + + - alert: XrayCheckerDown + expr: up{job="xray_checker"} == 0 + for: 2m + labels: + severity: critical + annotations: + summary: "xray-checker недоступен" + description: "Scrape xray_checker ({{ $labels.instance }}) возвращает up=0 более 2 минут." + + - alert: XrayCheckerMetricsStale + expr: | + ( + time() - tlast_over_time(up{job="xray_checker"}[30m]) > 180 + ) + or absent_over_time(up{job="xray_checker",instance="core-mon"}[30m]) + for: 1m + labels: + severity: critical + annotations: + summary: "Нет свежих метрик xray-checker" + description: "xray-checker на {{ $labels.instance }} не присылает up дольше 3 минут." + + - alert: XrayProxyDown + expr: xray_proxy_status != 1 + for: 2m + labels: + severity: critical + annotations: + summary: "Прокси {{ $labels.name }} не проходит проверку" + description: "xray_proxy_status {{ $labels.name }} ({{ $labels.address }}) не равен 1."