feat(monitoring): add core-mon ansible playbook, roles, workflows, and traefik routes
This commit is contained in:
1 parent
9e9b8e9caf
commit
f26e0bfe57
16 files changed
+352
-3
No files matched your search
@@ -0,0 +1,15 @@
|
||||
global:
|
||||
resolve_timeout: 5m
|
||||
|
||||
route:
|
||||
group_by: ['alertname', 'instance', 'severity']
|
||||
group_wait: 30s
|
||||
group_interval: 5m
|
||||
repeat_interval: 4h
|
||||
receiver: 'ntfy'
|
||||
|
||||
receivers:
|
||||
- name: 'ntfy'
|
||||
webhook_configs:
|
||||
- url: "http://ntfy:80/{{ ntfy_default_topic }}"
|
||||
send_resolved: true
|
||||
@@ -0,0 +1,38 @@
|
||||
groups:
|
||||
- name: homelab_alerts
|
||||
rules:
|
||||
- alert: HostDown
|
||||
expr: up == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Сервис или узел {{ $labels.instance }} недоступен"
|
||||
description: "Экспортер {{ $labels.instance }} (job={{ $labels.job }}) не отвечает более 2 минут."
|
||||
|
||||
- alert: HostDiskSpaceLow
|
||||
expr: (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100 < 15
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Мало свободного места на {{ $labels.instance }}"
|
||||
description: "На корневом разделе осталось менее 15% свободного места."
|
||||
|
||||
- alert: HostHighCpuLoad
|
||||
expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Высокая нагрузка CPU на {{ $labels.instance }}"
|
||||
description: "Утилизация CPU превышает 90% более 10 минут."
|
||||
|
||||
- alert: HostHighMemoryUsage
|
||||
expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 90
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Мало оперативной памяти на {{ $labels.instance }}"
|
||||
description: "Использование RAM превышает 90% более 10 минут."
|
||||
@@ -0,0 +1,82 @@
|
||||
name: monitoring
|
||||
|
||||
services:
|
||||
prometheus:
|
||||
image: prom/prometheus:v3.2.1
|
||||
container_name: monitoring-prometheus
|
||||
restart: unless-stopped
|
||||
command:
|
||||
- "--config.file=/etc/prometheus/prometheus.yml"
|
||||
- "--storage.tsdb.path=/prometheus"
|
||||
- "--storage.tsdb.retention.time={{ prometheus_retention }}"
|
||||
- "--web.enable-lifecycle"
|
||||
volumes:
|
||||
- ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
||||
- ./prometheus/alerts.yml:/etc/prometheus/alerts.yml:ro
|
||||
- ./prometheus_data:/prometheus
|
||||
ports:
|
||||
- "9090:9090"
|
||||
depends_on:
|
||||
- alertmanager
|
||||
|
||||
alertmanager:
|
||||
image: prom/alertmanager:v0.28.1
|
||||
container_name: monitoring-alertmanager
|
||||
restart: unless-stopped
|
||||
command:
|
||||
- "--config.file=/etc/alertmanager/alertmanager.yml"
|
||||
- "--storage.path=/alertmanager"
|
||||
volumes:
|
||||
- ./alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
|
||||
- ./alertmanager_data:/alertmanager
|
||||
ports:
|
||||
- "9093:9093"
|
||||
|
||||
ntfy:
|
||||
image: binwiederhier/ntfy:v2.11.0
|
||||
container_name: monitoring-ntfy
|
||||
restart: unless-stopped
|
||||
command:
|
||||
- "serve"
|
||||
environment:
|
||||
- NTFY_BASE_URL={{ ntfy_base_url }}
|
||||
- NTFY_BEHIND_PROXY=true
|
||||
- NTFY_CACHE_FILE=/var/cache/ntfy/cache.db
|
||||
- NTFY_ATTACHMENT_CACHE_DIR=/var/cache/ntfy/attachments
|
||||
volumes:
|
||||
- ./ntfy_data:/var/cache/ntfy
|
||||
ports:
|
||||
- "80:80"
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana:11.5.2
|
||||
container_name: monitoring-grafana
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
- GF_SECURITY_ADMIN_USER={{ grafana_admin_user }}
|
||||
- GF_SECURITY_ADMIN_PASSWORD={{ grafana_admin_password }}
|
||||
- GF_USERS_ALLOW_SIGN_UP=false
|
||||
- GF_SERVER_ROOT_URL={{ grafana_root_url }}
|
||||
volumes:
|
||||
- ./grafana_data:/var/lib/grafana
|
||||
- ./grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
ports:
|
||||
- "3000:3000"
|
||||
depends_on:
|
||||
- prometheus
|
||||
|
||||
node-exporter:
|
||||
image: prom/node-exporter:v1.9.0
|
||||
container_name: monitoring-node-exporter
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
- /proc:/host/proc:ro
|
||||
- /sys:/host/sys:ro
|
||||
- /:/rootfs:ro
|
||||
command:
|
||||
- '--path.procfs=/host/proc'
|
||||
- '--path.rootfs=/rootfs'
|
||||
- '--path.sysfs=/host/sys'
|
||||
- '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)'
|
||||
ports:
|
||||
- "9100:9100"
|
||||
@@ -0,0 +1,10 @@
|
||||
apiVersion: 1
|
||||
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
jsonData:
|
||||
timeInterval: 15s
|
||||
@@ -0,0 +1,40 @@
|
||||
global:
|
||||
scrape_interval: 15s
|
||||
evaluation_interval: 15s
|
||||
|
||||
rule_files:
|
||||
- "/etc/prometheus/alerts.yml"
|
||||
|
||||
alerting:
|
||||
alertmanagers:
|
||||
- static_configs:
|
||||
- targets:
|
||||
- "alertmanager:9093"
|
||||
|
||||
scrape_configs:
|
||||
- job_name: "prometheus"
|
||||
static_configs:
|
||||
- targets: ["localhost:9090"]
|
||||
|
||||
- job_name: "node-exporter"
|
||||
static_configs:
|
||||
- targets:
|
||||
- "192.168.1.108:9100"
|
||||
labels:
|
||||
host: "core-mon"
|
||||
env: "prod"
|
||||
- targets:
|
||||
- "192.168.1.104:9100"
|
||||
labels:
|
||||
host: "core-net"
|
||||
env: "prod"
|
||||
- targets:
|
||||
- "192.168.1.102:9100"
|
||||
labels:
|
||||
host: "core-secrets"
|
||||
env: "prod"
|
||||
- targets:
|
||||
- "45.154.185.208:9100"
|
||||
labels:
|
||||
host: "rus-central"
|
||||
env: "prod"
|
||||
Reference in new issue
Block a user