feat(monitoring): add core-mon ansible playbook, roles, workflows, and traefik routes
This commit is contained in:
1 parent
9e9b8e9caf
commit
f26e0bfe57
16 files changed
+352
-3
No files matched your search
@@ -13,7 +13,7 @@ before=${3:-}
|
||||
after=${4:-}
|
||||
|
||||
case "$machine" in
|
||||
core-net | core-secrets | rus-central | opnsense) ;;
|
||||
core-net | core-secrets | core-mon | rus-central | opnsense) ;;
|
||||
*)
|
||||
echo "Неизвестная машина: $machine" >&2
|
||||
exit 2
|
||||
|
||||
@@ -9,7 +9,7 @@ before=${3:-}
|
||||
after=${4:-}
|
||||
|
||||
case "$machine" in
|
||||
core-net | core-secrets | rus-central | opnsense) ;;
|
||||
core-net | core-secrets | core-mon | rus-central | opnsense) ;;
|
||||
*)
|
||||
echo "Неизвестная машина: $machine" >&2
|
||||
exit 2
|
||||
@@ -68,6 +68,20 @@ case "$machine" in
|
||||
export INFISICAL_ENCRYPTION_KEY="${ENCRYPTION_KEY}"
|
||||
export INFISICAL_AUTH_SECRET="${AUTH_SECRET}"
|
||||
;;
|
||||
core-mon)
|
||||
if ! infisical export --domain="${INFISICAL_DOMAIN}" --token="${TOKEN}" --env=prod --projectId="${INFISICAL_PROJECT_ID}" --path="/services/monitoring" --format=dotenv-export --silent 2>/dev/null | grep -q 'GRAFANA_ADMIN_PASSWORD'; then
|
||||
infisical secrets-manager folders create monitoring --path /services --env prod --projectId "${INFISICAL_PROJECT_ID}" --domain "${INFISICAL_DOMAIN}" --token "${TOKEN}" --silent 2>/dev/null || true
|
||||
GEN_GRAFANA_PASS=$(openssl rand -base64 32 | tr -dc 'a-zA-Z0-9' | head -c 24)
|
||||
infisical secrets set --domain="${INFISICAL_DOMAIN}" --token="${TOKEN}" --env=prod --projectId="${INFISICAL_PROJECT_ID}" --path="/services/monitoring" \
|
||||
"GRAFANA_ADMIN_USER=admin" \
|
||||
"GRAFANA_ADMIN_PASSWORD=${GEN_GRAFANA_PASS}" \
|
||||
--silent
|
||||
unset GEN_GRAFANA_PASS
|
||||
fi
|
||||
eval "$(infisical export --domain="${INFISICAL_DOMAIN}" --token="${TOKEN}" --env=prod --projectId="${INFISICAL_PROJECT_ID}" --path="/services/monitoring" --format=dotenv-export --silent)"
|
||||
export GRAFANA_ADMIN_USER="${GRAFANA_ADMIN_USER:-admin}"
|
||||
export GRAFANA_ADMIN_PASSWORD="${GRAFANA_ADMIN_PASSWORD:-}"
|
||||
;;
|
||||
rus-central)
|
||||
input_email="${INPUT_LETSENCRYPT_EMAIL:-}"
|
||||
eval "$(infisical export --domain="${INFISICAL_DOMAIN}" --token="${TOKEN}" --env=prod --projectId="${INFISICAL_PROJECT_ID}" --path="/services/netbird" --format=dotenv-export --silent)"
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
---
|
||||
monitoring_dir: /opt/monitoring
|
||||
grafana_domain: "graf.gunihome.ru"
|
||||
ntfy_domain: "ntfy.gunihome.ru"
|
||||
@@ -7,6 +7,9 @@ all:
|
||||
core-secrets:
|
||||
ansible_host: 192.168.1.102
|
||||
ansible_user: root
|
||||
core-mon:
|
||||
ansible_host: 192.168.1.108
|
||||
ansible_user: root
|
||||
rus-central:
|
||||
ansible_host: 45.154.185.208
|
||||
ansible_user: root
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
---
|
||||
- name: Подготовка машины core-mon
|
||||
hosts: core-mon
|
||||
gather_facts: false
|
||||
roles:
|
||||
- role: host
|
||||
tags: [host]
|
||||
|
||||
- name: Развертывание стека мониторинга на core-mon
|
||||
hosts: core-mon
|
||||
roles:
|
||||
- role: core-mon/monitoring
|
||||
tags: [monitoring]
|
||||
@@ -2,5 +2,6 @@
|
||||
- ansible.builtin.import_playbook: host.yml
|
||||
- ansible.builtin.import_playbook: core-net.yml
|
||||
- ansible.builtin.import_playbook: core-secrets.yml
|
||||
- ansible.builtin.import_playbook: core-mon.yml
|
||||
- ansible.builtin.import_playbook: rus-central.yml
|
||||
- ansible.builtin.import_playbook: opnsense.yml
|
||||
@@ -0,0 +1,10 @@
|
||||
---
|
||||
monitoring_dir: /opt/monitoring
|
||||
prometheus_retention: "30d"
|
||||
|
||||
grafana_admin_user: "{{ lookup('ansible.builtin.env', 'GRAFANA_ADMIN_USER', default='admin') }}"
|
||||
grafana_admin_password: "{{ lookup('ansible.builtin.env', 'GRAFANA_ADMIN_PASSWORD', default='') }}"
|
||||
grafana_root_url: "https://graf.gunihome.ru"
|
||||
|
||||
ntfy_base_url: "https://ntfy.gunihome.ru"
|
||||
ntfy_default_topic: "homelab-alerts"
|
||||
@@ -0,0 +1,64 @@
|
||||
---
|
||||
- name: Создать базовые каталоги мониторинга
|
||||
ansible.builtin.file:
|
||||
path: "{{ item.path }}"
|
||||
state: directory
|
||||
owner: "{{ item.owner }}"
|
||||
group: "{{ item.group }}"
|
||||
mode: "{{ item.mode }}"
|
||||
loop:
|
||||
- { path: "{{ monitoring_dir }}", owner: "root", group: "root", mode: "0755" }
|
||||
- { path: "{{ monitoring_dir }}/prometheus", owner: "root", group: "root", mode: "0755" }
|
||||
- { path: "{{ monitoring_dir }}/prometheus_data", owner: "65534", group: "65534", mode: "0777" }
|
||||
- { path: "{{ monitoring_dir }}/alertmanager", owner: "root", group: "root", mode: "0755" }
|
||||
- { path: "{{ monitoring_dir }}/alertmanager_data", owner: "65534", group: "65534", mode: "0777" }
|
||||
- { path: "{{ monitoring_dir }}/grafana", owner: "root", group: "root", mode: "0755" }
|
||||
- { path: "{{ monitoring_dir }}/grafana_data", owner: "472", group: "472", mode: "0755" }
|
||||
- { path: "{{ monitoring_dir }}/grafana/provisioning", owner: "root", group: "root", mode: "0755" }
|
||||
- { path: "{{ monitoring_dir }}/grafana/provisioning/datasources", owner: "root", group: "root", mode: "0755" }
|
||||
- { path: "{{ monitoring_dir }}/ntfy_data", owner: "root", group: "root", mode: "0777" }
|
||||
|
||||
- name: Установить конфигурацию Prometheus
|
||||
ansible.builtin.template:
|
||||
src: prometheus.yml.j2
|
||||
dest: "{{ monitoring_dir }}/prometheus/prometheus.yml"
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0644"
|
||||
|
||||
- name: Установить правила алертов Prometheus
|
||||
ansible.builtin.template:
|
||||
src: alerts.yml.j2
|
||||
dest: "{{ monitoring_dir }}/prometheus/alerts.yml"
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0644"
|
||||
|
||||
- name: Установить конфигурацию Alertmanager
|
||||
ansible.builtin.template:
|
||||
src: alertmanager.yml.j2
|
||||
dest: "{{ monitoring_dir }}/alertmanager/alertmanager.yml"
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0644"
|
||||
|
||||
- name: Установить источник данных Prometheus в Grafana
|
||||
ansible.builtin.template:
|
||||
src: grafana-datasources.yml.j2
|
||||
dest: "{{ monitoring_dir }}/grafana/provisioning/datasources/prometheus.yml"
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0644"
|
||||
|
||||
- name: Установить compose.yaml для стека мониторинга
|
||||
ansible.builtin.template:
|
||||
src: compose.yaml.j2
|
||||
dest: "{{ monitoring_dir }}/compose.yaml"
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0644"
|
||||
|
||||
- name: Запустить стек мониторинга через Docker Compose
|
||||
ansible.builtin.command: docker compose -f {{ monitoring_dir }}/compose.yaml up -d --remove-orphans
|
||||
register: compose_up
|
||||
changed_when: "'Started' in compose_up.stderr or 'Created' in compose_up.stderr or 'Recreated' in compose_up.stderr or 'Pulled' in compose_up.stderr"
|
||||
@@ -0,0 +1,15 @@
|
||||
global:
|
||||
resolve_timeout: 5m
|
||||
|
||||
route:
|
||||
group_by: ['alertname', 'instance', 'severity']
|
||||
group_wait: 30s
|
||||
group_interval: 5m
|
||||
repeat_interval: 4h
|
||||
receiver: 'ntfy'
|
||||
|
||||
receivers:
|
||||
- name: 'ntfy'
|
||||
webhook_configs:
|
||||
- url: "http://ntfy:80/{{ ntfy_default_topic }}"
|
||||
send_resolved: true
|
||||
@@ -0,0 +1,38 @@
|
||||
groups:
|
||||
- name: homelab_alerts
|
||||
rules:
|
||||
- alert: HostDown
|
||||
expr: up == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Сервис или узел {{ $labels.instance }} недоступен"
|
||||
description: "Экспортер {{ $labels.instance }} (job={{ $labels.job }}) не отвечает более 2 минут."
|
||||
|
||||
- alert: HostDiskSpaceLow
|
||||
expr: (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100 < 15
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Мало свободного места на {{ $labels.instance }}"
|
||||
description: "На корневом разделе осталось менее 15% свободного места."
|
||||
|
||||
- alert: HostHighCpuLoad
|
||||
expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Высокая нагрузка CPU на {{ $labels.instance }}"
|
||||
description: "Утилизация CPU превышает 90% более 10 минут."
|
||||
|
||||
- alert: HostHighMemoryUsage
|
||||
expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 90
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Мало оперативной памяти на {{ $labels.instance }}"
|
||||
description: "Использование RAM превышает 90% более 10 минут."
|
||||
@@ -0,0 +1,82 @@
|
||||
name: monitoring
|
||||
|
||||
services:
|
||||
prometheus:
|
||||
image: prom/prometheus:v3.2.1
|
||||
container_name: monitoring-prometheus
|
||||
restart: unless-stopped
|
||||
command:
|
||||
- "--config.file=/etc/prometheus/prometheus.yml"
|
||||
- "--storage.tsdb.path=/prometheus"
|
||||
- "--storage.tsdb.retention.time={{ prometheus_retention }}"
|
||||
- "--web.enable-lifecycle"
|
||||
volumes:
|
||||
- ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
||||
- ./prometheus/alerts.yml:/etc/prometheus/alerts.yml:ro
|
||||
- ./prometheus_data:/prometheus
|
||||
ports:
|
||||
- "9090:9090"
|
||||
depends_on:
|
||||
- alertmanager
|
||||
|
||||
alertmanager:
|
||||
image: prom/alertmanager:v0.28.1
|
||||
container_name: monitoring-alertmanager
|
||||
restart: unless-stopped
|
||||
command:
|
||||
- "--config.file=/etc/alertmanager/alertmanager.yml"
|
||||
- "--storage.path=/alertmanager"
|
||||
volumes:
|
||||
- ./alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
|
||||
- ./alertmanager_data:/alertmanager
|
||||
ports:
|
||||
- "9093:9093"
|
||||
|
||||
ntfy:
|
||||
image: binwiederhier/ntfy:v2.11.0
|
||||
container_name: monitoring-ntfy
|
||||
restart: unless-stopped
|
||||
command:
|
||||
- "serve"
|
||||
environment:
|
||||
- NTFY_BASE_URL={{ ntfy_base_url }}
|
||||
- NTFY_BEHIND_PROXY=true
|
||||
- NTFY_CACHE_FILE=/var/cache/ntfy/cache.db
|
||||
- NTFY_ATTACHMENT_CACHE_DIR=/var/cache/ntfy/attachments
|
||||
volumes:
|
||||
- ./ntfy_data:/var/cache/ntfy
|
||||
ports:
|
||||
- "80:80"
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana:11.5.2
|
||||
container_name: monitoring-grafana
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
- GF_SECURITY_ADMIN_USER={{ grafana_admin_user }}
|
||||
- GF_SECURITY_ADMIN_PASSWORD={{ grafana_admin_password }}
|
||||
- GF_USERS_ALLOW_SIGN_UP=false
|
||||
- GF_SERVER_ROOT_URL={{ grafana_root_url }}
|
||||
volumes:
|
||||
- ./grafana_data:/var/lib/grafana
|
||||
- ./grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
ports:
|
||||
- "3000:3000"
|
||||
depends_on:
|
||||
- prometheus
|
||||
|
||||
node-exporter:
|
||||
image: prom/node-exporter:v1.9.0
|
||||
container_name: monitoring-node-exporter
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
- /proc:/host/proc:ro
|
||||
- /sys:/host/sys:ro
|
||||
- /:/rootfs:ro
|
||||
command:
|
||||
- '--path.procfs=/host/proc'
|
||||
- '--path.rootfs=/rootfs'
|
||||
- '--path.sysfs=/host/sys'
|
||||
- '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)'
|
||||
ports:
|
||||
- "9100:9100"
|
||||
@@ -0,0 +1,10 @@
|
||||
apiVersion: 1
|
||||
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
jsonData:
|
||||
timeInterval: 15s
|
||||
@@ -0,0 +1,40 @@
|
||||
global:
|
||||
scrape_interval: 15s
|
||||
evaluation_interval: 15s
|
||||
|
||||
rule_files:
|
||||
- "/etc/prometheus/alerts.yml"
|
||||
|
||||
alerting:
|
||||
alertmanagers:
|
||||
- static_configs:
|
||||
- targets:
|
||||
- "alertmanager:9093"
|
||||
|
||||
scrape_configs:
|
||||
- job_name: "prometheus"
|
||||
static_configs:
|
||||
- targets: ["localhost:9090"]
|
||||
|
||||
- job_name: "node-exporter"
|
||||
static_configs:
|
||||
- targets:
|
||||
- "192.168.1.108:9100"
|
||||
labels:
|
||||
host: "core-mon"
|
||||
env: "prod"
|
||||
- targets:
|
||||
- "192.168.1.104:9100"
|
||||
labels:
|
||||
host: "core-net"
|
||||
env: "prod"
|
||||
- targets:
|
||||
- "192.168.1.102:9100"
|
||||
labels:
|
||||
host: "core-secrets"
|
||||
env: "prod"
|
||||
- targets:
|
||||
- "45.154.185.208:9100"
|
||||
labels:
|
||||
host: "rus-central"
|
||||
env: "prod"
|
||||
@@ -12,4 +12,4 @@ http:
|
||||
grafana:
|
||||
loadBalancer:
|
||||
servers:
|
||||
- url: "http://192.168.1.208:3000"
|
||||
- url: "http://192.168.1.108:3000"
|
||||
@@ -0,0 +1,15 @@
|
||||
http:
|
||||
routers:
|
||||
ntfy:
|
||||
rule: "Host(`ntfy.gunihome.ru`)"
|
||||
entryPoints:
|
||||
- websecure
|
||||
service: ntfy
|
||||
tls:
|
||||
certResolver: letsencrypt
|
||||
|
||||
services:
|
||||
ntfy:
|
||||
loadBalancer:
|
||||
servers:
|
||||
- url: "http://192.168.1.108:80"
|
||||
Reference in new issue
Block a user