feat(monitoring): add core-mon ansible playbook, roles, workflows, and traefik routes
ansible-core-mon / deploy (push) Failing after 7s
ansible-core-net / deploy (push) Successful in 26s
ansible-core-secrets / deploy (push) Successful in 12s
ansible-opnsense / deploy (push) Successful in 3s
ansible-rus-central / deploy (push) Successful in 44s

This commit is contained in:
karpenok committed 2026-11-11 16:00:53 +07:00
1 parent 9e9b8e9caf
commit f26e0bfe57
16 files changed
+352 -3

No files matched your search

+40
View File
@@ -0,0 +1,40 @@
name: ansible-core-mon
on:
push:
branches:
- main
paths:
- 'ansible/ansible.cfg'
- 'ansible/ci/**'
- 'ansible/roles/core-mon/**'
- 'ansible/roles/host/**'
- 'ansible/playbooks/core-mon.yml'
- 'ansible/playbooks/host.yml'
- 'ansible/playbooks/site.yml'
- 'ansible/inventories/prod/hosts.yml'
- 'ansible/inventories/prod/host_vars/core-mon.yml'
- 'ansible/inventories/prod/group_vars/**'
workflow_dispatch:
jobs:
deploy:
runs-on: homelab
container:
image: homelab-ansible:latest
timeout-minutes: 30
steps:
- name: Деплой core-mon
env:
INFISICAL_CLIENT_ID: ${{ secrets.INFISICAL_CLIENT_ID }}
INFISICAL_CLIENT_SECRET: ${{ secrets.INFISICAL_CLIENT_SECRET }}
run: |
set -eu
rm -rf "${HOME}/homelab-infra"
git clone http://192.168.1.217:3000/karpenok/homelab-infra.git "${HOME}/homelab-infra"
sha="${GITHUB_SHA:-${GITEA_SHA:-}}"
if [ -n "${sha}" ]; then
git -C "${HOME}/homelab-infra" checkout "${sha}"
fi
chmod -R go-w "${HOME}/homelab-infra"
exec bash "${HOME}/homelab-infra/ansible/ci/deploy-machine.sh" core-mon "${{ github.event_name }}" "${{ github.event.before }}" "${{ github.sha }}"
+1 -1
View File
@@ -13,7 +13,7 @@ before=${3:-}
after=${4:-} after=${4:-}
case "$machine" in case "$machine" in
core-net | core-secrets | rus-central | opnsense) ;; core-net | core-secrets | core-mon | rus-central | opnsense) ;;
*) *)
echo "Неизвестная машина: $machine" >&2 echo "Неизвестная машина: $machine" >&2
exit 2 exit 2
+15 -1
View File
@@ -9,7 +9,7 @@ before=${3:-}
after=${4:-} after=${4:-}
case "$machine" in case "$machine" in
core-net | core-secrets | rus-central | opnsense) ;; core-net | core-secrets | core-mon | rus-central | opnsense) ;;
*) *)
echo "Неизвестная машина: $machine" >&2 echo "Неизвестная машина: $machine" >&2
exit 2 exit 2
@@ -68,6 +68,20 @@ case "$machine" in
export INFISICAL_ENCRYPTION_KEY="${ENCRYPTION_KEY}" export INFISICAL_ENCRYPTION_KEY="${ENCRYPTION_KEY}"
export INFISICAL_AUTH_SECRET="${AUTH_SECRET}" export INFISICAL_AUTH_SECRET="${AUTH_SECRET}"
;; ;;
core-mon)
if ! infisical export --domain="${INFISICAL_DOMAIN}" --token="${TOKEN}" --env=prod --projectId="${INFISICAL_PROJECT_ID}" --path="/services/monitoring" --format=dotenv-export --silent 2>/dev/null | grep -q 'GRAFANA_ADMIN_PASSWORD'; then
infisical secrets-manager folders create monitoring --path /services --env prod --projectId "${INFISICAL_PROJECT_ID}" --domain "${INFISICAL_DOMAIN}" --token "${TOKEN}" --silent 2>/dev/null || true
GEN_GRAFANA_PASS=$(openssl rand -base64 32 | tr -dc 'a-zA-Z0-9' | head -c 24)
infisical secrets set --domain="${INFISICAL_DOMAIN}" --token="${TOKEN}" --env=prod --projectId="${INFISICAL_PROJECT_ID}" --path="/services/monitoring" \
"GRAFANA_ADMIN_USER=admin" \
"GRAFANA_ADMIN_PASSWORD=${GEN_GRAFANA_PASS}" \
--silent
unset GEN_GRAFANA_PASS
fi
eval "$(infisical export --domain="${INFISICAL_DOMAIN}" --token="${TOKEN}" --env=prod --projectId="${INFISICAL_PROJECT_ID}" --path="/services/monitoring" --format=dotenv-export --silent)"
export GRAFANA_ADMIN_USER="${GRAFANA_ADMIN_USER:-admin}"
export GRAFANA_ADMIN_PASSWORD="${GRAFANA_ADMIN_PASSWORD:-}"
;;
rus-central) rus-central)
input_email="${INPUT_LETSENCRYPT_EMAIL:-}" input_email="${INPUT_LETSENCRYPT_EMAIL:-}"
eval "$(infisical export --domain="${INFISICAL_DOMAIN}" --token="${TOKEN}" --env=prod --projectId="${INFISICAL_PROJECT_ID}" --path="/services/netbird" --format=dotenv-export --silent)" eval "$(infisical export --domain="${INFISICAL_DOMAIN}" --token="${TOKEN}" --env=prod --projectId="${INFISICAL_PROJECT_ID}" --path="/services/netbird" --format=dotenv-export --silent)"
@@ -0,0 +1,4 @@
---
monitoring_dir: /opt/monitoring
grafana_domain: "graf.gunihome.ru"
ntfy_domain: "ntfy.gunihome.ru"
+3
View File
@@ -7,6 +7,9 @@ all:
core-secrets: core-secrets:
ansible_host: 192.168.1.102 ansible_host: 192.168.1.102
ansible_user: root ansible_user: root
core-mon:
ansible_host: 192.168.1.108
ansible_user: root
rus-central: rus-central:
ansible_host: 45.154.185.208 ansible_host: 45.154.185.208
ansible_user: root ansible_user: root
+13
View File
@@ -0,0 +1,13 @@
---
- name: Подготовка машины core-mon
hosts: core-mon
gather_facts: false
roles:
- role: host
tags: [host]
- name: Развертывание стека мониторинга на core-mon
hosts: core-mon
roles:
- role: core-mon/monitoring
tags: [monitoring]
+1
View File
@@ -2,5 +2,6 @@
- ansible.builtin.import_playbook: host.yml - ansible.builtin.import_playbook: host.yml
- ansible.builtin.import_playbook: core-net.yml - ansible.builtin.import_playbook: core-net.yml
- ansible.builtin.import_playbook: core-secrets.yml - ansible.builtin.import_playbook: core-secrets.yml
- ansible.builtin.import_playbook: core-mon.yml
- ansible.builtin.import_playbook: rus-central.yml - ansible.builtin.import_playbook: rus-central.yml
- ansible.builtin.import_playbook: opnsense.yml - ansible.builtin.import_playbook: opnsense.yml
@@ -0,0 +1,10 @@
---
monitoring_dir: /opt/monitoring
prometheus_retention: "30d"
grafana_admin_user: "{{ lookup('ansible.builtin.env', 'GRAFANA_ADMIN_USER', default='admin') }}"
grafana_admin_password: "{{ lookup('ansible.builtin.env', 'GRAFANA_ADMIN_PASSWORD', default='') }}"
grafana_root_url: "https://graf.gunihome.ru"
ntfy_base_url: "https://ntfy.gunihome.ru"
ntfy_default_topic: "homelab-alerts"
@@ -0,0 +1,64 @@
---
- name: Создать базовые каталоги мониторинга
ansible.builtin.file:
path: "{{ item.path }}"
state: directory
owner: "{{ item.owner }}"
group: "{{ item.group }}"
mode: "{{ item.mode }}"
loop:
- { path: "{{ monitoring_dir }}", owner: "root", group: "root", mode: "0755" }
- { path: "{{ monitoring_dir }}/prometheus", owner: "root", group: "root", mode: "0755" }
- { path: "{{ monitoring_dir }}/prometheus_data", owner: "65534", group: "65534", mode: "0777" }
- { path: "{{ monitoring_dir }}/alertmanager", owner: "root", group: "root", mode: "0755" }
- { path: "{{ monitoring_dir }}/alertmanager_data", owner: "65534", group: "65534", mode: "0777" }
- { path: "{{ monitoring_dir }}/grafana", owner: "root", group: "root", mode: "0755" }
- { path: "{{ monitoring_dir }}/grafana_data", owner: "472", group: "472", mode: "0755" }
- { path: "{{ monitoring_dir }}/grafana/provisioning", owner: "root", group: "root", mode: "0755" }
- { path: "{{ monitoring_dir }}/grafana/provisioning/datasources", owner: "root", group: "root", mode: "0755" }
- { path: "{{ monitoring_dir }}/ntfy_data", owner: "root", group: "root", mode: "0777" }
- name: Установить конфигурацию Prometheus
ansible.builtin.template:
src: prometheus.yml.j2
dest: "{{ monitoring_dir }}/prometheus/prometheus.yml"
owner: root
group: root
mode: "0644"
- name: Установить правила алертов Prometheus
ansible.builtin.template:
src: alerts.yml.j2
dest: "{{ monitoring_dir }}/prometheus/alerts.yml"
owner: root
group: root
mode: "0644"
- name: Установить конфигурацию Alertmanager
ansible.builtin.template:
src: alertmanager.yml.j2
dest: "{{ monitoring_dir }}/alertmanager/alertmanager.yml"
owner: root
group: root
mode: "0644"
- name: Установить источник данных Prometheus в Grafana
ansible.builtin.template:
src: grafana-datasources.yml.j2
dest: "{{ monitoring_dir }}/grafana/provisioning/datasources/prometheus.yml"
owner: root
group: root
mode: "0644"
- name: Установить compose.yaml для стека мониторинга
ansible.builtin.template:
src: compose.yaml.j2
dest: "{{ monitoring_dir }}/compose.yaml"
owner: root
group: root
mode: "0644"
- name: Запустить стек мониторинга через Docker Compose
ansible.builtin.command: docker compose -f {{ monitoring_dir }}/compose.yaml up -d --remove-orphans
register: compose_up
changed_when: "'Started' in compose_up.stderr or 'Created' in compose_up.stderr or 'Recreated' in compose_up.stderr or 'Pulled' in compose_up.stderr"
@@ -0,0 +1,15 @@
global:
resolve_timeout: 5m
route:
group_by: ['alertname', 'instance', 'severity']
group_wait: 30s
group_interval: 5m
repeat_interval: 4h
receiver: 'ntfy'
receivers:
- name: 'ntfy'
webhook_configs:
- url: "http://ntfy:80/{{ ntfy_default_topic }}"
send_resolved: true
@@ -0,0 +1,38 @@
groups:
- name: homelab_alerts
rules:
- alert: HostDown
expr: up == 0
for: 2m
labels:
severity: critical
annotations:
summary: "Сервис или узел {{ $labels.instance }} недоступен"
description: "Экспортер {{ $labels.instance }} (job={{ $labels.job }}) не отвечает более 2 минут."
- alert: HostDiskSpaceLow
expr: (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100 < 15
for: 5m
labels:
severity: warning
annotations:
summary: "Мало свободного места на {{ $labels.instance }}"
description: "На корневом разделе осталось менее 15% свободного места."
- alert: HostHighCpuLoad
expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90
for: 10m
labels:
severity: warning
annotations:
summary: "Высокая нагрузка CPU на {{ $labels.instance }}"
description: "Утилизация CPU превышает 90% более 10 минут."
- alert: HostHighMemoryUsage
expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 90
for: 10m
labels:
severity: warning
annotations:
summary: "Мало оперативной памяти на {{ $labels.instance }}"
description: "Использование RAM превышает 90% более 10 минут."
@@ -0,0 +1,82 @@
name: monitoring
services:
prometheus:
image: prom/prometheus:v3.2.1
container_name: monitoring-prometheus
restart: unless-stopped
command:
- "--config.file=/etc/prometheus/prometheus.yml"
- "--storage.tsdb.path=/prometheus"
- "--storage.tsdb.retention.time={{ prometheus_retention }}"
- "--web.enable-lifecycle"
volumes:
- ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
- ./prometheus/alerts.yml:/etc/prometheus/alerts.yml:ro
- ./prometheus_data:/prometheus
ports:
- "9090:9090"
depends_on:
- alertmanager
alertmanager:
image: prom/alertmanager:v0.28.1
container_name: monitoring-alertmanager
restart: unless-stopped
command:
- "--config.file=/etc/alertmanager/alertmanager.yml"
- "--storage.path=/alertmanager"
volumes:
- ./alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
- ./alertmanager_data:/alertmanager
ports:
- "9093:9093"
ntfy:
image: binwiederhier/ntfy:v2.11.0
container_name: monitoring-ntfy
restart: unless-stopped
command:
- "serve"
environment:
- NTFY_BASE_URL={{ ntfy_base_url }}
- NTFY_BEHIND_PROXY=true
- NTFY_CACHE_FILE=/var/cache/ntfy/cache.db
- NTFY_ATTACHMENT_CACHE_DIR=/var/cache/ntfy/attachments
volumes:
- ./ntfy_data:/var/cache/ntfy
ports:
- "80:80"
grafana:
image: grafana/grafana:11.5.2
container_name: monitoring-grafana
restart: unless-stopped
environment:
- GF_SECURITY_ADMIN_USER={{ grafana_admin_user }}
- GF_SECURITY_ADMIN_PASSWORD={{ grafana_admin_password }}
- GF_USERS_ALLOW_SIGN_UP=false
- GF_SERVER_ROOT_URL={{ grafana_root_url }}
volumes:
- ./grafana_data:/var/lib/grafana
- ./grafana/provisioning:/etc/grafana/provisioning:ro
ports:
- "3000:3000"
depends_on:
- prometheus
node-exporter:
image: prom/node-exporter:v1.9.0
container_name: monitoring-node-exporter
restart: unless-stopped
volumes:
- /proc:/host/proc:ro
- /sys:/host/sys:ro
- /:/rootfs:ro
command:
- '--path.procfs=/host/proc'
- '--path.rootfs=/rootfs'
- '--path.sysfs=/host/sys'
- '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)'
ports:
- "9100:9100"
@@ -0,0 +1,10 @@
apiVersion: 1
datasources:
- name: Prometheus
type: prometheus
access: proxy
url: http://prometheus:9090
isDefault: true
jsonData:
timeInterval: 15s
@@ -0,0 +1,40 @@
global:
scrape_interval: 15s
evaluation_interval: 15s
rule_files:
- "/etc/prometheus/alerts.yml"
alerting:
alertmanagers:
- static_configs:
- targets:
- "alertmanager:9093"
scrape_configs:
- job_name: "prometheus"
static_configs:
- targets: ["localhost:9090"]
- job_name: "node-exporter"
static_configs:
- targets:
- "192.168.1.108:9100"
labels:
host: "core-mon"
env: "prod"
- targets:
- "192.168.1.104:9100"
labels:
host: "core-net"
env: "prod"
- targets:
- "192.168.1.102:9100"
labels:
host: "core-secrets"
env: "prod"
- targets:
- "45.154.185.208:9100"
labels:
host: "rus-central"
env: "prod"
@@ -12,4 +12,4 @@ http:
grafana: grafana:
loadBalancer: loadBalancer:
servers: servers:
- url: "http://192.168.1.208:3000" - url: "http://192.168.1.108:3000"
@@ -0,0 +1,15 @@
http:
routers:
ntfy:
rule: "Host(`ntfy.gunihome.ru`)"
entryPoints:
- websecure
service: ntfy
tls:
certResolver: letsencrypt
services:
ntfy:
loadBalancer:
servers:
- url: "http://192.168.1.108:80"