feat(monitoring): add core-mon ansible playbook, roles, workflows, and traefik routes
This commit is contained in:
1 parent
9e9b8e9caf
commit
f26e0bfe57
16 files changed
+352
-3
No files matched your search
@@ -0,0 +1,40 @@
|
|||||||
|
name: ansible-core-mon
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches:
|
||||||
|
- main
|
||||||
|
paths:
|
||||||
|
- 'ansible/ansible.cfg'
|
||||||
|
- 'ansible/ci/**'
|
||||||
|
- 'ansible/roles/core-mon/**'
|
||||||
|
- 'ansible/roles/host/**'
|
||||||
|
- 'ansible/playbooks/core-mon.yml'
|
||||||
|
- 'ansible/playbooks/host.yml'
|
||||||
|
- 'ansible/playbooks/site.yml'
|
||||||
|
- 'ansible/inventories/prod/hosts.yml'
|
||||||
|
- 'ansible/inventories/prod/host_vars/core-mon.yml'
|
||||||
|
- 'ansible/inventories/prod/group_vars/**'
|
||||||
|
workflow_dispatch:
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
deploy:
|
||||||
|
runs-on: homelab
|
||||||
|
container:
|
||||||
|
image: homelab-ansible:latest
|
||||||
|
timeout-minutes: 30
|
||||||
|
steps:
|
||||||
|
- name: Деплой core-mon
|
||||||
|
env:
|
||||||
|
INFISICAL_CLIENT_ID: ${{ secrets.INFISICAL_CLIENT_ID }}
|
||||||
|
INFISICAL_CLIENT_SECRET: ${{ secrets.INFISICAL_CLIENT_SECRET }}
|
||||||
|
run: |
|
||||||
|
set -eu
|
||||||
|
rm -rf "${HOME}/homelab-infra"
|
||||||
|
git clone http://192.168.1.217:3000/karpenok/homelab-infra.git "${HOME}/homelab-infra"
|
||||||
|
sha="${GITHUB_SHA:-${GITEA_SHA:-}}"
|
||||||
|
if [ -n "${sha}" ]; then
|
||||||
|
git -C "${HOME}/homelab-infra" checkout "${sha}"
|
||||||
|
fi
|
||||||
|
chmod -R go-w "${HOME}/homelab-infra"
|
||||||
|
exec bash "${HOME}/homelab-infra/ansible/ci/deploy-machine.sh" core-mon "${{ github.event_name }}" "${{ github.event.before }}" "${{ github.sha }}"
|
||||||
@@ -13,7 +13,7 @@ before=${3:-}
|
|||||||
after=${4:-}
|
after=${4:-}
|
||||||
|
|
||||||
case "$machine" in
|
case "$machine" in
|
||||||
core-net | core-secrets | rus-central | opnsense) ;;
|
core-net | core-secrets | core-mon | rus-central | opnsense) ;;
|
||||||
*)
|
*)
|
||||||
echo "Неизвестная машина: $machine" >&2
|
echo "Неизвестная машина: $machine" >&2
|
||||||
exit 2
|
exit 2
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ before=${3:-}
|
|||||||
after=${4:-}
|
after=${4:-}
|
||||||
|
|
||||||
case "$machine" in
|
case "$machine" in
|
||||||
core-net | core-secrets | rus-central | opnsense) ;;
|
core-net | core-secrets | core-mon | rus-central | opnsense) ;;
|
||||||
*)
|
*)
|
||||||
echo "Неизвестная машина: $machine" >&2
|
echo "Неизвестная машина: $machine" >&2
|
||||||
exit 2
|
exit 2
|
||||||
@@ -68,6 +68,20 @@ case "$machine" in
|
|||||||
export INFISICAL_ENCRYPTION_KEY="${ENCRYPTION_KEY}"
|
export INFISICAL_ENCRYPTION_KEY="${ENCRYPTION_KEY}"
|
||||||
export INFISICAL_AUTH_SECRET="${AUTH_SECRET}"
|
export INFISICAL_AUTH_SECRET="${AUTH_SECRET}"
|
||||||
;;
|
;;
|
||||||
|
core-mon)
|
||||||
|
if ! infisical export --domain="${INFISICAL_DOMAIN}" --token="${TOKEN}" --env=prod --projectId="${INFISICAL_PROJECT_ID}" --path="/services/monitoring" --format=dotenv-export --silent 2>/dev/null | grep -q 'GRAFANA_ADMIN_PASSWORD'; then
|
||||||
|
infisical secrets-manager folders create monitoring --path /services --env prod --projectId "${INFISICAL_PROJECT_ID}" --domain "${INFISICAL_DOMAIN}" --token "${TOKEN}" --silent 2>/dev/null || true
|
||||||
|
GEN_GRAFANA_PASS=$(openssl rand -base64 32 | tr -dc 'a-zA-Z0-9' | head -c 24)
|
||||||
|
infisical secrets set --domain="${INFISICAL_DOMAIN}" --token="${TOKEN}" --env=prod --projectId="${INFISICAL_PROJECT_ID}" --path="/services/monitoring" \
|
||||||
|
"GRAFANA_ADMIN_USER=admin" \
|
||||||
|
"GRAFANA_ADMIN_PASSWORD=${GEN_GRAFANA_PASS}" \
|
||||||
|
--silent
|
||||||
|
unset GEN_GRAFANA_PASS
|
||||||
|
fi
|
||||||
|
eval "$(infisical export --domain="${INFISICAL_DOMAIN}" --token="${TOKEN}" --env=prod --projectId="${INFISICAL_PROJECT_ID}" --path="/services/monitoring" --format=dotenv-export --silent)"
|
||||||
|
export GRAFANA_ADMIN_USER="${GRAFANA_ADMIN_USER:-admin}"
|
||||||
|
export GRAFANA_ADMIN_PASSWORD="${GRAFANA_ADMIN_PASSWORD:-}"
|
||||||
|
;;
|
||||||
rus-central)
|
rus-central)
|
||||||
input_email="${INPUT_LETSENCRYPT_EMAIL:-}"
|
input_email="${INPUT_LETSENCRYPT_EMAIL:-}"
|
||||||
eval "$(infisical export --domain="${INFISICAL_DOMAIN}" --token="${TOKEN}" --env=prod --projectId="${INFISICAL_PROJECT_ID}" --path="/services/netbird" --format=dotenv-export --silent)"
|
eval "$(infisical export --domain="${INFISICAL_DOMAIN}" --token="${TOKEN}" --env=prod --projectId="${INFISICAL_PROJECT_ID}" --path="/services/netbird" --format=dotenv-export --silent)"
|
||||||
|
|||||||
@@ -0,0 +1,4 @@
|
|||||||
|
---
|
||||||
|
monitoring_dir: /opt/monitoring
|
||||||
|
grafana_domain: "graf.gunihome.ru"
|
||||||
|
ntfy_domain: "ntfy.gunihome.ru"
|
||||||
@@ -7,6 +7,9 @@ all:
|
|||||||
core-secrets:
|
core-secrets:
|
||||||
ansible_host: 192.168.1.102
|
ansible_host: 192.168.1.102
|
||||||
ansible_user: root
|
ansible_user: root
|
||||||
|
core-mon:
|
||||||
|
ansible_host: 192.168.1.108
|
||||||
|
ansible_user: root
|
||||||
rus-central:
|
rus-central:
|
||||||
ansible_host: 45.154.185.208
|
ansible_host: 45.154.185.208
|
||||||
ansible_user: root
|
ansible_user: root
|
||||||
|
|||||||
@@ -0,0 +1,13 @@
|
|||||||
|
---
|
||||||
|
- name: Подготовка машины core-mon
|
||||||
|
hosts: core-mon
|
||||||
|
gather_facts: false
|
||||||
|
roles:
|
||||||
|
- role: host
|
||||||
|
tags: [host]
|
||||||
|
|
||||||
|
- name: Развертывание стека мониторинга на core-mon
|
||||||
|
hosts: core-mon
|
||||||
|
roles:
|
||||||
|
- role: core-mon/monitoring
|
||||||
|
tags: [monitoring]
|
||||||
@@ -2,5 +2,6 @@
|
|||||||
- ansible.builtin.import_playbook: host.yml
|
- ansible.builtin.import_playbook: host.yml
|
||||||
- ansible.builtin.import_playbook: core-net.yml
|
- ansible.builtin.import_playbook: core-net.yml
|
||||||
- ansible.builtin.import_playbook: core-secrets.yml
|
- ansible.builtin.import_playbook: core-secrets.yml
|
||||||
|
- ansible.builtin.import_playbook: core-mon.yml
|
||||||
- ansible.builtin.import_playbook: rus-central.yml
|
- ansible.builtin.import_playbook: rus-central.yml
|
||||||
- ansible.builtin.import_playbook: opnsense.yml
|
- ansible.builtin.import_playbook: opnsense.yml
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
---
|
||||||
|
monitoring_dir: /opt/monitoring
|
||||||
|
prometheus_retention: "30d"
|
||||||
|
|
||||||
|
grafana_admin_user: "{{ lookup('ansible.builtin.env', 'GRAFANA_ADMIN_USER', default='admin') }}"
|
||||||
|
grafana_admin_password: "{{ lookup('ansible.builtin.env', 'GRAFANA_ADMIN_PASSWORD', default='') }}"
|
||||||
|
grafana_root_url: "https://graf.gunihome.ru"
|
||||||
|
|
||||||
|
ntfy_base_url: "https://ntfy.gunihome.ru"
|
||||||
|
ntfy_default_topic: "homelab-alerts"
|
||||||
@@ -0,0 +1,64 @@
|
|||||||
|
---
|
||||||
|
- name: Создать базовые каталоги мониторинга
|
||||||
|
ansible.builtin.file:
|
||||||
|
path: "{{ item.path }}"
|
||||||
|
state: directory
|
||||||
|
owner: "{{ item.owner }}"
|
||||||
|
group: "{{ item.group }}"
|
||||||
|
mode: "{{ item.mode }}"
|
||||||
|
loop:
|
||||||
|
- { path: "{{ monitoring_dir }}", owner: "root", group: "root", mode: "0755" }
|
||||||
|
- { path: "{{ monitoring_dir }}/prometheus", owner: "root", group: "root", mode: "0755" }
|
||||||
|
- { path: "{{ monitoring_dir }}/prometheus_data", owner: "65534", group: "65534", mode: "0777" }
|
||||||
|
- { path: "{{ monitoring_dir }}/alertmanager", owner: "root", group: "root", mode: "0755" }
|
||||||
|
- { path: "{{ monitoring_dir }}/alertmanager_data", owner: "65534", group: "65534", mode: "0777" }
|
||||||
|
- { path: "{{ monitoring_dir }}/grafana", owner: "root", group: "root", mode: "0755" }
|
||||||
|
- { path: "{{ monitoring_dir }}/grafana_data", owner: "472", group: "472", mode: "0755" }
|
||||||
|
- { path: "{{ monitoring_dir }}/grafana/provisioning", owner: "root", group: "root", mode: "0755" }
|
||||||
|
- { path: "{{ monitoring_dir }}/grafana/provisioning/datasources", owner: "root", group: "root", mode: "0755" }
|
||||||
|
- { path: "{{ monitoring_dir }}/ntfy_data", owner: "root", group: "root", mode: "0777" }
|
||||||
|
|
||||||
|
- name: Установить конфигурацию Prometheus
|
||||||
|
ansible.builtin.template:
|
||||||
|
src: prometheus.yml.j2
|
||||||
|
dest: "{{ monitoring_dir }}/prometheus/prometheus.yml"
|
||||||
|
owner: root
|
||||||
|
group: root
|
||||||
|
mode: "0644"
|
||||||
|
|
||||||
|
- name: Установить правила алертов Prometheus
|
||||||
|
ansible.builtin.template:
|
||||||
|
src: alerts.yml.j2
|
||||||
|
dest: "{{ monitoring_dir }}/prometheus/alerts.yml"
|
||||||
|
owner: root
|
||||||
|
group: root
|
||||||
|
mode: "0644"
|
||||||
|
|
||||||
|
- name: Установить конфигурацию Alertmanager
|
||||||
|
ansible.builtin.template:
|
||||||
|
src: alertmanager.yml.j2
|
||||||
|
dest: "{{ monitoring_dir }}/alertmanager/alertmanager.yml"
|
||||||
|
owner: root
|
||||||
|
group: root
|
||||||
|
mode: "0644"
|
||||||
|
|
||||||
|
- name: Установить источник данных Prometheus в Grafana
|
||||||
|
ansible.builtin.template:
|
||||||
|
src: grafana-datasources.yml.j2
|
||||||
|
dest: "{{ monitoring_dir }}/grafana/provisioning/datasources/prometheus.yml"
|
||||||
|
owner: root
|
||||||
|
group: root
|
||||||
|
mode: "0644"
|
||||||
|
|
||||||
|
- name: Установить compose.yaml для стека мониторинга
|
||||||
|
ansible.builtin.template:
|
||||||
|
src: compose.yaml.j2
|
||||||
|
dest: "{{ monitoring_dir }}/compose.yaml"
|
||||||
|
owner: root
|
||||||
|
group: root
|
||||||
|
mode: "0644"
|
||||||
|
|
||||||
|
- name: Запустить стек мониторинга через Docker Compose
|
||||||
|
ansible.builtin.command: docker compose -f {{ monitoring_dir }}/compose.yaml up -d --remove-orphans
|
||||||
|
register: compose_up
|
||||||
|
changed_when: "'Started' in compose_up.stderr or 'Created' in compose_up.stderr or 'Recreated' in compose_up.stderr or 'Pulled' in compose_up.stderr"
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
global:
|
||||||
|
resolve_timeout: 5m
|
||||||
|
|
||||||
|
route:
|
||||||
|
group_by: ['alertname', 'instance', 'severity']
|
||||||
|
group_wait: 30s
|
||||||
|
group_interval: 5m
|
||||||
|
repeat_interval: 4h
|
||||||
|
receiver: 'ntfy'
|
||||||
|
|
||||||
|
receivers:
|
||||||
|
- name: 'ntfy'
|
||||||
|
webhook_configs:
|
||||||
|
- url: "http://ntfy:80/{{ ntfy_default_topic }}"
|
||||||
|
send_resolved: true
|
||||||
@@ -0,0 +1,38 @@
|
|||||||
|
groups:
|
||||||
|
- name: homelab_alerts
|
||||||
|
rules:
|
||||||
|
- alert: HostDown
|
||||||
|
expr: up == 0
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Сервис или узел {{ $labels.instance }} недоступен"
|
||||||
|
description: "Экспортер {{ $labels.instance }} (job={{ $labels.job }}) не отвечает более 2 минут."
|
||||||
|
|
||||||
|
- alert: HostDiskSpaceLow
|
||||||
|
expr: (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100 < 15
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Мало свободного места на {{ $labels.instance }}"
|
||||||
|
description: "На корневом разделе осталось менее 15% свободного места."
|
||||||
|
|
||||||
|
- alert: HostHighCpuLoad
|
||||||
|
expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90
|
||||||
|
for: 10m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Высокая нагрузка CPU на {{ $labels.instance }}"
|
||||||
|
description: "Утилизация CPU превышает 90% более 10 минут."
|
||||||
|
|
||||||
|
- alert: HostHighMemoryUsage
|
||||||
|
expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 90
|
||||||
|
for: 10m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Мало оперативной памяти на {{ $labels.instance }}"
|
||||||
|
description: "Использование RAM превышает 90% более 10 минут."
|
||||||
@@ -0,0 +1,82 @@
|
|||||||
|
name: monitoring
|
||||||
|
|
||||||
|
services:
|
||||||
|
prometheus:
|
||||||
|
image: prom/prometheus:v3.2.1
|
||||||
|
container_name: monitoring-prometheus
|
||||||
|
restart: unless-stopped
|
||||||
|
command:
|
||||||
|
- "--config.file=/etc/prometheus/prometheus.yml"
|
||||||
|
- "--storage.tsdb.path=/prometheus"
|
||||||
|
- "--storage.tsdb.retention.time={{ prometheus_retention }}"
|
||||||
|
- "--web.enable-lifecycle"
|
||||||
|
volumes:
|
||||||
|
- ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
||||||
|
- ./prometheus/alerts.yml:/etc/prometheus/alerts.yml:ro
|
||||||
|
- ./prometheus_data:/prometheus
|
||||||
|
ports:
|
||||||
|
- "9090:9090"
|
||||||
|
depends_on:
|
||||||
|
- alertmanager
|
||||||
|
|
||||||
|
alertmanager:
|
||||||
|
image: prom/alertmanager:v0.28.1
|
||||||
|
container_name: monitoring-alertmanager
|
||||||
|
restart: unless-stopped
|
||||||
|
command:
|
||||||
|
- "--config.file=/etc/alertmanager/alertmanager.yml"
|
||||||
|
- "--storage.path=/alertmanager"
|
||||||
|
volumes:
|
||||||
|
- ./alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
|
||||||
|
- ./alertmanager_data:/alertmanager
|
||||||
|
ports:
|
||||||
|
- "9093:9093"
|
||||||
|
|
||||||
|
ntfy:
|
||||||
|
image: binwiederhier/ntfy:v2.11.0
|
||||||
|
container_name: monitoring-ntfy
|
||||||
|
restart: unless-stopped
|
||||||
|
command:
|
||||||
|
- "serve"
|
||||||
|
environment:
|
||||||
|
- NTFY_BASE_URL={{ ntfy_base_url }}
|
||||||
|
- NTFY_BEHIND_PROXY=true
|
||||||
|
- NTFY_CACHE_FILE=/var/cache/ntfy/cache.db
|
||||||
|
- NTFY_ATTACHMENT_CACHE_DIR=/var/cache/ntfy/attachments
|
||||||
|
volumes:
|
||||||
|
- ./ntfy_data:/var/cache/ntfy
|
||||||
|
ports:
|
||||||
|
- "80:80"
|
||||||
|
|
||||||
|
grafana:
|
||||||
|
image: grafana/grafana:11.5.2
|
||||||
|
container_name: monitoring-grafana
|
||||||
|
restart: unless-stopped
|
||||||
|
environment:
|
||||||
|
- GF_SECURITY_ADMIN_USER={{ grafana_admin_user }}
|
||||||
|
- GF_SECURITY_ADMIN_PASSWORD={{ grafana_admin_password }}
|
||||||
|
- GF_USERS_ALLOW_SIGN_UP=false
|
||||||
|
- GF_SERVER_ROOT_URL={{ grafana_root_url }}
|
||||||
|
volumes:
|
||||||
|
- ./grafana_data:/var/lib/grafana
|
||||||
|
- ./grafana/provisioning:/etc/grafana/provisioning:ro
|
||||||
|
ports:
|
||||||
|
- "3000:3000"
|
||||||
|
depends_on:
|
||||||
|
- prometheus
|
||||||
|
|
||||||
|
node-exporter:
|
||||||
|
image: prom/node-exporter:v1.9.0
|
||||||
|
container_name: monitoring-node-exporter
|
||||||
|
restart: unless-stopped
|
||||||
|
volumes:
|
||||||
|
- /proc:/host/proc:ro
|
||||||
|
- /sys:/host/sys:ro
|
||||||
|
- /:/rootfs:ro
|
||||||
|
command:
|
||||||
|
- '--path.procfs=/host/proc'
|
||||||
|
- '--path.rootfs=/rootfs'
|
||||||
|
- '--path.sysfs=/host/sys'
|
||||||
|
- '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)'
|
||||||
|
ports:
|
||||||
|
- "9100:9100"
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
apiVersion: 1
|
||||||
|
|
||||||
|
datasources:
|
||||||
|
- name: Prometheus
|
||||||
|
type: prometheus
|
||||||
|
access: proxy
|
||||||
|
url: http://prometheus:9090
|
||||||
|
isDefault: true
|
||||||
|
jsonData:
|
||||||
|
timeInterval: 15s
|
||||||
@@ -0,0 +1,40 @@
|
|||||||
|
global:
|
||||||
|
scrape_interval: 15s
|
||||||
|
evaluation_interval: 15s
|
||||||
|
|
||||||
|
rule_files:
|
||||||
|
- "/etc/prometheus/alerts.yml"
|
||||||
|
|
||||||
|
alerting:
|
||||||
|
alertmanagers:
|
||||||
|
- static_configs:
|
||||||
|
- targets:
|
||||||
|
- "alertmanager:9093"
|
||||||
|
|
||||||
|
scrape_configs:
|
||||||
|
- job_name: "prometheus"
|
||||||
|
static_configs:
|
||||||
|
- targets: ["localhost:9090"]
|
||||||
|
|
||||||
|
- job_name: "node-exporter"
|
||||||
|
static_configs:
|
||||||
|
- targets:
|
||||||
|
- "192.168.1.108:9100"
|
||||||
|
labels:
|
||||||
|
host: "core-mon"
|
||||||
|
env: "prod"
|
||||||
|
- targets:
|
||||||
|
- "192.168.1.104:9100"
|
||||||
|
labels:
|
||||||
|
host: "core-net"
|
||||||
|
env: "prod"
|
||||||
|
- targets:
|
||||||
|
- "192.168.1.102:9100"
|
||||||
|
labels:
|
||||||
|
host: "core-secrets"
|
||||||
|
env: "prod"
|
||||||
|
- targets:
|
||||||
|
- "45.154.185.208:9100"
|
||||||
|
labels:
|
||||||
|
host: "rus-central"
|
||||||
|
env: "prod"
|
||||||
@@ -12,4 +12,4 @@ http:
|
|||||||
grafana:
|
grafana:
|
||||||
loadBalancer:
|
loadBalancer:
|
||||||
servers:
|
servers:
|
||||||
- url: "http://192.168.1.208:3000"
|
- url: "http://192.168.1.108:3000"
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
http:
|
||||||
|
routers:
|
||||||
|
ntfy:
|
||||||
|
rule: "Host(`ntfy.gunihome.ru`)"
|
||||||
|
entryPoints:
|
||||||
|
- websecure
|
||||||
|
service: ntfy
|
||||||
|
tls:
|
||||||
|
certResolver: letsencrypt
|
||||||
|
|
||||||
|
services:
|
||||||
|
ntfy:
|
||||||
|
loadBalancer:
|
||||||
|
servers:
|
||||||
|
- url: "http://192.168.1.108:80"
|
||||||
Reference in new issue
Block a user