From f26e0bfe578443064c11b0b341fbc589ad16f82c Mon Sep 17 00:00:00 2001 From: karpenok Date: Wed, 11 Nov 2026 16:00:53 +0700 Subject: [PATCH] feat(monitoring): add core-mon ansible playbook, roles, workflows, and traefik routes --- .gitea/workflows/ansible-core-mon.yml | 40 +++++++++ ansible/ci/ansible-target.sh | 2 +- ansible/ci/deploy-machine.sh | 16 +++- .../inventories/prod/host_vars/core-mon.yml | 4 + ansible/inventories/prod/hosts.yml | 3 + ansible/playbooks/core-mon.yml | 13 +++ ansible/playbooks/site.yml | 1 + .../core-mon/monitoring/defaults/main.yml | 10 +++ .../roles/core-mon/monitoring/tasks/main.yml | 64 +++++++++++++++ .../monitoring/templates/alertmanager.yml.j2 | 15 ++++ .../monitoring/templates/alerts.yml.j2 | 38 +++++++++ .../monitoring/templates/compose.yaml.j2 | 82 +++++++++++++++++++ .../templates/grafana-datasources.yml.j2 | 10 +++ .../monitoring/templates/prometheus.yml.j2 | 40 +++++++++ .../traefik/files/dynamic/grafana.yml | 2 +- .../core-net/traefik/files/dynamic/ntfy.yml | 15 ++++ 16 files changed, 352 insertions(+), 3 deletions(-) create mode 100644 .gitea/workflows/ansible-core-mon.yml create mode 100644 ansible/inventories/prod/host_vars/core-mon.yml create mode 100644 ansible/playbooks/core-mon.yml create mode 100644 ansible/roles/core-mon/monitoring/defaults/main.yml create mode 100644 ansible/roles/core-mon/monitoring/tasks/main.yml create mode 100644 ansible/roles/core-mon/monitoring/templates/alertmanager.yml.j2 create mode 100644 ansible/roles/core-mon/monitoring/templates/alerts.yml.j2 create mode 100644 ansible/roles/core-mon/monitoring/templates/compose.yaml.j2 create mode 100644 ansible/roles/core-mon/monitoring/templates/grafana-datasources.yml.j2 create mode 100644 ansible/roles/core-mon/monitoring/templates/prometheus.yml.j2 create mode 100644 ansible/roles/core-net/traefik/files/dynamic/ntfy.yml diff --git a/.gitea/workflows/ansible-core-mon.yml b/.gitea/workflows/ansible-core-mon.yml new file mode 100644 index 0000000..c7fcf3e --- /dev/null +++ b/.gitea/workflows/ansible-core-mon.yml @@ -0,0 +1,40 @@ +name: ansible-core-mon + +on: + push: + branches: + - main + paths: + - 'ansible/ansible.cfg' + - 'ansible/ci/**' + - 'ansible/roles/core-mon/**' + - 'ansible/roles/host/**' + - 'ansible/playbooks/core-mon.yml' + - 'ansible/playbooks/host.yml' + - 'ansible/playbooks/site.yml' + - 'ansible/inventories/prod/hosts.yml' + - 'ansible/inventories/prod/host_vars/core-mon.yml' + - 'ansible/inventories/prod/group_vars/**' + workflow_dispatch: + +jobs: + deploy: + runs-on: homelab + container: + image: homelab-ansible:latest + timeout-minutes: 30 + steps: + - name: Деплой core-mon + env: + INFISICAL_CLIENT_ID: ${{ secrets.INFISICAL_CLIENT_ID }} + INFISICAL_CLIENT_SECRET: ${{ secrets.INFISICAL_CLIENT_SECRET }} + run: | + set -eu + rm -rf "${HOME}/homelab-infra" + git clone http://192.168.1.217:3000/karpenok/homelab-infra.git "${HOME}/homelab-infra" + sha="${GITHUB_SHA:-${GITEA_SHA:-}}" + if [ -n "${sha}" ]; then + git -C "${HOME}/homelab-infra" checkout "${sha}" + fi + chmod -R go-w "${HOME}/homelab-infra" + exec bash "${HOME}/homelab-infra/ansible/ci/deploy-machine.sh" core-mon "${{ github.event_name }}" "${{ github.event.before }}" "${{ github.sha }}" diff --git a/ansible/ci/ansible-target.sh b/ansible/ci/ansible-target.sh index 15f48ed..df3bf0c 100755 --- a/ansible/ci/ansible-target.sh +++ b/ansible/ci/ansible-target.sh @@ -13,7 +13,7 @@ before=${3:-} after=${4:-} case "$machine" in - core-net | core-secrets | rus-central | opnsense) ;; + core-net | core-secrets | core-mon | rus-central | opnsense) ;; *) echo "Неизвестная машина: $machine" >&2 exit 2 diff --git a/ansible/ci/deploy-machine.sh b/ansible/ci/deploy-machine.sh index f6d49b0..67f4e58 100755 --- a/ansible/ci/deploy-machine.sh +++ b/ansible/ci/deploy-machine.sh @@ -9,7 +9,7 @@ before=${3:-} after=${4:-} case "$machine" in - core-net | core-secrets | rus-central | opnsense) ;; + core-net | core-secrets | core-mon | rus-central | opnsense) ;; *) echo "Неизвестная машина: $machine" >&2 exit 2 @@ -68,6 +68,20 @@ case "$machine" in export INFISICAL_ENCRYPTION_KEY="${ENCRYPTION_KEY}" export INFISICAL_AUTH_SECRET="${AUTH_SECRET}" ;; + core-mon) + if ! infisical export --domain="${INFISICAL_DOMAIN}" --token="${TOKEN}" --env=prod --projectId="${INFISICAL_PROJECT_ID}" --path="/services/monitoring" --format=dotenv-export --silent 2>/dev/null | grep -q 'GRAFANA_ADMIN_PASSWORD'; then + infisical secrets-manager folders create monitoring --path /services --env prod --projectId "${INFISICAL_PROJECT_ID}" --domain "${INFISICAL_DOMAIN}" --token "${TOKEN}" --silent 2>/dev/null || true + GEN_GRAFANA_PASS=$(openssl rand -base64 32 | tr -dc 'a-zA-Z0-9' | head -c 24) + infisical secrets set --domain="${INFISICAL_DOMAIN}" --token="${TOKEN}" --env=prod --projectId="${INFISICAL_PROJECT_ID}" --path="/services/monitoring" \ + "GRAFANA_ADMIN_USER=admin" \ + "GRAFANA_ADMIN_PASSWORD=${GEN_GRAFANA_PASS}" \ + --silent + unset GEN_GRAFANA_PASS + fi + eval "$(infisical export --domain="${INFISICAL_DOMAIN}" --token="${TOKEN}" --env=prod --projectId="${INFISICAL_PROJECT_ID}" --path="/services/monitoring" --format=dotenv-export --silent)" + export GRAFANA_ADMIN_USER="${GRAFANA_ADMIN_USER:-admin}" + export GRAFANA_ADMIN_PASSWORD="${GRAFANA_ADMIN_PASSWORD:-}" + ;; rus-central) input_email="${INPUT_LETSENCRYPT_EMAIL:-}" eval "$(infisical export --domain="${INFISICAL_DOMAIN}" --token="${TOKEN}" --env=prod --projectId="${INFISICAL_PROJECT_ID}" --path="/services/netbird" --format=dotenv-export --silent)" diff --git a/ansible/inventories/prod/host_vars/core-mon.yml b/ansible/inventories/prod/host_vars/core-mon.yml new file mode 100644 index 0000000..71f9b61 --- /dev/null +++ b/ansible/inventories/prod/host_vars/core-mon.yml @@ -0,0 +1,4 @@ +--- +monitoring_dir: /opt/monitoring +grafana_domain: "graf.gunihome.ru" +ntfy_domain: "ntfy.gunihome.ru" diff --git a/ansible/inventories/prod/hosts.yml b/ansible/inventories/prod/hosts.yml index 9208557..6cbf7ba 100644 --- a/ansible/inventories/prod/hosts.yml +++ b/ansible/inventories/prod/hosts.yml @@ -7,6 +7,9 @@ all: core-secrets: ansible_host: 192.168.1.102 ansible_user: root + core-mon: + ansible_host: 192.168.1.108 + ansible_user: root rus-central: ansible_host: 45.154.185.208 ansible_user: root diff --git a/ansible/playbooks/core-mon.yml b/ansible/playbooks/core-mon.yml new file mode 100644 index 0000000..7961fa6 --- /dev/null +++ b/ansible/playbooks/core-mon.yml @@ -0,0 +1,13 @@ +--- +- name: Подготовка машины core-mon + hosts: core-mon + gather_facts: false + roles: + - role: host + tags: [host] + +- name: Развертывание стека мониторинга на core-mon + hosts: core-mon + roles: + - role: core-mon/monitoring + tags: [monitoring] diff --git a/ansible/playbooks/site.yml b/ansible/playbooks/site.yml index 7144078..d60df84 100644 --- a/ansible/playbooks/site.yml +++ b/ansible/playbooks/site.yml @@ -2,5 +2,6 @@ - ansible.builtin.import_playbook: host.yml - ansible.builtin.import_playbook: core-net.yml - ansible.builtin.import_playbook: core-secrets.yml +- ansible.builtin.import_playbook: core-mon.yml - ansible.builtin.import_playbook: rus-central.yml - ansible.builtin.import_playbook: opnsense.yml diff --git a/ansible/roles/core-mon/monitoring/defaults/main.yml b/ansible/roles/core-mon/monitoring/defaults/main.yml new file mode 100644 index 0000000..a6ae42d --- /dev/null +++ b/ansible/roles/core-mon/monitoring/defaults/main.yml @@ -0,0 +1,10 @@ +--- +monitoring_dir: /opt/monitoring +prometheus_retention: "30d" + +grafana_admin_user: "{{ lookup('ansible.builtin.env', 'GRAFANA_ADMIN_USER', default='admin') }}" +grafana_admin_password: "{{ lookup('ansible.builtin.env', 'GRAFANA_ADMIN_PASSWORD', default='') }}" +grafana_root_url: "https://graf.gunihome.ru" + +ntfy_base_url: "https://ntfy.gunihome.ru" +ntfy_default_topic: "homelab-alerts" diff --git a/ansible/roles/core-mon/monitoring/tasks/main.yml b/ansible/roles/core-mon/monitoring/tasks/main.yml new file mode 100644 index 0000000..88301db --- /dev/null +++ b/ansible/roles/core-mon/monitoring/tasks/main.yml @@ -0,0 +1,64 @@ +--- +- name: Создать базовые каталоги мониторинга + ansible.builtin.file: + path: "{{ item.path }}" + state: directory + owner: "{{ item.owner }}" + group: "{{ item.group }}" + mode: "{{ item.mode }}" + loop: + - { path: "{{ monitoring_dir }}", owner: "root", group: "root", mode: "0755" } + - { path: "{{ monitoring_dir }}/prometheus", owner: "root", group: "root", mode: "0755" } + - { path: "{{ monitoring_dir }}/prometheus_data", owner: "65534", group: "65534", mode: "0777" } + - { path: "{{ monitoring_dir }}/alertmanager", owner: "root", group: "root", mode: "0755" } + - { path: "{{ monitoring_dir }}/alertmanager_data", owner: "65534", group: "65534", mode: "0777" } + - { path: "{{ monitoring_dir }}/grafana", owner: "root", group: "root", mode: "0755" } + - { path: "{{ monitoring_dir }}/grafana_data", owner: "472", group: "472", mode: "0755" } + - { path: "{{ monitoring_dir }}/grafana/provisioning", owner: "root", group: "root", mode: "0755" } + - { path: "{{ monitoring_dir }}/grafana/provisioning/datasources", owner: "root", group: "root", mode: "0755" } + - { path: "{{ monitoring_dir }}/ntfy_data", owner: "root", group: "root", mode: "0777" } + +- name: Установить конфигурацию Prometheus + ansible.builtin.template: + src: prometheus.yml.j2 + dest: "{{ monitoring_dir }}/prometheus/prometheus.yml" + owner: root + group: root + mode: "0644" + +- name: Установить правила алертов Prometheus + ansible.builtin.template: + src: alerts.yml.j2 + dest: "{{ monitoring_dir }}/prometheus/alerts.yml" + owner: root + group: root + mode: "0644" + +- name: Установить конфигурацию Alertmanager + ansible.builtin.template: + src: alertmanager.yml.j2 + dest: "{{ monitoring_dir }}/alertmanager/alertmanager.yml" + owner: root + group: root + mode: "0644" + +- name: Установить источник данных Prometheus в Grafana + ansible.builtin.template: + src: grafana-datasources.yml.j2 + dest: "{{ monitoring_dir }}/grafana/provisioning/datasources/prometheus.yml" + owner: root + group: root + mode: "0644" + +- name: Установить compose.yaml для стека мониторинга + ansible.builtin.template: + src: compose.yaml.j2 + dest: "{{ monitoring_dir }}/compose.yaml" + owner: root + group: root + mode: "0644" + +- name: Запустить стек мониторинга через Docker Compose + ansible.builtin.command: docker compose -f {{ monitoring_dir }}/compose.yaml up -d --remove-orphans + register: compose_up + changed_when: "'Started' in compose_up.stderr or 'Created' in compose_up.stderr or 'Recreated' in compose_up.stderr or 'Pulled' in compose_up.stderr" diff --git a/ansible/roles/core-mon/monitoring/templates/alertmanager.yml.j2 b/ansible/roles/core-mon/monitoring/templates/alertmanager.yml.j2 new file mode 100644 index 0000000..cd2003a --- /dev/null +++ b/ansible/roles/core-mon/monitoring/templates/alertmanager.yml.j2 @@ -0,0 +1,15 @@ +global: + resolve_timeout: 5m + +route: + group_by: ['alertname', 'instance', 'severity'] + group_wait: 30s + group_interval: 5m + repeat_interval: 4h + receiver: 'ntfy' + +receivers: + - name: 'ntfy' + webhook_configs: + - url: "http://ntfy:80/{{ ntfy_default_topic }}" + send_resolved: true diff --git a/ansible/roles/core-mon/monitoring/templates/alerts.yml.j2 b/ansible/roles/core-mon/monitoring/templates/alerts.yml.j2 new file mode 100644 index 0000000..8c1593d --- /dev/null +++ b/ansible/roles/core-mon/monitoring/templates/alerts.yml.j2 @@ -0,0 +1,38 @@ +groups: + - name: homelab_alerts + rules: + - alert: HostDown + expr: up == 0 + for: 2m + labels: + severity: critical + annotations: + summary: "Сервис или узел {{ $labels.instance }} недоступен" + description: "Экспортер {{ $labels.instance }} (job={{ $labels.job }}) не отвечает более 2 минут." + + - alert: HostDiskSpaceLow + expr: (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100 < 15 + for: 5m + labels: + severity: warning + annotations: + summary: "Мало свободного места на {{ $labels.instance }}" + description: "На корневом разделе осталось менее 15% свободного места." + + - alert: HostHighCpuLoad + expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90 + for: 10m + labels: + severity: warning + annotations: + summary: "Высокая нагрузка CPU на {{ $labels.instance }}" + description: "Утилизация CPU превышает 90% более 10 минут." + + - alert: HostHighMemoryUsage + expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 90 + for: 10m + labels: + severity: warning + annotations: + summary: "Мало оперативной памяти на {{ $labels.instance }}" + description: "Использование RAM превышает 90% более 10 минут." diff --git a/ansible/roles/core-mon/monitoring/templates/compose.yaml.j2 b/ansible/roles/core-mon/monitoring/templates/compose.yaml.j2 new file mode 100644 index 0000000..831253d --- /dev/null +++ b/ansible/roles/core-mon/monitoring/templates/compose.yaml.j2 @@ -0,0 +1,82 @@ +name: monitoring + +services: + prometheus: + image: prom/prometheus:v3.2.1 + container_name: monitoring-prometheus + restart: unless-stopped + command: + - "--config.file=/etc/prometheus/prometheus.yml" + - "--storage.tsdb.path=/prometheus" + - "--storage.tsdb.retention.time={{ prometheus_retention }}" + - "--web.enable-lifecycle" + volumes: + - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ./prometheus/alerts.yml:/etc/prometheus/alerts.yml:ro + - ./prometheus_data:/prometheus + ports: + - "9090:9090" + depends_on: + - alertmanager + + alertmanager: + image: prom/alertmanager:v0.28.1 + container_name: monitoring-alertmanager + restart: unless-stopped + command: + - "--config.file=/etc/alertmanager/alertmanager.yml" + - "--storage.path=/alertmanager" + volumes: + - ./alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro + - ./alertmanager_data:/alertmanager + ports: + - "9093:9093" + + ntfy: + image: binwiederhier/ntfy:v2.11.0 + container_name: monitoring-ntfy + restart: unless-stopped + command: + - "serve" + environment: + - NTFY_BASE_URL={{ ntfy_base_url }} + - NTFY_BEHIND_PROXY=true + - NTFY_CACHE_FILE=/var/cache/ntfy/cache.db + - NTFY_ATTACHMENT_CACHE_DIR=/var/cache/ntfy/attachments + volumes: + - ./ntfy_data:/var/cache/ntfy + ports: + - "80:80" + + grafana: + image: grafana/grafana:11.5.2 + container_name: monitoring-grafana + restart: unless-stopped + environment: + - GF_SECURITY_ADMIN_USER={{ grafana_admin_user }} + - GF_SECURITY_ADMIN_PASSWORD={{ grafana_admin_password }} + - GF_USERS_ALLOW_SIGN_UP=false + - GF_SERVER_ROOT_URL={{ grafana_root_url }} + volumes: + - ./grafana_data:/var/lib/grafana + - ./grafana/provisioning:/etc/grafana/provisioning:ro + ports: + - "3000:3000" + depends_on: + - prometheus + + node-exporter: + image: prom/node-exporter:v1.9.0 + container_name: monitoring-node-exporter + restart: unless-stopped + volumes: + - /proc:/host/proc:ro + - /sys:/host/sys:ro + - /:/rootfs:ro + command: + - '--path.procfs=/host/proc' + - '--path.rootfs=/rootfs' + - '--path.sysfs=/host/sys' + - '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)' + ports: + - "9100:9100" diff --git a/ansible/roles/core-mon/monitoring/templates/grafana-datasources.yml.j2 b/ansible/roles/core-mon/monitoring/templates/grafana-datasources.yml.j2 new file mode 100644 index 0000000..e761ed1 --- /dev/null +++ b/ansible/roles/core-mon/monitoring/templates/grafana-datasources.yml.j2 @@ -0,0 +1,10 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + jsonData: + timeInterval: 15s diff --git a/ansible/roles/core-mon/monitoring/templates/prometheus.yml.j2 b/ansible/roles/core-mon/monitoring/templates/prometheus.yml.j2 new file mode 100644 index 0000000..abe6268 --- /dev/null +++ b/ansible/roles/core-mon/monitoring/templates/prometheus.yml.j2 @@ -0,0 +1,40 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + +rule_files: + - "/etc/prometheus/alerts.yml" + +alerting: + alertmanagers: + - static_configs: + - targets: + - "alertmanager:9093" + +scrape_configs: + - job_name: "prometheus" + static_configs: + - targets: ["localhost:9090"] + + - job_name: "node-exporter" + static_configs: + - targets: + - "192.168.1.108:9100" + labels: + host: "core-mon" + env: "prod" + - targets: + - "192.168.1.104:9100" + labels: + host: "core-net" + env: "prod" + - targets: + - "192.168.1.102:9100" + labels: + host: "core-secrets" + env: "prod" + - targets: + - "45.154.185.208:9100" + labels: + host: "rus-central" + env: "prod" diff --git a/ansible/roles/core-net/traefik/files/dynamic/grafana.yml b/ansible/roles/core-net/traefik/files/dynamic/grafana.yml index 31c6d31..e2d9b01 100644 --- a/ansible/roles/core-net/traefik/files/dynamic/grafana.yml +++ b/ansible/roles/core-net/traefik/files/dynamic/grafana.yml @@ -12,4 +12,4 @@ http: grafana: loadBalancer: servers: - - url: "http://192.168.1.208:3000" + - url: "http://192.168.1.108:3000" diff --git a/ansible/roles/core-net/traefik/files/dynamic/ntfy.yml b/ansible/roles/core-net/traefik/files/dynamic/ntfy.yml new file mode 100644 index 0000000..6c1ace7 --- /dev/null +++ b/ansible/roles/core-net/traefik/files/dynamic/ntfy.yml @@ -0,0 +1,15 @@ +http: + routers: + ntfy: + rule: "Host(`ntfy.gunihome.ru`)" + entryPoints: + - websecure + service: ntfy + tls: + certResolver: letsencrypt + + services: + ntfy: + loadBalancer: + servers: + - url: "http://192.168.1.108:80"