feat: VictoriaMetrics и NetBird между панелью и ger-nod-02
ansible-core-net / deploy (push) Canceled after 0s
ansible-core-secrets / deploy (push) Canceled after 0s
ansible-ger-nod-02 / deploy (push) Canceled after 0s
ansible-opnsense / deploy (push) Canceled after 0s
ansible-core-mon / deploy (push) Successful in 58s
ansible-rus-central / deploy (push) Successful in 44s

Prometheus остаётся для LAN. Нода отдаёт метрики через vmagent, а панель ходит на неё по отдельному ключу NetBird, не через группу роутеров.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
karpenokandCursor committed 2026-11-12 16:50:44 +07:00
1 parent 5704272c18
commit 72a9c86e52
46 files changed
+13496 -25

No files matched your search

@@ -11,6 +11,13 @@ monitoring_node_exporter_port: 9100
monitoring_scrape_job_name: node-exporter
monitoring_env_label: prod
victoriametrics_image: victoriametrics/victoria-metrics:v1.153.0
vmagent_image: victoriametrics/vmagent:v1.153.0
cadvisor_image: gcr.io/cadvisor/cadvisor:v0.49.2
xray_checker_image: kutovoys/xray-checker:v1.3.1
xray_checker_subscription_url: "{{ lookup('env', 'XRAY_CHECKER_SUBSCRIPTION_URL', default='') }}"
vmagent_cluster: homelab
grafana_admin_user: "{{ lookup('ansible.builtin.env', 'ADMIN_USER', default='karpenok') }}"
grafana_admin_password: "{{ lookup('ansible.builtin.env', 'ADMIN_PASSWORD', default='') }}"
grafana_admin_email: "{{ lookup('ansible.builtin.env', 'MAIL_ADMIN_USER', default=grafana_admin_user ~ '@gunihome.ru') }}"
@@ -0,0 +1,817 @@
{
"__inputs": [
{
"name": "DS_PROMETHEUS",
"label": "Prometheus",
"description": "Prometheus as the datasource is obligatory",
"type": "datasource",
"pluginId": "prometheus",
"pluginName": "Prometheus"
}
],
"__requires": [
{
"type": "grafana",
"id": "grafana",
"name": "Grafana",
"version": "7.4.5"
},
{
"type": "panel",
"id": "graph",
"name": "Graph",
"version": ""
},
{
"type": "datasource",
"id": "prometheus",
"name": "Prometheus",
"version": "1.0.0"
},
{
"type": "panel",
"id": "table",
"name": "Table",
"version": ""
}
],
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"gnetId": 14282,
"graphTooltip": 0,
"id": null,
"iteration": 1617715580880,
"links": [],
"panels": [
{
"collapsed": false,
"datasource": "${DS_PROMETHEUS}",
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 0
},
"id": 8,
"panels": [],
"title": "CPU",
"type": "row"
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "${DS_PROMETHEUS}",
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 24,
"x": 0,
"y": 1
},
"hiddenSeries": false,
"id": 15,
"legend": {
"alignAsTable": true,
"avg": true,
"current": false,
"max": true,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null as zero",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.5",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": true,
"steppedLine": false,
"targets": [
{
"expr": "sum(rate(container_cpu_usage_seconds_total{instance=~\"$host\",name=~\"$container\",name=~\".+\"}[5m])) by (name) *100",
"hide": false,
"interval": "",
"legendFormat": "{{name}}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "CPU Usage",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:606",
"format": "percent",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:607",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"collapsed": false,
"datasource": "${DS_PROMETHEUS}",
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 8
},
"id": 11,
"panels": [],
"title": "Memory",
"type": "row"
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "${DS_PROMETHEUS}",
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 9
},
"hiddenSeries": false,
"id": 9,
"legend": {
"alignAsTable": true,
"avg": true,
"current": false,
"max": true,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null as zero",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.5",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": true,
"steppedLine": false,
"targets": [
{
"expr": "sum(container_memory_rss{instance=~\"$host\",name=~\"$container\",name=~\".+\"}) by (name)",
"hide": false,
"interval": "",
"legendFormat": "{{name}}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Memory Usage",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:606",
"format": "bytes",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:607",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "${DS_PROMETHEUS}",
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 9
},
"hiddenSeries": false,
"id": 14,
"legend": {
"alignAsTable": true,
"avg": true,
"current": false,
"max": true,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null as zero",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.5",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": true,
"steppedLine": false,
"targets": [
{
"expr": "sum(container_memory_cache{instance=~\"$host\",name=~\"$container\",name=~\".+\"}) by (name)",
"hide": false,
"interval": "",
"legendFormat": "{{name}}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Memory Cached",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:606",
"format": "bytes",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:607",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"collapsed": false,
"datasource": "${DS_PROMETHEUS}",
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 17
},
"id": 2,
"panels": [],
"title": "Network",
"type": "row"
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "${DS_PROMETHEUS}",
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 18
},
"hiddenSeries": false,
"id": 4,
"legend": {
"alignAsTable": true,
"avg": true,
"current": false,
"hideEmpty": false,
"hideZero": false,
"max": true,
"min": false,
"rightSide": true,
"show": true,
"sideWidth": null,
"total": false,
"values": true
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.5",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "sum(rate(container_network_receive_bytes_total{instance=~\"$host\",name=~\"$container\",name=~\".+\"}[5m])) by (name)",
"hide": false,
"interval": "",
"legendFormat": "{{name}}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Received Network Traffic",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:674",
"format": "Bps",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:675",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "${DS_PROMETHEUS}",
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 18
},
"hiddenSeries": false,
"id": 6,
"legend": {
"alignAsTable": true,
"avg": true,
"current": false,
"max": true,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.5",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "sum(rate(container_network_transmit_bytes_total{instance=~\"$host\",name=~\"$container\",name=~\".+\"}[5m])) by (name)",
"interval": "",
"legendFormat": "{{name}}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Sent Network Traffic",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:832",
"format": "Bps",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:833",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"collapsed": false,
"datasource": "${DS_PROMETHEUS}",
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 26
},
"id": 19,
"panels": [],
"title": "Misc",
"type": "row"
},
{
"datasource": "${DS_PROMETHEUS}",
"fieldConfig": {
"defaults": {
"custom": {
"align": null,
"filterable": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "id"
},
"properties": [
{
"id": "custom.width",
"value": 260
}
]
},
{
"matcher": {
"id": "byName",
"options": "Running"
},
"properties": [
{
"id": "unit",
"value": "d"
},
{
"id": "decimals",
"value": 1
},
{
"id": "custom.displayMode",
"value": "color-text"
},
{
"id": "color",
"value": {
"fixedColor": "dark-green",
"mode": "fixed"
}
}
]
}
]
},
"gridPos": {
"h": 10,
"w": 24,
"x": 0,
"y": 27
},
"id": 17,
"options": {
"showHeader": true,
"sortBy": []
},
"pluginVersion": "7.4.5",
"targets": [
{
"expr": "(time() - container_start_time_seconds{instance=~\"$host\",name=~\"$container\",name=~\".+\"})/86400",
"format": "table",
"instant": true,
"interval": "",
"legendFormat": "{{name}}",
"refId": "A"
}
],
"timeFrom": null,
"timeShift": null,
"title": "Containers Info",
"transformations": [
{
"id": "filterFieldsByName",
"options": {
"include": {
"names": [
"container_label_com_docker_compose_project",
"container_label_com_docker_compose_project_working_dir",
"image",
"instance",
"name",
"Value",
"container_label_com_docker_compose_service"
]
}
}
},
{
"id": "organize",
"options": {
"excludeByName": {},
"indexByName": {},
"renameByName": {
"Value": "Running",
"container_label_com_docker_compose_project": "Label",
"container_label_com_docker_compose_project_working_dir": "Working dir",
"container_label_com_docker_compose_service": "Service",
"image": "Registry Image",
"instance": "Instance",
"name": "Name"
}
}
}
],
"type": "table"
}
],
"schemaVersion": 27,
"style": "dark",
"tags": [
"cadvisor",
"docker"
],
"templating": {
"list": [
{
"allValue": ".*",
"current": {},
"datasource": "${DS_PROMETHEUS}",
"definition": "label_values({__name__=~\"container.*\"},instance)",
"description": null,
"error": null,
"hide": 0,
"includeAll": true,
"label": "Host",
"multi": false,
"name": "host",
"options": [],
"query": {
"query": "label_values({__name__=~\"container.*\"},instance)",
"refId": "Prometheus-host-Variable-Query"
},
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 5,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"allValue": ".*",
"current": {},
"datasource": "${DS_PROMETHEUS}",
"definition": "label_values({__name__=~\"container.*\", instance=~\"$host\"},name)",
"description": null,
"error": null,
"hide": 0,
"includeAll": true,
"label": "Container",
"multi": false,
"name": "container",
"options": [],
"query": {
"query": "label_values({__name__=~\"container.*\", instance=~\"$host\"},name)",
"refId": "Prometheus-container-Variable-Query"
},
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 0,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
}
]
},
"time": {
"from": "now-6h",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "Cadvisor exporter",
"uid": "pMEd7m0Mz",
"version": 1,
"description": "Simple exporter for cadvisor only"
}
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
@@ -19,6 +19,9 @@
- { path: "{{ monitoring_dir }}/grafana/provisioning/dashboards", owner: "root", group: "root", mode: "0755" }
- { path: "{{ monitoring_dir }}/ntfy_data", owner: "root", group: "root", mode: "0777" }
- { path: "{{ monitoring_dir }}/alertmanager-ntfy", owner: "root", group: "root", mode: "0755" }
- { path: "{{ monitoring_dir }}/victoria-metrics-data", owner: "65534", group: "65534", mode: "0775" }
- { path: "{{ monitoring_dir }}/vmagent", owner: "root", group: "root", mode: "0755" }
- { path: "{{ monitoring_dir }}/vmagent/conf.d", owner: "root", group: "root", mode: "0755" }
- name: Сбросить список целей scrape node-exporter
ansible.builtin.set_fact:
@@ -82,13 +85,14 @@
group: root
mode: "0644"
- name: Установить источник данных Prometheus в Grafana
- name: Установить источники данных Grafana
ansible.builtin.template:
src: grafana-datasources.yml.j2
dest: "{{ monitoring_dir }}/grafana/provisioning/datasources/prometheus.yml"
owner: root
group: root
mode: "0644"
register: grafana_datasources
- name: Установить провайдер дашбордов Grafana
ansible.builtin.template:
@@ -106,6 +110,51 @@
group: root
mode: "0644"
- name: Конфиг vmagent на core-mon
ansible.builtin.template:
src: "{{ item.src }}"
dest: "{{ item.dest }}"
owner: root
group: root
mode: "0644"
loop:
- { src: vmagent-scrape.yml.j2, dest: "{{ monitoring_dir }}/vmagent/scrape.yml" }
- { src: vmagent-nodeexporter.yml.j2, dest: "{{ monitoring_dir }}/vmagent/conf.d/nodeexporter.yml" }
- { src: vmagent-cadvisor.yml.j2, dest: "{{ monitoring_dir }}/vmagent/conf.d/cadvisor.yml" }
- name: Убрать scrape Xray Checker без ссылки подписки
ansible.builtin.file:
path: "{{ item }}"
state: absent
loop:
- "{{ monitoring_dir }}/vmagent/conf.d/xraychecker.yml"
- "{{ monitoring_dir }}/xray-checker.env"
when: xray_checker_subscription_url | length == 0
- name: Ссылки подписки для Xray Checker нет
ansible.builtin.debug:
msg: "Ссылки Xray Checker нет, контейнер не запускается."
when: xray_checker_subscription_url | length == 0
- name: Файл окружения Xray Checker
ansible.builtin.template:
src: xray-checker.env.j2
dest: "{{ monitoring_dir }}/xray-checker.env"
owner: root
group: root
mode: "0600"
no_log: true
when: xray_checker_subscription_url | length > 0
- name: Scrape Xray Checker
ansible.builtin.template:
src: vmagent-xray.yml.j2
dest: "{{ monitoring_dir }}/vmagent/conf.d/xraychecker.yml"
owner: root
group: root
mode: "0644"
when: xray_checker_subscription_url | length > 0
- name: Установить compose.yaml для стека мониторинга
ansible.builtin.template:
src: compose.yaml.j2
@@ -151,6 +200,20 @@
failed_when: false
no_log: true
- name: Дождаться VictoriaMetrics
ansible.builtin.uri:
url: "http://{{ ansible_host }}:8428/health"
status_code: 200
register: victoriametrics_health
retries: 20
delay: 2
until: victoriametrics_health.status == 200
- name: Перезапустить Grafana, чтобы подхватить datasource
ansible.builtin.command: docker compose -f {{ monitoring_dir }}/compose.yaml restart grafana
when: grafana_datasources is changed
changed_when: true
- name: Дождаться готовности Grafana
ansible.builtin.uri:
url: "http://127.0.0.1:3000/api/health"
@@ -78,4 +78,64 @@ services:
depends_on:
- prometheus
vmsingle:
image: {{ victoriametrics_image }}
container_name: monitoring-vmsingle
hostname: vmsingle
restart: unless-stopped
command:
- --storageDataPath=/victoria-metrics-data
volumes:
- ./victoria-metrics-data:/victoria-metrics-data
ports:
- "{{ ansible_host }}:8428:8428"
cadvisor:
image: {{ cadvisor_image }}
container_name: monitoring-cadvisor
restart: unless-stopped
privileged: true
command:
- --listen_ip=0.0.0.0
- --port=9101
- --docker_only=true
volumes:
- /:/rootfs:ro
- /var/run:/var/run:ro
- /sys:/sys:ro
- /var/lib/docker:/var/lib/docker:ro
- /dev/disk:/dev/disk:ro
ports:
- "127.0.0.1:9101:9101"
{% if xray_checker_subscription_url | length > 0 %}
xray-checker:
image: {{ xray_checker_image }}
container_name: monitoring-xray-checker
hostname: xraychecker
restart: unless-stopped
env_file:
- ./xray-checker.env
ports:
- "127.0.0.1:2112:2112"
{% endif %}
vmagent:
image: {{ vmagent_image }}
container_name: monitoring-vmagent
restart: unless-stopped
command:
- -httpListenAddr=0.0.0.0:8429
- -promscrape.config=/etc/vmagent/scrape.yml
- -promscrape.configCheckInterval=60s
- -remoteWrite.url=http://vmsingle:8428/api/v1/write
volumes:
- ./vmagent/scrape.yml:/etc/vmagent/scrape.yml:ro
- ./vmagent/conf.d:/etc/vmagent/conf.d:ro
ports:
- "127.0.0.1:8429:8429"
depends_on:
- vmsingle
- cadvisor
@@ -10,3 +10,12 @@ datasources:
jsonData:
timeInterval: 15s
- name: VictoriaMetrics
uid: victoriametrics
type: prometheus
access: proxy
url: http://vmsingle:8428
isDefault: false
jsonData:
timeInterval: 15s
@@ -0,0 +1,7 @@
- job_name: integrations/cAdvisor
scrape_interval: 15s
static_configs:
- targets: ["cadvisor:9101"]
labels:
instance: "{{ inventory_hostname }}"
cluster: "{{ vmagent_cluster }}"
@@ -0,0 +1,7 @@
- job_name: integrations/node_exporter
scrape_interval: 15s
static_configs:
- targets: ["{{ ansible_host }}:9100"]
labels:
instance: "{{ inventory_hostname }}"
cluster: "{{ vmagent_cluster }}"
@@ -0,0 +1,5 @@
scrape_config_files:
- "/etc/vmagent/conf.d/*.yml"
global:
scrape_interval: 15s
@@ -0,0 +1,7 @@
- job_name: xray_checker
scrape_interval: 15s
static_configs:
- targets: ["xray-checker:2112"]
labels:
instance: "{{ inventory_hostname }}"
cluster: "{{ vmagent_cluster }}"
@@ -0,0 +1,3 @@
SUBSCRIPTION_URL={{ xray_checker_subscription_url }}
PROXY_STATUS_CHECK_URL=http://google.com/generate_204
PROXY_CHECK_INTERVAL=60