From 72a9c86e52f446db352b5226103909a972f9dfc1 Mon Sep 17 00:00:00 2001 From: karpenok Date: Thu, 12 Nov 2026 16:50:44 +0700 Subject: [PATCH] =?UTF-8?q?feat:=20VictoriaMetrics=20=D0=B8=20NetBird=20?= =?UTF-8?q?=D0=BC=D0=B5=D0=B6=D0=B4=D1=83=20=D0=BF=D0=B0=D0=BD=D0=B5=D0=BB?= =?UTF-8?q?=D1=8C=D1=8E=20=D0=B8=20ger-nod-02?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Prometheus остаётся для LAN. Нода отдаёт метрики через vmagent, а панель ходит на неё по отдельному ключу NetBird, не через группу роутеров. Co-authored-by: Cursor --- .gitea/workflows/ansible-core-net.yml | 2 + .gitea/workflows/ansible-ger-nod-02.yml | 2 + BOOTSTRAP.md | 17 +- ansible/ci/ansible-target.sh | 10 + ansible/ci/deploy-machine.sh | 9 + .../inventories/prod/host_vars/ger-nod-02.yml | 3 + ansible/inventories/prod/hosts.yml | 3 +- ansible/playbooks/core-net.yml | 4 + ansible/playbooks/ger-nod-02.yml | 4 + .../core-mon/monitoring/defaults/main.yml | 7 + .../files/dashboards/vm-cadvisor.json | 817 +++++ .../files/dashboards/vm-cpu-system.json | 1132 +++++++ .../files/dashboards/vm-filesystem-disks.json | 1419 +++++++++ .../files/dashboards/vm-memory.json | 2199 ++++++++++++++ .../files/dashboards/vm-network.json | 2634 +++++++++++++++++ .../files/dashboards/vm-node-overview.json | 2077 +++++++++++++ .../dashboards/vm-remnawave-dashboard.json | 2479 ++++++++++++++++ .../roles/core-mon/monitoring/tasks/main.yml | 65 +- .../monitoring/templates/compose.yaml.j2 | 60 + .../templates/grafana-datasources.yml.j2 | 9 + .../templates/vmagent-cadvisor.yml.j2 | 7 + .../templates/vmagent-nodeexporter.yml.j2 | 7 + .../templates/vmagent-scrape.yml.j2 | 5 + .../monitoring/templates/vmagent-xray.yml.j2 | 7 + .../monitoring/templates/xray-checker.env.j2 | 3 + ansible/roles/core-net/netbird/tasks/main.yml | 4 + .../core-net/remnawave/defaults/main.yml | 3 + .../roles/core-net/remnawave/tasks/main.yml | 48 +- .../remnawave/templates/bootstrap-panel.sh.j2 | 52 +- ansible/roles/core-net/vmagent/tasks/main.yml | 6 + .../roles/ger-nod-02/netbird/tasks/main.yml | 4 + .../roles/ger-nod-02/remnanode/tasks/main.yml | 46 +- .../roles/ger-nod-02/vmagent/tasks/main.yml | 7 + ansible/roles/host/defaults/main.yml | 3 + ansible/roles/host/tasks/main.yml | 27 +- .../roles/netbird_client/defaults/main.yml | 4 + ansible/roles/netbird_client/tasks/main.yml | 86 + .../rus-central/netbird/defaults/main.yml | 3 + .../roles/rus-central/netbird/tasks/main.yml | 63 +- ansible/roles/vmagent/defaults/main.yml | 16 + ansible/roles/vmagent/tasks/main.yml | 106 + .../roles/vmagent/templates/cadvisor.yml.j2 | 7 + .../roles/vmagent/templates/compose.yaml.j2 | 33 + .../vmagent/templates/nodeexporter.yml.j2 | 7 + .../roles/vmagent/templates/remnawave.yml.j2 | 10 + ansible/roles/vmagent/templates/scrape.yml.j2 | 5 + 46 files changed, 13496 insertions(+), 25 deletions(-) create mode 100644 ansible/roles/core-mon/monitoring/files/dashboards/vm-cadvisor.json create mode 100644 ansible/roles/core-mon/monitoring/files/dashboards/vm-cpu-system.json create mode 100644 ansible/roles/core-mon/monitoring/files/dashboards/vm-filesystem-disks.json create mode 100644 ansible/roles/core-mon/monitoring/files/dashboards/vm-memory.json create mode 100644 ansible/roles/core-mon/monitoring/files/dashboards/vm-network.json create mode 100644 ansible/roles/core-mon/monitoring/files/dashboards/vm-node-overview.json create mode 100644 ansible/roles/core-mon/monitoring/files/dashboards/vm-remnawave-dashboard.json create mode 100644 ansible/roles/core-mon/monitoring/templates/vmagent-cadvisor.yml.j2 create mode 100644 ansible/roles/core-mon/monitoring/templates/vmagent-nodeexporter.yml.j2 create mode 100644 ansible/roles/core-mon/monitoring/templates/vmagent-scrape.yml.j2 create mode 100644 ansible/roles/core-mon/monitoring/templates/vmagent-xray.yml.j2 create mode 100644 ansible/roles/core-mon/monitoring/templates/xray-checker.env.j2 create mode 100644 ansible/roles/core-net/netbird/tasks/main.yml create mode 100644 ansible/roles/core-net/vmagent/tasks/main.yml create mode 100644 ansible/roles/ger-nod-02/netbird/tasks/main.yml create mode 100644 ansible/roles/ger-nod-02/vmagent/tasks/main.yml create mode 100644 ansible/roles/netbird_client/defaults/main.yml create mode 100644 ansible/roles/netbird_client/tasks/main.yml create mode 100644 ansible/roles/vmagent/defaults/main.yml create mode 100644 ansible/roles/vmagent/tasks/main.yml create mode 100644 ansible/roles/vmagent/templates/cadvisor.yml.j2 create mode 100644 ansible/roles/vmagent/templates/compose.yaml.j2 create mode 100644 ansible/roles/vmagent/templates/nodeexporter.yml.j2 create mode 100644 ansible/roles/vmagent/templates/remnawave.yml.j2 create mode 100644 ansible/roles/vmagent/templates/scrape.yml.j2 diff --git a/.gitea/workflows/ansible-core-net.yml b/.gitea/workflows/ansible-core-net.yml index 5186ea0..98f39bd 100644 --- a/.gitea/workflows/ansible-core-net.yml +++ b/.gitea/workflows/ansible-core-net.yml @@ -8,6 +8,8 @@ on: - 'ansible/ansible.cfg' - 'ansible/ci/**' - 'ansible/roles/core-net/**' + - 'ansible/roles/netbird_client/**' + - 'ansible/roles/vmagent/**' - 'ansible/roles/host/**' - 'ansible/playbooks/core-net.yml' - 'ansible/playbooks/host.yml' diff --git a/.gitea/workflows/ansible-ger-nod-02.yml b/.gitea/workflows/ansible-ger-nod-02.yml index 15c24b4..5ebf145 100644 --- a/.gitea/workflows/ansible-ger-nod-02.yml +++ b/.gitea/workflows/ansible-ger-nod-02.yml @@ -8,6 +8,8 @@ on: - 'ansible/ansible.cfg' - 'ansible/ci/**' - 'ansible/roles/ger-nod-02/**' + - 'ansible/roles/netbird_client/**' + - 'ansible/roles/vmagent/**' - 'ansible/roles/host/**' - 'ansible/playbooks/ger-nod-02.yml' - 'ansible/playbooks/host.yml' diff --git a/BOOTSTRAP.md b/BOOTSTRAP.md index 3713d5f..1d6cec4 100644 --- a/BOOTSTRAP.md +++ b/BOOTSTRAP.md @@ -6,11 +6,11 @@ | Машина | Плейбук | Роли | Каталоги | |---|---|---|---| -| `core-net` | `playbooks/core-net.yml` | `roles/core-net/traefik`; `roles/core-net/remnawave`; `roles/core-net/authentik` *(опц., по умолчанию выкл.)* | `/opt/traefik`; `/opt/remnawave`; `/opt/authentik` *(если включён)* | +| `core-net` | `playbooks/core-net.yml` | `roles/core-net/traefik`; `roles/core-net/netbird`; `roles/core-net/remnawave`; `roles/core-net/vmagent`; `roles/core-net/authentik` *(опц., по умолчанию выкл.)* | `/opt/traefik`; клиент NetBird; `/opt/remnawave`; `/opt/vmagent`; `/opt/authentik` *(если включён)* | | `core-secrets` | `playbooks/core-secrets.yml` | `roles/core-secrets/infisical`, `roles/core-secrets/vaultwarden` | `/opt/infisical`, `/opt/vaultwarden` | -| `core-mon` | `playbooks/core-mon.yml` | `roles/core-mon/monitoring` | `/opt/monitoring` | +| `core-mon` | `playbooks/core-mon.yml` | `roles/core-mon/monitoring` | `/opt/monitoring` (Prometheus и VictoriaMetrics) | | `rus-central` | `playbooks/rus-central.yml` | `roles/rus-central/netbird` | `/opt/netbird` | -| `ger-nod-02` | `playbooks/ger-nod-02.yml` | `roles/ger-nod-02/remnanode` | `/opt/remnanode` | +| `ger-nod-02` | `playbooks/ger-nod-02.yml` | `roles/ger-nod-02/netbird`; `roles/ger-nod-02/remnanode`; `roles/ger-nod-02/vmagent` | клиент NetBird; `/opt/remnanode`; `/opt/vmagent` | | `opnsense` | `playbooks/opnsense.yml` | `roles/opnsense/netbird` | REST API (`192.168.1.1`) | Полная сверка одной машины: `ansible-playbook playbooks/site.yml --limit <машина>`. Gitea в автоматизацию не входит: без неё не запустить пайплайны. @@ -34,7 +34,7 @@ Terraform в этом репозитории описывает только у | **CT 109** | `pve` | `192.168.1.217` | `gitea` | Gitea (`:3000`), Postgres (`:5432`), Gitea Actions Runner (`dind`) | | **CT 901** | `netcore` | `192.168.1.102` | `core-secrets` | Infisical (`:8080`), Postgres, Redis, Vaultwarden (`:8000`) | | **CT 902** | `netcore` | `192.168.1.104` | `core-net` | Traefik (`:80`, `:443`); Remnawave Panel (`127.0.0.1:3000`) + Subscription (`127.0.0.1:3010`); Authentik (`127.0.0.1:9000`) — опционально, по умолчанию выкл. | -| **CT 903** | `netcore` | `192.168.1.108` | `core-mon` | Prometheus (`:9090`), Alertmanager (`:9093`), Grafana (`:3000`), Ntfy (`:80`) | +| **CT 903** | `netcore` | `192.168.1.108` | `core-mon` | Prometheus (`:9090`), VictoriaMetrics (`:8428`), Alertmanager (`:9093`), Grafana (`:3000`), Ntfy (`:80`) | ### 1.3. Внешние узлы | Имя | IP | Роль | Стек | @@ -80,7 +80,8 @@ API Endpoint (внутренний LAN): `http://192.168.1.102:8080/api` - `NETBIRD_DATASTORE_KEY`: 32-байтный ключ SQLite/datastore. - `NETBIRD_SESSION_KEY`: 32-байтный ключ веб-сессий. - `NETBIRD_API_TOKEN`: сервисный API-токен управления (создаётся и сохраняется пайплайном автоматически). - - `NETBIRD_SETUP_KEY`: многоразовый ключ регистрации узлов (создаётся и сохраняется пайплайном автоматически). + - `NETBIRD_SETUP_KEY`: многоразовый ключ регистрации роутеров (`homelab-routers`). Панель и ноды им не пользуются. + - `NETBIRD_REMNAWAVE_SETUP_KEY`: многоразовый ключ `remnawave-peers` для core-net и ger-nod-02. В группу роутеров не входит. - *(Параметры `NETBIRD_ADMIN_NAME`, `NETBIRD_ADMIN_PASSWORD`, `NETBIRD_ADMIN_EMAIL` наследуются из `/USER/ADMIN`)*. - `/services/opnsense`: - `OPNSENSE_API_KEY`: API-ключ администратора OPNsense. @@ -90,6 +91,7 @@ API Endpoint (внутренний LAN): `http://192.168.1.102:8080/api` - `REMNAWAVE_APP_SECRET`, `REMNAWAVE_POSTGRES_PASSWORD`, `REMNAWAVE_METRICS_PASS`, `REMNAWAVE_WEBHOOK_SECRET_HEADER`: генерируются пайплайном при первом прогоне, если пусто. - `REMNAWAVE_API_TOKEN`: сервисный API-токен панели (`infra-automation`, scopes `*`), создаётся и сохраняется пайплайном (при сбое — создать в UI и положить сюда вручную). - `REMNANODE_SECRET_KEY`: SECRET_KEY ноды Ger-nod-02 (`GET /api/keygen/`), сохраняется пайплайном. + - `XRAY_CHECKER_SUBSCRIPTION_URL`: ссылка подписки пользователя `homelab` для Xray Checker на core-mon. В лог не пишется. - `/proxmox/netcore`: - `PROXMOX_ENDPOINT`: `https://192.168.1.100:8006/` - `PROXMOX_API_TOKEN`: токен `root@pam!terraform=UUID` @@ -151,7 +153,7 @@ API Endpoint (внутренний LAN): `http://192.168.1.102:8080/api` - `/ci-cd` → `CLOUDFLARE_API` - `/VPS` → `PASS_GER_NOD_02` (пароль нового VPS) 2. Запустить **`ansible-core-net`** (полный прогон или `--tags remnawave` после того, как Traefik уже с роутами `rw`/`sub`). -3. Роль поднимает `/opt/remnawave` (panel+db+redis) и `/opt/remnawave/subscription`, создаёт `REMNAWAVE_ADMIN_*`, каждый прогон **перевыпускает** API-токен `infra-automation` и кладёт его в Infisical (старый сразу недействителен). Config profile **`DE-Yali`**, сквад `Default-Squad` на inbound Steal-Yali, нода **`Ger-nod-02`**, host **`yali.gunis.top`**. Пользователи — файл `ansible/roles/core-net/remnawave/files/users.yml` (добавление/изменение/удаление = правка файла). +3. Роль поднимает `/opt/remnawave` (panel+db+redis) и `/opt/remnawave/subscription`, создаёт `REMNAWAVE_ADMIN_*`, каждый прогон **перевыпускает** API-токен `infra-automation` и кладёт его в Infisical (старый сразу недействителен). Config profile **`DE-Yali`**, сквад `Default-Squad` на inbound Steal-Yali, нода **`Ger-nod-02`**, host **`yali.gunis.top`**. Пользователи — файл `ansible/roles/core-net/remnawave/files/users.yml` (добавление/изменение/удаление = правка файла). Когда peer `ger-nod-02` уже есть в NetBird, адрес ноды в панели меняется на его NetBird IP. Пока peer нет, остаётся `150.241.64.209`. Ссылка подписки `homelab` пишется в `XRAY_CHECKER_SUBSCRIPTION_URL` и в лог не попадает. 4. Проверить `https://rw.gunihome.ru` и `https://sub.gunihome.ru`. > [!IMPORTANT] @@ -161,8 +163,9 @@ API Endpoint (внутренний LAN): `http://192.168.1.102:8080/api` 1. После успешного `ansible-core-net` (чтобы в Infisical был `REMNANODE_SECRET_KEY`) запустить **`ansible-ger-nod-02`**. 2. Первый прогон: как у rus-central — `ansible_password` из `/VPS` → `PASS_GER_NOD_02`; роль `host` кладёт ключ и отключает пароль. Роль `remnanode` держит A-запись `yali.gunis.top` в Cloudflare, выпускает LE по **HTTP-01** (порт 80, без TXT `_acme-challenge` в зоне), поднимает remnanode + nginx SelfSteal. 3. В панели нода должна стать зелёной; браузер на `https://yali.gunis.top` — маскировочный сайт. +4. Клиент NetBird ходит на `https://central.gunihome.ru` ключом `remnawave-peers`. Пока peer панели не виден, `NODE_PORT` 2222 остаётся открыт для WAN egress панели. Когда peer `core-net` есть, 2222 разрешён только с его NetBird IP. Публичный `:9100` закрыт: метрики ноды уходят vmagent-ом в VictoriaMetrics на `192.168.1.108:8428` через маршрут `homelab-lan`. В Prometheus эта нода больше не входит. ### Шаг 5. NetBird (rus-central) и связка с OPNsense 1. Запустить пайплайн **`ansible-rus-central`** в Gitea Actions. Ручной запуск сверяет машину целиком. Push в `roles/rus-central/netbird` ставит только NetBird. -2. Раннер считывает ключи из `/services/netbird`, деплоит стек в `/opt/netbird` на `45.154.185.208`, автоматически регистрирует первого суперпользователя (`POST /api/setup`), создаёт сервисный токен (`NETBIRD_API_TOKEN`), группу `homelab-routers`, маршрут к подсети `192.168.1.0/24` и многоразовый ключ установки (`NETBIRD_SETUP_KEY`), после чего сохраняет `NETBIRD_API_TOKEN` и `NETBIRD_SETUP_KEY` в Infisical. +2. Раннер считывает ключи из `/services/netbird`, деплоит стек в `/opt/netbird` на `45.154.185.208`, автоматически регистрирует первого суперпользователя (`POST /api/setup`), создаёт сервисный токен (`NETBIRD_API_TOKEN`), группу `homelab-routers`, маршрут к подсети `192.168.1.0/24` и многоразовый ключ установки (`NETBIRD_SETUP_KEY`). Отдельно создаётся группа `remnawave` и ключ `remnawave-peers` (`NETBIRD_REMNAWAVE_SETUP_KEY`): он не добавляет peer в `homelab-routers`. Оба ключа сохраняются в Infisical. 3. Запустить пайплайн **`ansible-opnsense`** в Gitea Actions. Раннер обращается по HTTPS REST API к `192.168.1.1`, проверяет/устанавливает плагин `os-netbird`, включает службу и регистрирует шлюз OPNsense на `https://central.gunihome.ru:443` по `NETBIRD_SETUP_KEY`. diff --git a/ansible/ci/ansible-target.sh b/ansible/ci/ansible-target.sh index 015ff8d..3418581 100755 --- a/ansible/ci/ansible-target.sh +++ b/ansible/ci/ansible-target.sh @@ -64,6 +64,16 @@ if [ "$full" -eq 0 ] && [ -n "$changed" ]; then ansible/ansible.cfg | ansible/ci/* | ansible/roles/host/* | ansible/playbooks/host.yml | ansible/playbooks/site.yml | ansible/playbooks/"$machine".yml | ansible/inventories/prod/hosts.yml | ansible/inventories/prod/group_vars/* | ansible/inventories/prod/host_vars/"$machine".yml) full_from_files=1 ;; + ansible/roles/netbird_client/*) + case "$machine" in + core-net | ger-nod-02) add_tag netbird ;; + esac + ;; + ansible/roles/vmagent/*) + case "$machine" in + core-net | ger-nod-02) add_tag vmagent ;; + esac + ;; ansible/roles/"$machine"/*/*) product=${file#ansible/roles/"$machine"/} product=${product%%/*} diff --git a/ansible/ci/deploy-machine.sh b/ansible/ci/deploy-machine.sh index 927bbef..5659eda 100755 --- a/ansible/ci/deploy-machine.sh +++ b/ansible/ci/deploy-machine.sh @@ -64,6 +64,9 @@ case "$machine" in export REMNAWAVE_WEBHOOK_SECRET_HEADER="${REMNAWAVE_WEBHOOK_SECRET_HEADER:-}" export REMNAWAVE_API_TOKEN="${REMNAWAVE_API_TOKEN:-}" export REMNANODE_SECRET_KEY="${REMNANODE_SECRET_KEY:-}" + eval "$(infisical export --domain="${INFISICAL_DOMAIN}" --token="${TOKEN}" --env=prod --projectId="${INFISICAL_PROJECT_ID}" --path="/services/netbird" --format=dotenv-export --silent)" + export NETBIRD_API_TOKEN="${NETBIRD_API_TOKEN:-}" + export NETBIRD_REMNAWAVE_SETUP_KEY="${NETBIRD_REMNAWAVE_SETUP_KEY:-}" ;; core-secrets) eval "$(infisical export --domain="${INFISICAL_DOMAIN}" --token="${TOKEN}" --env=prod --projectId="${INFISICAL_PROJECT_ID}" --path="/services/infisical" --format=dotenv-export --silent)" @@ -77,6 +80,8 @@ case "$machine" in if [ -n "${MAIL_ADMIN_USER:-}" ]; then extra+=(-e "grafana_admin_email=${MAIL_ADMIN_USER}") fi + eval "$(infisical export --domain="${INFISICAL_DOMAIN}" --token="${TOKEN}" --env=prod --projectId="${INFISICAL_PROJECT_ID}" --path="/services/remnawave" --format=dotenv-export --silent 2>/dev/null || true)" + export XRAY_CHECKER_SUBSCRIPTION_URL="${XRAY_CHECKER_SUBSCRIPTION_URL:-}" ;; rus-central) input_email="${INPUT_LETSENCRYPT_EMAIL:-}" @@ -89,6 +94,7 @@ case "$machine" in export NETBIRD_ADMIN_PASSWORD="${NETBIRD_ADMIN_PASSWORD:-${ADMIN_PASSWORD:-}}" export NETBIRD_API_TOKEN="${NETBIRD_API_TOKEN:-}" export NETBIRD_SETUP_KEY="${NETBIRD_SETUP_KEY:-}" + export NETBIRD_REMNAWAVE_SETUP_KEY="${NETBIRD_REMNAWAVE_SETUP_KEY:-}" if [ -n "$input_email" ]; then NETBIRD_LETSENCRYPT_EMAIL="$input_email" fi @@ -110,6 +116,9 @@ case "$machine" in eval "$(infisical export --domain="${INFISICAL_DOMAIN}" --token="${TOKEN}" --env=prod --projectId="${INFISICAL_PROJECT_ID}" --path="/VPS" --format=dotenv-export --silent)" # CLOUDFLARE_API уже из /ci-cd выше (только A-запись, без DNS-01). export REMNANODE_SECRET_KEY="${REMNANODE_SECRET_KEY:-}" + eval "$(infisical export --domain="${INFISICAL_DOMAIN}" --token="${TOKEN}" --env=prod --projectId="${INFISICAL_PROJECT_ID}" --path="/services/netbird" --format=dotenv-export --silent 2>/dev/null || true)" + export NETBIRD_API_TOKEN="${NETBIRD_API_TOKEN:-}" + export NETBIRD_REMNAWAVE_SETUP_KEY="${NETBIRD_REMNAWAVE_SETUP_KEY:-}" export CLOUDFLARE_API="${CLOUDFLARE_API:-}" export PASS_GER_NOD_02="${PASS_GER_NOD_02:-}" if [ -z "${REMNANODE_SECRET_KEY}" ]; then diff --git a/ansible/inventories/prod/host_vars/ger-nod-02.yml b/ansible/inventories/prod/host_vars/ger-nod-02.yml index 3ecc38b..331f1e0 100644 --- a/ansible/inventories/prod/host_vars/ger-nod-02.yml +++ b/ansible/inventories/prod/host_vars/ger-nod-02.yml @@ -6,3 +6,6 @@ remnanode_selfsteal_domain: "yali.gunis.top" remnanode_panel_egress_ips: - "213.128.28.107" # провайдер 1 (МТС) - "185.160.137.140" # провайдер 2 +monitoring_scrape_enabled: false +monitoring_node_exporter_public: false +monitoring_node_exporter_listen: "127.0.0.1:9100" diff --git a/ansible/inventories/prod/hosts.yml b/ansible/inventories/prod/hosts.yml index 188c336..2baf51f 100644 --- a/ansible/inventories/prod/hosts.yml +++ b/ansible/inventories/prod/hosts.yml @@ -1,5 +1,6 @@ # Машины. Состав каждой машины — в playbooks/<хост>.yml. -# Scrape Prometheus (node-exporter :9100): все записи ниже, кроме monitoring_scrape_exclude_hosts в group_vars/роли monitoring. +# Scrape Prometheus (node-exporter :9100): все записи ниже, кроме monitoring_scrape_exclude_hosts +# и хостов с monitoring_scrape_enabled: false (ger-nod-02 отдаёт метрики через vmagent). all: hosts: core-net: diff --git a/ansible/playbooks/core-net.yml b/ansible/playbooks/core-net.yml index 93043cf..87ea6b7 100644 --- a/ansible/playbooks/core-net.yml +++ b/ansible/playbooks/core-net.yml @@ -5,7 +5,11 @@ roles: - role: core-net/traefik tags: [traefik] + - role: core-net/netbird + tags: [netbird] - role: core-net/remnawave tags: [remnawave] - role: core-net/authentik tags: [authentik] + - role: core-net/vmagent + tags: [vmagent] diff --git a/ansible/playbooks/ger-nod-02.yml b/ansible/playbooks/ger-nod-02.yml index 82594ca..26934c0 100644 --- a/ansible/playbooks/ger-nod-02.yml +++ b/ansible/playbooks/ger-nod-02.yml @@ -3,5 +3,9 @@ hosts: ger-nod-02 gather_facts: false roles: + - role: ger-nod-02/netbird + tags: [netbird] - role: ger-nod-02/remnanode tags: [remnanode] + - role: ger-nod-02/vmagent + tags: [vmagent] diff --git a/ansible/roles/core-mon/monitoring/defaults/main.yml b/ansible/roles/core-mon/monitoring/defaults/main.yml index e51a7ba..cdcce80 100644 --- a/ansible/roles/core-mon/monitoring/defaults/main.yml +++ b/ansible/roles/core-mon/monitoring/defaults/main.yml @@ -11,6 +11,13 @@ monitoring_node_exporter_port: 9100 monitoring_scrape_job_name: node-exporter monitoring_env_label: prod +victoriametrics_image: victoriametrics/victoria-metrics:v1.153.0 +vmagent_image: victoriametrics/vmagent:v1.153.0 +cadvisor_image: gcr.io/cadvisor/cadvisor:v0.49.2 +xray_checker_image: kutovoys/xray-checker:v1.3.1 +xray_checker_subscription_url: "{{ lookup('env', 'XRAY_CHECKER_SUBSCRIPTION_URL', default='') }}" +vmagent_cluster: homelab + grafana_admin_user: "{{ lookup('ansible.builtin.env', 'ADMIN_USER', default='karpenok') }}" grafana_admin_password: "{{ lookup('ansible.builtin.env', 'ADMIN_PASSWORD', default='') }}" grafana_admin_email: "{{ lookup('ansible.builtin.env', 'MAIL_ADMIN_USER', default=grafana_admin_user ~ '@gunihome.ru') }}" diff --git a/ansible/roles/core-mon/monitoring/files/dashboards/vm-cadvisor.json b/ansible/roles/core-mon/monitoring/files/dashboards/vm-cadvisor.json new file mode 100644 index 0000000..62acd9a --- /dev/null +++ b/ansible/roles/core-mon/monitoring/files/dashboards/vm-cadvisor.json @@ -0,0 +1,817 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "Prometheus as the datasource is obligatory", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "7.4.5" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": 14282, + "graphTooltip": 0, + "id": null, + "iteration": 1617715580880, + "links": [], + "panels": [ + { + "collapsed": false, + "datasource": "${DS_PROMETHEUS}", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 8, + "panels": [], + "title": "CPU", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 1 + }, + "hiddenSeries": false, + "id": 15, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_cpu_usage_seconds_total{instance=~\"$host\",name=~\"$container\",name=~\".+\"}[5m])) by (name) *100", + "hide": false, + "interval": "", + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:606", + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:607", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": "${DS_PROMETHEUS}", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 11, + "panels": [], + "title": "Memory", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "hiddenSeries": false, + "id": 9, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_rss{instance=~\"$host\",name=~\"$container\",name=~\".+\"}) by (name)", + "hide": false, + "interval": "", + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:606", + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:607", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_cache{instance=~\"$host\",name=~\"$container\",name=~\".+\"}) by (name)", + "hide": false, + "interval": "", + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Cached", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:606", + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:607", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": "${DS_PROMETHEUS}", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 2, + "panels": [], + "title": "Network", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 18 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_network_receive_bytes_total{instance=~\"$host\",name=~\"$container\",name=~\".+\"}[5m])) by (name)", + "hide": false, + "interval": "", + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Received Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:674", + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:675", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 18 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_network_transmit_bytes_total{instance=~\"$host\",name=~\"$container\",name=~\".+\"}[5m])) by (name)", + "interval": "", + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Sent Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:832", + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:833", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": "${DS_PROMETHEUS}", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 19, + "panels": [], + "title": "Misc", + "type": "row" + }, + { + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "custom": { + "align": null, + "filterable": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "id" + }, + "properties": [ + { + "id": "custom.width", + "value": 260 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Running" + }, + "properties": [ + { + "id": "unit", + "value": "d" + }, + { + "id": "decimals", + "value": 1 + }, + { + "id": "custom.displayMode", + "value": "color-text" + }, + { + "id": "color", + "value": { + "fixedColor": "dark-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 17, + "options": { + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "7.4.5", + "targets": [ + { + "expr": "(time() - container_start_time_seconds{instance=~\"$host\",name=~\"$container\",name=~\".+\"})/86400", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Containers Info", + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "container_label_com_docker_compose_project", + "container_label_com_docker_compose_project_working_dir", + "image", + "instance", + "name", + "Value", + "container_label_com_docker_compose_service" + ] + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": {}, + "renameByName": { + "Value": "Running", + "container_label_com_docker_compose_project": "Label", + "container_label_com_docker_compose_project_working_dir": "Working dir", + "container_label_com_docker_compose_service": "Service", + "image": "Registry Image", + "instance": "Instance", + "name": "Name" + } + } + } + ], + "type": "table" + } + ], + "schemaVersion": 27, + "style": "dark", + "tags": [ + "cadvisor", + "docker" + ], + "templating": { + "list": [ + { + "allValue": ".*", + "current": {}, + "datasource": "${DS_PROMETHEUS}", + "definition": "label_values({__name__=~\"container.*\"},instance)", + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": "Host", + "multi": false, + "name": "host", + "options": [], + "query": { + "query": "label_values({__name__=~\"container.*\"},instance)", + "refId": "Prometheus-host-Variable-Query" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": {}, + "datasource": "${DS_PROMETHEUS}", + "definition": "label_values({__name__=~\"container.*\", instance=~\"$host\"},name)", + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": "Container", + "multi": false, + "name": "container", + "options": [], + "query": { + "query": "label_values({__name__=~\"container.*\", instance=~\"$host\"},name)", + "refId": "Prometheus-container-Variable-Query" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Cadvisor exporter", + "uid": "pMEd7m0Mz", + "version": 1, + "description": "Simple exporter for cadvisor only" +} diff --git a/ansible/roles/core-mon/monitoring/files/dashboards/vm-cpu-system.json b/ansible/roles/core-mon/monitoring/files/dashboards/vm-cpu-system.json new file mode 100644 index 0000000..749b23e --- /dev/null +++ b/ansible/roles/core-mon/monitoring/files/dashboards/vm-cpu-system.json @@ -0,0 +1,1132 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": { + "type": "loki", + "uid": "${loki_datasource}" + }, + "enable": true, + "expr": "{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\", transport=\"kernel\", level=\"emerg\"}", + "hide": true, + "iconColor": "light-purple", + "name": "Critical system event", + "tagKeys": "cluster,job,instance,level", + "titleFormat": "Critical system event" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "enable": true, + "expr": "changes(\nsum by (instance) (\n group by (instance,release) (node_uname_info{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"})\n )\n[$__interval:1m] offset -$__interval) > 1\n", + "hide": true, + "iconColor": "light-blue", + "name": "Kernel update", + "step": "5m", + "tagKeys": "cluster,job,instance", + "textFormat": "", + "titleFormat": "Kernel update" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "enable": true, + "expr": "increase(node_vmstat_oom_kill{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__interval])", + "hide": true, + "iconColor": "light-purple", + "name": "OOMkill", + "tagKeys": "cluster,job,instance", + "textFormat": "", + "titleFormat": "OOMkill" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "enable": true, + "expr": "node_boot_time_seconds{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}*1000 > $__from < $__to", + "hide": true, + "iconColor": "light-yellow", + "name": "Reboot", + "tagKeys": "cluster,job,instance", + "titleFormat": "Reboot", + "useValueForTime": "on" + }, + { + "datasource": { + "type": "loki", + "uid": "${loki_datasource}" + }, + "enable": true, + "expr": "{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\", unit=\"init.scope\"} |= \"code=exited, status=1/FAILURE\"", + "hide": true, + "iconColor": "light-orange", + "name": "Service failed", + "tagKeys": "cluster,job,instance,level", + "titleFormat": "Service failed" + }, + { + "datasource": { + "type": "loki", + "uid": "${loki_datasource}" + }, + "enable": true, + "expr": "{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\", unit=\"systemd-logind.service\"}|= \"New session\"", + "hide": true, + "name": "Session closed", + "tagKeys": "cluster,job,instance,level", + "titleFormat": "Session closed" + }, + { + "datasource": { + "type": "loki", + "uid": "${loki_datasource}" + }, + "enable": true, + "expr": "{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\", unit=\"systemd-logind.service\"}|= \"New session\"", + "hide": true, + "name": "Session opened", + "tagKeys": "cluster,job,instance,level", + "titleFormat": "Session opened" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 2, + "links": [ + { + "keepTime": true, + "title": "Back to Linux node / fleet", + "type": "link", + "url": "/d/node-fleet" + }, + { + "keepTime": true, + "title": "Back to Linux node / overview", + "type": "link", + "url": "/d/nodes" + }, + { + "asDropdown": true, + "includeVars": true, + "keepTime": true, + "tags": [ + "linux-node-integration" + ], + "title": "All Linux node / dashboards", + "type": "dashboards" + } + ], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "System", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Total CPU utilization percent is a metric that indicates the overall level of central processing unit (CPU) usage in a computer system.\nIt represents the combined load placed on all CPU cores or processors.\n\nFor instance, if the total CPU utilization percent is 50%, it means that,\non average, half of the CPU's processing capacity is being used to execute tasks. A higher percentage indicates that the CPU is working more intensively, potentially leading to system slowdowns if it remains consistently high.\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-BlYlRd" + }, + "decimals": 1, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "(((count by (instance) (count(node_cpu_seconds_total{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}) by (cpu, instance))) \n- \navg by (instance) (sum by (instance, mode)(irate(node_cpu_seconds_total{mode='idle',job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])))) * 100) \n/ \ncount by(instance) (count(node_cpu_seconds_total{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}) by (cpu, instance))\n", + "legendFormat": "CPU usage", + "refId": "A" + } + ], + "title": "CPU usage", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Total CPU utilization percent is a metric that indicates the overall level of central processing unit (CPU) usage in a computer system.\nIt represents the combined load placed on all CPU cores or processors.\n\nFor instance, if the total CPU utilization percent is 50%, it means that,\non average, half of the CPU's processing capacity is being used to execute tasks. A higher percentage indicates that the CPU is working more intensively, potentially leading to system slowdowns if it remains consistently high.\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-BlYlRd" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "scheme", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 9, + "x": 6, + "y": 1 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "(\n (1 - sum without (mode) (rate(node_cpu_seconds_total{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\", mode=~\"idle|iowait|steal\"}[$__rate_interval])))\n/ ignoring(cpu) group_left\n count without (cpu, mode) (node_cpu_seconds_total{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\", mode=\"idle\"})\n) * 100\n", + "legendFormat": "CPU {{cpu}}", + "refId": "A" + } + ], + "title": "CPU usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "- System: Processes executing in kernel mode.\n- User: Normal processes executing in user mode.\n- Nice: Niced processes executing in user mode.\n- Idle: Waiting for something to happen.\n- Iowait: Waiting for I/O to complete.\n- Irq: Servicing interrupts.\n- Softirq: Servicing softirqs.\n- Steal: Time spent in other operating systems when running in a virtualized environment.\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 80, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "idle" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "interrupt" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-purple", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "user" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "system|privileged" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 9, + "x": 15, + "y": 1 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by(instance, mode) (irate(node_cpu_seconds_total{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])) \n/ on(instance) \ngroup_left sum by (instance)((irate(node_cpu_seconds_total{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))) * 100\n", + "legendFormat": "{{ mode }}", + "refId": "A" + } + ], + "title": "CPU usage by modes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "System load average over the previous 1, 5, and 15 minute ranges.\n\nA measurement of how many processes are waiting for CPU cycles. The maximum number is the number of CPU cores for the node.\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "Cores" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-orange", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_load1{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "1m", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_load5{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "5m", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_load15{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "15m", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "count without (cpu) (node_cpu_seconds_total{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\", mode=\"idle\"})", + "legendFormat": "Cores", + "refId": "D" + } + ], + "title": "Load average", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Context switches occur when the operating system switches from running one process to another. Interrupts are signals sent to the CPU by external devices to request its attention.\n\nA high number of context switches or interrupts can indicate that the system is overloaded or that there are problems with specific devices or processes.\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_context_switches_total{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "legendFormat": "Context switches", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_intr_total{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "legendFormat": "Interrupts", + "refId": "B" + } + ], + "title": "Context switches/Interrupts", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 7, + "panels": [], + "title": "Time", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Current system timezone.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "text" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 15 + }, + "id": 8, + "options": { + "colorMode": "fixed", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^time_zone$/", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_time_zone_offset_seconds{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "format": "table", + "refId": "A" + } + ], + "title": "Timezone", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Status of time synchronization.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed" + }, + "custom": { + "fillOpacity": 70, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1 + }, + "mappings": [ + { + "options": { + "0": { + "color": "light-yellow", + "index": 1, + "text": "Not in sync" + }, + "1": { + "color": "light-green", + "index": 0, + "text": "In sync" + } + }, + "type": "value" + } + ], + "noValue": "No data.", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 21, + "x": 3, + "y": 15 + }, + "id": 9, + "maxDataPoints": 50, + "options": { + "colWidth": 0.9, + "legend": false, + "rowHeight": 0.9, + "showValue": "auto", + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_timex_sync_status{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "NTP status", + "refId": "A" + } + ], + "title": "NTP status", + "type": "status-history" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Time synchronization is essential to ensure accurate timekeeping, which is critical for many system operations such as logging, authentication, and network communication, as well as distributed systems or clusters where data consistency is important.\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "noValue": "No data.", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "seconds" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_timex_estimated_error_seconds{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "Estimated error in seconds", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_timex_offset_seconds{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "Time offset", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_timex_maxerror_seconds{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "Maximum error in seconds", + "refId": "C" + } + ], + "title": "Time synchronized drift", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "30s", + "schemaVersion": 40, + "tags": [ + "linux-node-integration" + ], + "templating": { + "list": [ + { + "current": { + "text": "VictoriaMetrics", + "value": "victoriametrics", + "selected": true + }, + "label": "Data source", + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "(?!grafanacloud-usage|grafanacloud-ml-metrics).+", + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "includeAll": true, + "label": "Cluster", + "multi": true, + "name": "cluster", + "options": [], + "query": "label_values(node_uname_info{job=~\"integrations/(node_exporter|unix)\"}, cluster)", + "refresh": 2, + "sort": 1, + "type": "query" + }, + { + "allValue": ".+", + "current": { + "text": "All", + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "includeAll": true, + "label": "Job", + "multi": true, + "name": "job", + "options": [], + "query": "label_values(node_uname_info{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\"}, job)", + "refresh": 2, + "sort": 1, + "type": "query" + }, + { + "allValue": ".+", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "includeAll": false, + "label": "Instance", + "name": "instance", + "options": [], + "query": "label_values(node_uname_info{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\"}, instance)", + "refresh": 2, + "sort": 1, + "type": "query" + }, + { + "current": { + "text": "loki", + "value": "de87nbxyse41sb" + }, + "hide": 2, + "label": "Loki data source", + "name": "loki_datasource", + "options": [], + "query": "loki", + "refresh": 1, + "regex": "(?!grafanacloud.+usage-insights|grafanacloud.+alert-state-history).+", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "default", + "title": "Linux node / CPU and system", + "uid": "node-system", + "version": 1, + "weekStart": "" +} diff --git a/ansible/roles/core-mon/monitoring/files/dashboards/vm-filesystem-disks.json b/ansible/roles/core-mon/monitoring/files/dashboards/vm-filesystem-disks.json new file mode 100644 index 0000000..4ee3739 --- /dev/null +++ b/ansible/roles/core-mon/monitoring/files/dashboards/vm-filesystem-disks.json @@ -0,0 +1,1419 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": { + "type": "loki", + "uid": "${loki_datasource}" + }, + "enable": true, + "expr": "{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\", transport=\"kernel\", level=\"emerg\"}", + "hide": true, + "iconColor": "light-purple", + "name": "Critical system event", + "tagKeys": "cluster,job,instance,level", + "titleFormat": "Critical system event" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "enable": true, + "expr": "changes(\nsum by (instance) (\n group by (instance,release) (node_uname_info{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"})\n )\n[$__interval:1m] offset -$__interval) > 1\n", + "hide": true, + "iconColor": "light-blue", + "name": "Kernel update", + "step": "5m", + "tagKeys": "cluster,job,instance", + "textFormat": "", + "titleFormat": "Kernel update" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "enable": true, + "expr": "increase(node_vmstat_oom_kill{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__interval])", + "hide": true, + "iconColor": "light-purple", + "name": "OOMkill", + "tagKeys": "cluster,job,instance", + "textFormat": "", + "titleFormat": "OOMkill" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "enable": true, + "expr": "node_boot_time_seconds{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}*1000 > $__from < $__to", + "hide": true, + "iconColor": "light-yellow", + "name": "Reboot", + "tagKeys": "cluster,job,instance", + "titleFormat": "Reboot", + "useValueForTime": "on" + }, + { + "datasource": { + "type": "loki", + "uid": "${loki_datasource}" + }, + "enable": true, + "expr": "{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\", unit=\"init.scope\"} |= \"code=exited, status=1/FAILURE\"", + "hide": true, + "iconColor": "light-orange", + "name": "Service failed", + "tagKeys": "cluster,job,instance,level", + "titleFormat": "Service failed" + }, + { + "datasource": { + "type": "loki", + "uid": "${loki_datasource}" + }, + "enable": true, + "expr": "{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\", unit=\"systemd-logind.service\"}|= \"New session\"", + "hide": true, + "name": "Session closed", + "tagKeys": "cluster,job,instance,level", + "titleFormat": "Session closed" + }, + { + "datasource": { + "type": "loki", + "uid": "${loki_datasource}" + }, + "enable": true, + "expr": "{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\", unit=\"systemd-logind.service\"}|= \"New session\"", + "hide": true, + "name": "Session opened", + "tagKeys": "cluster,job,instance,level", + "titleFormat": "Session opened" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 3, + "links": [ + { + "keepTime": true, + "title": "Back to Linux node / fleet", + "type": "link", + "url": "/d/node-fleet" + }, + { + "keepTime": true, + "title": "Back to Linux node / overview", + "type": "link", + "url": "/d/nodes" + }, + { + "asDropdown": true, + "includeVars": true, + "keepTime": true, + "tags": [ + "linux-node-integration" + ], + "title": "All Linux node / dashboards", + "type": "dashboards" + } + ], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "Filesystem", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Filesystem space utilisation in bytes, by mountpoint.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 1, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_filesystem_avail_bytes{fstype!=\"\", mountpoint!=\"\", job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "{{ mountpoint }} free", + "refId": "A" + } + ], + "title": "Filesystem space availabe", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Disk utilisation in percent, by mountpoint. Some duplication can occur if the same filesystem is mounted in multiple locations.", + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "light-blue", + "value": null + }, + { + "color": "light-yellow", + "value": 0.8 + }, + { + "color": "light-red", + "value": 0.9 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Mounted on" + }, + "properties": [ + { + "id": "custom.width", + "value": "260" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Size" + }, + "properties": [ + { + "id": "custom.width", + "value": "80" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Used" + }, + "properties": [ + { + "id": "custom.width", + "value": "80" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Available" + }, + "properties": [ + { + "id": "custom.width", + "value": "80" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Used, %" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "mode": "basic", + "type": "gauge" + } + }, + { + "id": "max", + "value": 1 + }, + { + "id": "min", + "value": 0 + }, + { + "id": "unit", + "value": "percentunit" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 3, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_filesystem_size_bytes{fstype!=\"\", mountpoint!=\"\", job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "format": "table", + "instant": true, + "refId": "TOTAL" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_filesystem_avail_bytes{fstype!=\"\", mountpoint!=\"\", job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "format": "table", + "instant": true, + "legendFormat": "{{ mountpoint }} free", + "refId": "FREE" + } + ], + "title": "Disk space usage", + "transformations": [ + { + "id": "groupBy", + "options": { + "fields": { + "Value #FREE": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "Value #TOTAL": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "mountpoint": { + "aggregations": [], + "operation": "groupby" + } + } + } + }, + { + "id": "merge", + "options": {} + }, + { + "id": "calculateField", + "options": { + "alias": "Used", + "binary": { + "left": "Value #TOTAL (lastNotNull)", + "operator": "-", + "reducer": "sum", + "right": "Value #FREE (lastNotNull)" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "calculateField", + "options": { + "alias": "Used, %", + "binary": { + "left": "Used", + "operator": "/", + "reducer": "sum", + "right": "Value #TOTAL (lastNotNull)" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": { + "Used": 3, + "Used, %": 4, + "Value #FREE (lastNotNull)": 2, + "Value #TOTAL (lastNotNull)": 1, + "mountpoint": 0 + }, + "renameByName": { + "Value #FREE (lastNotNull)": "Available", + "Value #TOTAL (lastNotNull)": "Size", + "mountpoint": "Mounted on" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "desc": false, + "field": "Mounted on" + } + ] + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The inode is a data structure in a Unix-style file system that describes a file-system object such as a file or a directory.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 1, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_filesystem_files_free{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\", fstype!=\"\", mountpoint!=\"\"}", + "legendFormat": "{{ mountpoint }} inodes free", + "refId": "A" + } + ], + "title": "Free inodes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The inode is a data structure in a Unix-style file system that describes a file-system object such as a file or a directory.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 1, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_filesystem_files{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\", fstype!=\"\", mountpoint!=\"\"}", + "legendFormat": "{{ mountpoint }} inodes total", + "refId": "A" + } + ], + "title": "Total inodes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 1, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 17 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_filesystem_device_error{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\", fstype!=\"\", mountpoint!=\"\"}", + "legendFormat": "{{ mountpoint }} device error", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_filesystem_readonly{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\", fstype!=\"\", mountpoint!=\"\"}", + "legendFormat": "{{ mountpoint }} read-only", + "refId": "B" + } + ], + "title": "Filesystems with errors / read-only", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "File descriptor is a handle to an open file or input/output (I/O) resource, such as a network socket or a pipe.\nThe operating system uses file descriptors to keep track of open files and I/O resources, and provides a way for programs to read from and write to them.\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 1, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 17 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "process_max_fds{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "Maximum open file descriptors", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "process_open_fds{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "Open file descriptors", + "refId": "B" + } + ], + "title": "File descriptors", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 8, + "panels": [], + "title": "Disk", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Disk read/writes in bytes per second.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 1, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/time|used|busy|util/" + }, + "properties": [ + { + "id": "custom.axisSoftMax", + "value": 100 + }, + { + "id": "custom.drawStyle", + "value": "points" + }, + { + "id": "unit", + "value": "percent" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 26 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_disk_read_bytes_total{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\", device!=\"\"}[$__rate_interval])", + "legendFormat": "{{ device }} read", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_disk_written_bytes_total{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\", device!=\"\"}[$__rate_interval])", + "legendFormat": "{{ device }} written", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_disk_io_time_seconds_total{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\", device!=\"\"}[$__rate_interval])", + "legendFormat": "{{ device }} io util", + "refId": "C" + } + ], + "title": "Disk reads/writes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The number of I/O requests per second for the device/volume.\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": true, + "axisColorMode": "text", + "axisLabel": "write(-) | read(+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 1, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "iops" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/write|written/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 26 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_disk_reads_completed_total{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\", device!=\"\"}[$__rate_interval])\n", + "legendFormat": "{{ device }} reads", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_disk_writes_completed_total{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\", device!=\"\"}[$__rate_interval])\n", + "legendFormat": "{{ device }} writes", + "refId": "B" + } + ], + "title": "Disk I/O", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The average time for requests issued to the device to be served.\nThis includes the time spent by the requests in queue and the time spent servicing them.'\n", + "fieldConfig": { + "defaults": { + "custom": { + "axisCenteredZero": true, + "axisLabel": "write(-) | read(+)", + "fillOpacity": 1, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never" + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/write|written/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 34 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_disk_read_time_seconds_total{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\", device!=\"\"}[$__rate_interval])\n/\nirate(node_disk_reads_completed_total{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\", device!=\"\"}[$__rate_interval])\n", + "legendFormat": "{{ device }} avg read time", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_disk_write_time_seconds_total{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\", device!=\"\"}[$__rate_interval])\n/\nirate(node_disk_writes_completed_total{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\", device!=\"\"}[$__rate_interval])\n", + "legendFormat": "{{ device }} avg write time", + "refId": "B" + } + ], + "title": "Disk average wait time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Disk average IO queue.", + "fieldConfig": { + "defaults": { + "custom": { + "axisCenteredZero": true, + "axisLabel": "write(-) | read(+)", + "fillOpacity": 1, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never" + } + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/write|written/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 34 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_disk_io_time_weighted_seconds_total{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\", device!=\"\"}[$__rate_interval])", + "legendFormat": "{{ device }} avg queue", + "refId": "A" + } + ], + "title": "Disk average queue", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "30s", + "schemaVersion": 40, + "tags": [ + "linux-node-integration" + ], + "templating": { + "list": [ + { + "current": { + "text": "VictoriaMetrics", + "value": "victoriametrics", + "selected": true + }, + "label": "Data source", + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "(?!grafanacloud-usage|grafanacloud-ml-metrics).+", + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "includeAll": true, + "label": "Cluster", + "multi": true, + "name": "cluster", + "options": [], + "query": "label_values(node_uname_info{job=~\"integrations/(node_exporter|unix)\"}, cluster)", + "refresh": 2, + "sort": 1, + "type": "query" + }, + { + "allValue": ".+", + "current": { + "text": "All", + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "includeAll": true, + "label": "Job", + "multi": true, + "name": "job", + "options": [], + "query": "label_values(node_uname_info{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\"}, job)", + "refresh": 2, + "sort": 1, + "type": "query" + }, + { + "allValue": ".+", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "includeAll": false, + "label": "Instance", + "name": "instance", + "options": [], + "query": "label_values(node_uname_info{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\"}, instance)", + "refresh": 2, + "sort": 1, + "type": "query" + }, + { + "current": { + "text": "loki", + "value": "de87nbxyse41sb" + }, + "hide": 2, + "label": "Loki data source", + "name": "loki_datasource", + "options": [], + "query": "loki", + "refresh": 1, + "regex": "(?!grafanacloud.+usage-insights|grafanacloud.+alert-state-history).+", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "default", + "title": "Linux node / filesystem and disks", + "uid": "node-disk", + "version": 1, + "weekStart": "" +} diff --git a/ansible/roles/core-mon/monitoring/files/dashboards/vm-memory.json b/ansible/roles/core-mon/monitoring/files/dashboards/vm-memory.json new file mode 100644 index 0000000..eec63ac --- /dev/null +++ b/ansible/roles/core-mon/monitoring/files/dashboards/vm-memory.json @@ -0,0 +1,2199 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": { + "type": "loki", + "uid": "${loki_datasource}" + }, + "enable": true, + "expr": "{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\", transport=\"kernel\", level=\"emerg\"}", + "hide": true, + "iconColor": "light-purple", + "name": "Critical system event", + "tagKeys": "cluster,job,instance,level", + "titleFormat": "Critical system event" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "enable": true, + "expr": "changes(\nsum by (instance) (\n group by (instance,release) (node_uname_info{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"})\n )\n[$__interval:1m] offset -$__interval) > 1\n", + "hide": true, + "iconColor": "light-blue", + "name": "Kernel update", + "step": "5m", + "tagKeys": "cluster,job,instance", + "textFormat": "", + "titleFormat": "Kernel update" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "enable": true, + "expr": "increase(node_vmstat_oom_kill{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__interval])", + "hide": true, + "iconColor": "light-purple", + "name": "OOMkill", + "tagKeys": "cluster,job,instance", + "textFormat": "", + "titleFormat": "OOMkill" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "enable": true, + "expr": "node_boot_time_seconds{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}*1000 > $__from < $__to", + "hide": true, + "iconColor": "light-yellow", + "name": "Reboot", + "tagKeys": "cluster,job,instance", + "titleFormat": "Reboot", + "useValueForTime": "on" + }, + { + "datasource": { + "type": "loki", + "uid": "${loki_datasource}" + }, + "enable": true, + "expr": "{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\", unit=\"init.scope\"} |= \"code=exited, status=1/FAILURE\"", + "hide": true, + "iconColor": "light-orange", + "name": "Service failed", + "tagKeys": "cluster,job,instance,level", + "titleFormat": "Service failed" + }, + { + "datasource": { + "type": "loki", + "uid": "${loki_datasource}" + }, + "enable": true, + "expr": "{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\", unit=\"systemd-logind.service\"}|= \"New session\"", + "hide": true, + "name": "Session closed", + "tagKeys": "cluster,job,instance,level", + "titleFormat": "Session closed" + }, + { + "datasource": { + "type": "loki", + "uid": "${loki_datasource}" + }, + "enable": true, + "expr": "{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\", unit=\"systemd-logind.service\"}|= \"New session\"", + "hide": true, + "name": "Session opened", + "tagKeys": "cluster,job,instance,level", + "titleFormat": "Session opened" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 6, + "links": [ + { + "keepTime": true, + "title": "Back to Linux node / fleet", + "type": "link", + "url": "/d/node-fleet" + }, + { + "keepTime": true, + "title": "Back to Linux node / overview", + "type": "link", + "url": "/d/nodes" + }, + { + "asDropdown": true, + "includeVars": true, + "keepTime": true, + "tags": [ + "linux-node-integration" + ], + "title": "All Linux node / dashboards", + "type": "dashboards" + } + ], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "RAM (random-access memory) currently in use by the operating system and running applications, in percent.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-BlYlRd" + }, + "decimals": 1, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "100 -\n(\n avg by (instance) (node_memory_MemAvailable_bytes{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}) /\n avg by (instance) (node_memory_MemTotal_bytes{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"})\n* 100\n)\n", + "refId": "A" + } + ], + "title": "Memory usage", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "- Used: The amount of physical memory currently in use by the system.\n- Cached: The amount of physical memory used for caching data from disk. The Linux kernel uses available memory to cache data that is read from or written to disk. This helps speed up disk access times.\n- Free: The amount of physical memory that is currently not in use.\n- Buffers: The amount of physical memory used for temporary storage of data being transferred between devices or applications.\n- Available: The amount of physical memory that is available for use by applications. This takes into account memory that is currently being used for caching but can be freed up if needed.\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*(T|t)otal.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-orange", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + } + ] + }, + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "Memory total", + "Memory used" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 18, + "x": 6, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "(\n node_memory_MemTotal_bytes{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}\n-\n node_memory_MemFree_bytes{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}\n-\n node_memory_Buffers_bytes{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}\n-\n node_memory_Cached_bytes{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}\n)\n", + "legendFormat": "Memory used", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_Cached_bytes{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "Memory cached", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_MemAvailable_bytes{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "Memory available", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_Buffers_bytes{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "Memory buffers", + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_MemFree_bytes{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "Memory free", + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_MemTotal_bytes{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "Memory total", + "refId": "F" + } + ], + "title": "Memory usage", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 3, + "panels": [], + "title": "Vmstat", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Page-In - Return of pages to physical memory. This is a common and normal event.\n\nPage-Out - process of writing pages to disk. Unlike page-in, page-outs can indicate trouble.\nWhen the kernel detects low memory, it attempts to free memory by paging out. \nWhile occasional page-outs are normal, excessive and frequent page-outs can lead to thrashing.\nThrashing is a state in which the kernel spends more time managing paging activity than running applications, resulting in poor system performance.\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "out(-) | in(+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/transmit|tx|out/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_vmstat_pgpgin{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "legendFormat": "Page-In", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_vmstat_pgpgout{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "legendFormat": "Page-Out", + "refId": "B" + } + ], + "title": "Memory pages in / out", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Compared to the speed of the CPU and main memory, writing pages out to disk is relatively slow.\nNonetheless, it is a preferable option to crashing or killing off processes.\n\nThe process of writing pages out to disk to free memory is known as swapping-out.\nIf a page fault occurs because the page is on disk, in the swap area, rather than in memory,\nthe kernel will read the page back in from the disk to satisfy the page fault. \nThis is known as swapping-in.\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "out(-) | in(+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/transmit|tx|out/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_vmstat_pswpin{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "legendFormat": "Pages swapped in", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_vmstat_pswpout{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "legendFormat": "Pages swapped out", + "refId": "B" + } + ], + "title": "Memory pages swapping in / out", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "A page fault is an exception raised by the memory when a process accesses a memory page without the necessary preparations,\nrequiring a mapping to be added to the process's virtual address space.\n\nThe page contents may also need to be loaded from a backing store such as a disk.\nWhile the MMU detects the page fault, the operating system's kernel handles the exception by either making the required page accessible in physical memory or denying an illegal memory access.\nValid page faults are common and necessary to increase memory availability in any operating system that uses virtual memory, including Windows, macOS, and the Linux kernel.\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 15 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_vmstat_pgmajfault{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "legendFormat": "Major page fault operations", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_vmstat_pgfault{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])\n-\nirate(node_vmstat_pgmajfault{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])\n", + "legendFormat": "Minor page fault operations", + "refId": "B" + } + ], + "title": "Memory page faults", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Out Of Memory killer is a process used by the Linux kernel when the system is running critically low on memory.\n\nThis can happen when the kernel has allocated more memory than is available for its processes.\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 15 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "increase(node_vmstat_oom_kill{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__interval])", + "legendFormat": "OOM killer invocations", + "refId": "A" + } + ], + "title": "OOM Killer", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 8, + "panels": [], + "title": "Memstat", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "- Inactive: Memory which has been less recently used. It is more eligible to be reclaimed for other purposes.\n- Active: Memory that has been used more recently and usually not reclaimed unless absolutely necessary.\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*(T|t)otal.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-orange", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_Active_bytes{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "Active", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_Inactive_bytes{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "Inactive", + "refId": "B" + } + ], + "title": "Memory active / inactive", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "- Inactive_file: File-backed memory on inactive LRU list.\n- Inactive_anon: Anonymous and swap cache on inactive LRU list, including tmpfs (shmem).\n- Active_file: File-backed memory on active LRU list.\n- Active_anon: Anonymous and swap cache on active least-recently-used (LRU) list, including tmpfs.\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*(T|t)otal.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-orange", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_Inactive_file_bytes{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "Inactive_file", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_Inactive_anon_bytes{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "Inactive_anon", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_Active_file_bytes{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "Active_file", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_Active_anon_bytes{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "Active_anon", + "refId": "D" + } + ], + "title": "Memory active / inactive details", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "- Committed_AS - Amount of memory presently allocated on the system.\n- CommitLimit - Amount of memory currently available to be allocated on the system.\n", + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 30, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never" + }, + "min": 0, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*(T|t)otal.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-orange", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_Committed_AS_bytes{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "Commited_AS", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_CommitLimit_bytes{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "CommitLimit", + "refId": "B" + } + ], + "title": "Memory commited", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "- Mapped: This refers to the memory used in mapped page files that have been memory mapped, such as libraries.\n- Shmem: This is the memory used by shared memory, which is shared between multiple processes, including RAM disks.\n- ShmemHugePages: This is the memory used by shared memory and tmpfs allocated with huge pages.\n- ShmemPmdMapped: This is the amount of shared memory (shmem/tmpfs) backed by huge pages.\n", + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 30, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never" + }, + "min": 0, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*(T|t)otal.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-orange", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_Mapped_bytes{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "Mapped", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_Shmem_bytes{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "Shmem", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_ShmemPmdMapped_bytes{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "ShmemPmdMapped", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_ShmemHugePages_bytes{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "ShmemHugePages", + "refId": "D" + } + ], + "title": "Memory shared and mapped", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "- Writeback: This refers to the memory that is currently being actively written back to the disk.\n- WritebackTmp: This is the memory used by FUSE for temporary writeback buffers.\n- Dirty: This type of memory is waiting to be written back to the disk.\n", + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 30, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never" + }, + "min": 0, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*(T|t)otal.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-orange", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_Writeback_bytes{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "Writeback", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_WritebackTmp_bytes{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "WritebackTmp", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_Dirty_bytes{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "Dirty", + "refId": "C" + } + ], + "title": "Memory writeback and dirty", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Virtual Memory Allocation is a type of memory allocation in Linux that allows a process to request a contiguous block of memory larger than the amount of physically available memory. This is achieved by mapping the requested memory to virtual addresses that are backed by a combination of physical memory and swap space on disk.\n\n- VmallocChunk: Largest contiguous block of vmalloc area which is free.\n- VmallocTotal: Total size of vmalloc memory area.\n- VmallocUsed: Amount of vmalloc area which is used.\n", + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 30, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never" + }, + "min": 0, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*(T|t)otal.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-orange", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 14, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_VmallocChunk_bytes{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "VmallocChunk", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_VmallocTotal_bytes{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "VmallocTotal", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_VmallocUsed_bytes{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "VmallocUsed", + "refId": "C" + } + ], + "title": "Memory Vmalloc", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Slab Allocation is a type of memory allocation in Linux that allows the kernel to efficiently manage the allocation and deallocation of small and frequently used data structures, such as network packets, file system objects, and process descriptors.\n\nThe Slab Allocator maintains a cache of pre-allocated objects of a fixed size and type, called slabs. When an application requests an object of a particular size and type, the Slab Allocator checks if a pre-allocated object of that size and type is available in the cache. If an object is available, it is returned to the application; if not, a new slab of objects is allocated and added to the cache.\n\n- SUnreclaim: Part of Slab, that cannot be reclaimed on memory pressure.\n- SReclaimable: Part of Slab, that might be reclaimed, such as caches.\n", + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 30, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never" + }, + "min": 0, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*(T|t)otal.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-orange", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 48 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_SUnreclaim_bytes{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "SUnreclaim", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_SReclaimable_bytes{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "SReclaimable", + "refId": "B" + } + ], + "title": "Memory slab", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Memory Anonymous refers to the portion of the virtual memory that is used by a process for dynamically allocated memory that is not backed by any file or device.\n\nThis type of memory is commonly used for heap memory allocation, which is used by programs to allocate and free memory dynamically during runtime.\n\nMemory Anonymous is different from Memory Mapped files, which refer to portions of the virtual memory space that are backed by a file or device,\nand from Memory Shared with other processes,\nwhich refers to memory regions that can be accessed and modified by multiple processes.\n\n- AnonHugePages: Memory in anonymous huge pages.\n- AnonPages: Memory in user pages not backed by files.\n", + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 30, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never" + }, + "min": 0, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*(T|t)otal.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-orange", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 48 + }, + "id": 16, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_AnonHugePages_bytes{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "AnonHugePages", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_AnonPages_bytes{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "AnonPages", + "refId": "B" + } + ], + "title": "Memory slab", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Huge Pages are a feature that allows for the allocation of larger memory pages than the standard 4KB page size. By using larger page sizes, the kernel can reduce the overhead associated with managing a large number of smaller pages, which can improve system performance for certain workloads.\n\n- HugePages_Free: Huge pages in the pool that are not yet allocated.\n- HugePages_Rsvd: Huge pages for which a commitment to allocate from the pool has been made, but no allocation has yet been made.\n- HugePages_Surp: Huge pages in the pool above the value in /proc/sys/vm/nr_hugepages.\n", + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 30, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never" + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 56 + }, + "id": 17, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_HugePages_Free{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "HugePages_Free", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_HugePages_Rsvd{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "HugePages_Rsvd", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_HugePages_Surp{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "HugePages_Surp", + "refId": "C" + } + ], + "title": "Memory HugePages counter", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Huge Pages are a feature that allows for the allocation of larger memory pages than the standard 4KB page size. By using larger page sizes, the kernel can reduce the overhead associated with managing a large number of smaller pages, which can improve system performance for certain workloads.\n", + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 30, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never" + }, + "min": 0, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*(T|t)otal.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-orange", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 56 + }, + "id": 18, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_HugePages_Total{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "Huge pages total size", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_Hugepagesize_bytes{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "Huge page size", + "refId": "B" + } + ], + "title": "Memory HugePages size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Direct Map memory refers to the portion of the kernel's virtual address space that is directly mapped to physical memory. This mapping is set up by the kernel during boot time and is used to provide fast access to certain critical kernel data structures, such as page tables and interrupt descriptor tables.\n", + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 30, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never" + }, + "min": 0, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*(T|t)otal.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-orange", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 64 + }, + "id": 19, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_DirectMap1G_bytes{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "DirectMap1G", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_DirectMap2M_bytes{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "DirectMap2M", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_DirectMap4k_bytes{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "DirectMap4k", + "refId": "C" + } + ], + "title": "Memory direct map", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Memory bounce is a technique used in the Linux kernel to handle situations where direct memory access (DMA) is required but the physical memory being accessed is not contiguous. This can happen when a device, such as a network interface card or a disk controller, requires access to a large amount of memory that is not available as a single contiguous block.\n\nTo handle this situation, the kernel uses a technique called memory bouncing. In memory bouncing, the kernel sets up a temporary buffer in physical memory that is large enough to hold the entire data block being transferred by the device. The data is then copied from the non-contiguous source memory to the temporary buffer, which is physically contiguous.\n\n- Bounce: Memory used for block device bounce buffers.\n", + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 30, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never" + }, + "min": 0, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*(T|t)otal.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-orange", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 64 + }, + "id": 20, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_Bounce_bytes{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "Bounce", + "refId": "A" + } + ], + "title": "Memory bounce", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "30s", + "schemaVersion": 40, + "tags": [ + "linux-node-integration" + ], + "templating": { + "list": [ + { + "current": { + "text": "VictoriaMetrics", + "value": "victoriametrics", + "selected": true + }, + "label": "Data source", + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "(?!grafanacloud-usage|grafanacloud-ml-metrics).+", + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "includeAll": true, + "label": "Cluster", + "multi": true, + "name": "cluster", + "options": [], + "query": "label_values(node_uname_info{job=~\"integrations/(node_exporter|unix)\"}, cluster)", + "refresh": 2, + "sort": 1, + "type": "query" + }, + { + "allValue": ".+", + "current": { + "text": "All", + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "includeAll": true, + "label": "Job", + "multi": true, + "name": "job", + "options": [], + "query": "label_values(node_uname_info{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\"}, job)", + "refresh": 2, + "sort": 1, + "type": "query" + }, + { + "allValue": ".+", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "includeAll": false, + "label": "Instance", + "name": "instance", + "options": [], + "query": "label_values(node_uname_info{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\"}, instance)", + "refresh": 2, + "sort": 1, + "type": "query" + }, + { + "current": { + "text": "loki", + "value": "de87nbxyse41sb" + }, + "hide": 2, + "label": "Loki data source", + "name": "loki_datasource", + "options": [], + "query": "loki", + "refresh": 1, + "regex": "(?!grafanacloud.+usage-insights|grafanacloud.+alert-state-history).+", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "default", + "title": "Linux node / memory", + "uid": "node-memory", + "version": 1, + "weekStart": "" +} diff --git a/ansible/roles/core-mon/monitoring/files/dashboards/vm-network.json b/ansible/roles/core-mon/monitoring/files/dashboards/vm-network.json new file mode 100644 index 0000000..437e08e --- /dev/null +++ b/ansible/roles/core-mon/monitoring/files/dashboards/vm-network.json @@ -0,0 +1,2634 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": { + "type": "loki", + "uid": "${loki_datasource}" + }, + "enable": true, + "expr": "{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\", transport=\"kernel\", level=\"emerg\"}", + "hide": true, + "iconColor": "light-purple", + "name": "Critical system event", + "tagKeys": "cluster,job,instance,level", + "titleFormat": "Critical system event" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "enable": true, + "expr": "changes(\nsum by (instance) (\n group by (instance,release) (node_uname_info{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"})\n )\n[$__interval:1m] offset -$__interval) > 1\n", + "hide": true, + "iconColor": "light-blue", + "name": "Kernel update", + "step": "5m", + "tagKeys": "cluster,job,instance", + "textFormat": "", + "titleFormat": "Kernel update" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "enable": true, + "expr": "increase(node_vmstat_oom_kill{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__interval])", + "hide": true, + "iconColor": "light-purple", + "name": "OOMkill", + "tagKeys": "cluster,job,instance", + "textFormat": "", + "titleFormat": "OOMkill" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "enable": true, + "expr": "node_boot_time_seconds{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}*1000 > $__from < $__to", + "hide": true, + "iconColor": "light-yellow", + "name": "Reboot", + "tagKeys": "cluster,job,instance", + "titleFormat": "Reboot", + "useValueForTime": "on" + }, + { + "datasource": { + "type": "loki", + "uid": "${loki_datasource}" + }, + "enable": true, + "expr": "{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\", unit=\"init.scope\"} |= \"code=exited, status=1/FAILURE\"", + "hide": true, + "iconColor": "light-orange", + "name": "Service failed", + "tagKeys": "cluster,job,instance,level", + "titleFormat": "Service failed" + }, + { + "datasource": { + "type": "loki", + "uid": "${loki_datasource}" + }, + "enable": true, + "expr": "{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\", unit=\"systemd-logind.service\"}|= \"New session\"", + "hide": true, + "name": "Session closed", + "tagKeys": "cluster,job,instance,level", + "titleFormat": "Session closed" + }, + { + "datasource": { + "type": "loki", + "uid": "${loki_datasource}" + }, + "enable": true, + "expr": "{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\", unit=\"systemd-logind.service\"}|= \"New session\"", + "hide": true, + "name": "Session opened", + "tagKeys": "cluster,job,instance,level", + "titleFormat": "Session opened" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 7, + "links": [ + { + "keepTime": true, + "title": "Back to Linux node / fleet", + "type": "link", + "url": "/d/node-fleet" + }, + { + "keepTime": true, + "title": "Back to Linux node / overview", + "type": "link", + "url": "/d/nodes" + }, + { + "asDropdown": true, + "includeVars": true, + "keepTime": true, + "tags": [ + "linux-node-integration" + ], + "title": "All Linux node / dashboards", + "type": "dashboards" + } + ], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "Network", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Network interfaces overview.", + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Speed" + }, + "properties": [ + { + "id": "unit", + "value": "bps" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "Transmitted|Received" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "gauge" + } + }, + { + "id": "color", + "value": { + "mode": "continuous-BlYlRd" + } + }, + { + "id": "max", + "value": 100000000 + }, + { + "id": "unit", + "value": "bps" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "Carrier|Up" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "mappings", + "value": [ + { + "options": { + "0": { + "color": "light-red", + "index": 0, + "text": "Down" + }, + "1": { + "color": "light-green", + "index": 1, + "text": "Up" + } + }, + "type": "value" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_network_up{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "format": "table", + "instant": true, + "legendFormat": "{{device}}", + "refId": "Up" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_network_carrier{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "format": "table", + "instant": true, + "legendFormat": "{{device}}", + "refId": "Carrier" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_network_transmit_bytes_total{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])*8", + "format": "table", + "instant": false, + "legendFormat": "{{ device }} transmitted", + "refId": "Transmitted" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_network_receive_bytes_total{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])*8", + "format": "table", + "instant": false, + "legendFormat": "{{ device }} received", + "refId": "Received" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_arp_entries{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "format": "table", + "instant": true, + "refId": "ARP entries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_network_mtu_bytes{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "format": "table", + "instant": true, + "refId": "MTU" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_network_speed_bytes{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"} * 8", + "format": "table", + "instant": true, + "refId": "Speed" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_network_transmit_queue_length{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "format": "table", + "instant": true, + "refId": "Queue length" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_network_info{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "format": "table", + "instant": true, + "refId": "Info" + } + ], + "title": "Network interfaces overview", + "transformations": [ + { + "id": "joinByField", + "options": { + "byField": "device", + "mode": "outer" + } + }, + { + "id": "filterFieldsByName", + "options": { + "include": { + "pattern": "device|duplex|address|Value.+" + } + } + }, + { + "id": "renameByRegex", + "options": { + "regex": "(Value) #(.*)", + "renamePattern": "$2" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Info": true + }, + "renameByName": { + "address": "Address", + "device": "Interface", + "duplex": "Duplex" + } + } + }, + { + "id": "organize", + "options": { + "indexByName": { + "Carrier": 2, + "Interface": 0, + "Received": 3, + "Transmitted": 4, + "Up": 1 + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Network traffic (bits per sec) measures data transmitted and received.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "out(-) | in(+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/transmit|tx|out/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_network_receive_bytes_total{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])*8\n# only show interfaces that had traffic change at least once during selected dashboard interval:\nand\nincrease(\n node_network_receive_bytes_total{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__range]\n ) > 0\n", + "legendFormat": "{{ device }} received", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_network_transmit_bytes_total{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])*8\n# only show interfaces that had traffic change at least once during selected dashboard interval:\nand\nincrease(\n node_network_transmit_bytes_total{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__range]\n ) > 0\n", + "legendFormat": "{{ device }} transmitted", + "refId": "B" + } + ], + "title": "Network traffic", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Network interfaces carrier status", + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed" + }, + "custom": { + "fillOpacity": 70, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1 + }, + "mappings": [ + { + "options": { + "0": { + "color": "light-red", + "index": 0, + "text": "Down" + }, + "1": { + "color": "light-green", + "index": 1, + "text": "Up" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 4, + "maxDataPoints": 50, + "options": { + "colWidth": 0.9, + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "rowHeight": 0.9, + "showValue": "never", + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_network_carrier{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "{{device}}", + "refId": "A" + } + ], + "title": "Network interfaces carrier status", + "type": "status-history" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Network errors refer to issues that occur during the transmission of data across a network. \n\nThese errors can result from various factors, including physical issues, jitter, collisions, noise and interference.\n\nMonitoring network errors is essential for diagnosing and resolving issues, as they can indicate problems with network hardware or environmental factors affecting network quality.\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "out(-) | in(+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "noValue": "No errors", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/transmit|tx|out/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 17 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_network_receive_errs_total{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "legendFormat": "{{ device }} errors received", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_network_transmit_errs_total{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "legendFormat": "{{ device }} errors transmitted", + "refId": "B" + } + ], + "title": "Network errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Dropped packets occur when data packets traveling through a network are intentionally discarded or lost due to congestion, resource limitations, or network configuration issues. \n\nCommon causes include network congestion, buffer overflows, QoS settings, and network errors, as corrupted or incomplete packets may be discarded by receiving devices.\n\nDropped packets can impact network performance and lead to issues such as degraded voice or video quality in real-time applications.\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "out(-) | in(+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "noValue": "No dropped packets", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/transmit|tx|out/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 17 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_network_receive_drop_total{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "legendFormat": "{{ device }} received dropped", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_network_transmit_drop_total{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "legendFormat": "{{ device }} transmitted dropped", + "refId": "B" + } + ], + "title": "Dropped packets", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Network packet count tracks the number of data packets transmitted and received over a network connection, providing insight into network activity and performance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "out(-) | in(+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/transmit|tx|out/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 25 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_network_receive_packets_total{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "legendFormat": "{{ device }} received", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_network_transmit_packets_total{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "legendFormat": "{{ device }} transmitted", + "refId": "B" + } + ], + "title": "Network packets", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Multicast packets received and transmitted.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "out(-) | in(+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/transmit|tx|out/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 25 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_network_receive_multicast_total{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "legendFormat": "{{ device }} received", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_network_transmit_multicast_total{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "legendFormat": "{{ device }} transmitted", + "refId": "B" + } + ], + "title": "Multicast packets", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Network FIFO (First-In, First-Out) refers to a buffer used by the network stack to store packets in a queue.\nIt is a mechanism used to manage network traffic and ensure that packets are delivered to their destination in the order they were received.\nPackets are stored in the FIFO buffer until they can be transmitted or processed further.\n", + "fieldConfig": { + "defaults": { + "custom": { + "axisCenteredZero": false, + "axisLabel": "out(-) | in(+)", + "fillOpacity": 30, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never" + }, + "decimals": 1, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/transmit|tx|out/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 33 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_network_receive_fifo_total{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "legendFormat": "{{ device }} received", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_network_transmit_fifo_total{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "legendFormat": "{{ device }} transmitted", + "refId": "B" + } + ], + "title": "Network FIFO", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "- Compressed received: \nNumber of correctly received compressed packets. This counters is only meaningful for interfaces which support packet compression (e.g. CSLIP, PPP).\n\n- Compressed transmitted:\nNumber of transmitted compressed packets. This counters is only meaningful for interfaces which support packet compression (e.g. CSLIP, PPP).\n\nhttps://docs.kernel.org/networking/statistics.html\n", + "fieldConfig": { + "defaults": { + "custom": { + "axisCenteredZero": false, + "axisLabel": "out(-) | in(+)", + "fillOpacity": 30, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never" + }, + "decimals": 1, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/transmit|tx|out/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 33 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_network_receive_compressed_total{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "legendFormat": "{{ device }} received", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_network_transmit_compressed_total{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "legendFormat": "{{ device }} transmitted", + "refId": "B" + } + ], + "title": "Compressed packets", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "NF Conntrack is a component of the Linux kernel's netfilter framework that provides stateful packet inspection to track and manage network connections,\nenforce firewall rules, perform NAT, and manage network address/port translation.\n", + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 0, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never" + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 41 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_nf_conntrack_entries{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "NF conntrack entries", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_nf_conntrack_entries_limit{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "NF conntrack limits", + "refId": "B" + } + ], + "title": "NF conntrack", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Softnet packets are received by the network and queued for processing by the kernel's networking stack.\nSoftnet packets are usually generated by network traffic that is directed to the local host, and they are typically processed by the kernel's networking subsystem before being passed on to the relevant application. \n", + "fieldConfig": { + "defaults": { + "custom": { + "axisCenteredZero": false, + "axisLabel": "Dropped(-) | Processed(+)", + "fillOpacity": 30, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never" + }, + "decimals": 1, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/dropped/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 41 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_softnet_processed_total{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "legendFormat": "CPU {{ cpu }} processed", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_softnet_dropped_total{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "legendFormat": "CPU {{ cpu }} dropped", + "refId": "B" + } + ], + "title": "Softnet packets", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "\"Softnet out of quota\" is a network-related metric in Linux that measures the number of times the kernel's softirq processing was unable to handle incoming network traffic due to insufficient softirq processing capacity.\nThis means that the kernel has reached its processing capacity limit for incoming packets, and any additional packets will be dropped or deferred.\n", + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 30, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never" + }, + "decimals": 1, + "unit": "pps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 49 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_softnet_times_squeezed_total{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "legendFormat": "CPU {{ cpu }} out of quota", + "refId": "A" + } + ], + "title": "Softnet out of quota", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 57 + }, + "id": 14, + "panels": [], + "title": "Network sockets", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Number of sockets currently in use.", + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 30, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never" + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 58 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_sockstat_sockets_used{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "IPv4 sockets in use", + "refId": "A" + } + ], + "title": "Sockets in use", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "TCP sockets are used for establishing and managing network connections between two endpoints over the TCP/IP protocol.\n\nOrphan sockets: If a process terminates unexpectedly or is terminated without closing its sockets properly, the sockets may become orphaned.\n", + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 30, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never" + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 66 + }, + "id": 16, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_sockstat_TCP_alloc{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "Allocated", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_sockstat_TCP_inuse{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "IPv4 in use", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_sockstat_TCP6_inuse{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "IPv6 in use", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_sockstat_TCP_orphan{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "Orphan sockets", + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_sockstat_TCP_tw{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "Time wait", + "refId": "E" + } + ], + "title": "Sockets TCP", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "UDP (User Datagram Protocol) and UDPlite (UDP-Lite) sockets are used for transmitting and receiving data over the UDP and UDPlite protocols, respectively.\nBoth UDP and UDPlite are connectionless protocols that do not provide a reliable data delivery mechanism.\n", + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 30, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never" + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 66 + }, + "id": 17, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_sockstat_UDPLITE_inuse{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "IPv4 UDPLITE in use", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_sockstat_UDP_inuse{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "IPv4 UDP in use", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_sockstat_UDPLITE6_inuse{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "IPv6 UDPLITE in use", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_sockstat_UDP6_inuse{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "IPv6 UDP in use", + "refId": "D" + } + ], + "title": "Sockets UDP", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Memory currently in use for sockets.\n", + "fieldConfig": { + "defaults": { + "custom": { + "axisLabel": "Pages", + "fillOpacity": 30, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never" + } + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/bytes/" + }, + "properties": [ + { + "id": "custom.axisLabel", + "value": "Bytes" + }, + { + "id": "custom.drawStyle", + "value": "bars" + }, + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "normal" + } + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "unit", + "value": "bytes" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 74 + }, + "id": 18, + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_sockstat_TCP_mem{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "Memory pages allocated for TCP sockets", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_sockstat_UDP_mem{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "Memory pages allocated for UDP sockets", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_sockstat_TCP_mem_bytes{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "Memory bytes allocated for TCP sockets", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_sockstat_UDP_mem_bytes{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "Memory bytes allocated for UDP sockets", + "refId": "D" + } + ], + "title": "Sockets memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "FRAG (IP fragment) sockets: Used to receive and process fragmented IP packets. FRAG sockets are useful in network monitoring and analysis.\n\nRAW sockets: Allow applications to send and receive raw IP packets directly without the need for a transport protocol like TCP or UDP.\n", + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 30, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never" + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 74 + }, + "id": 19, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_sockstat_FRAG_inuse{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "IPv4 Frag sockets in use", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_sockstat_FRAG6_inuse{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "IPv6 Frag sockets in use", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_sockstat_RAW_inuse{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "IPv4 Raw sockets in use", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_sockstat_RAW6_inuse{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "IPv6 Raw sockets in use", + "refId": "D" + } + ], + "title": "Sockets other", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 82 + }, + "id": 20, + "panels": [], + "title": "Network netstat", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Rate of IP octets received and transmitted.", + "fieldConfig": { + "defaults": { + "custom": { + "axisCenteredZero": false, + "axisLabel": "out(-) | in(+)", + "fillOpacity": 30, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never" + }, + "decimals": 1, + "unit": "oct/s" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/transmit|tx|out/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 83 + }, + "id": 21, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_netstat_IpExt_InOctets{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "legendFormat": "Octets received", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_netstat_IpExt_OutOctets{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "legendFormat": "Octets transmitted", + "refId": "B" + } + ], + "title": "IP octets", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Rate of TCP segments received and transmitted.", + "fieldConfig": { + "defaults": { + "custom": { + "axisCenteredZero": false, + "axisLabel": "out(-) | in(+)", + "fillOpacity": 30, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never" + }, + "decimals": 1, + "unit": "seg/s" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/transmit|tx|out/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 91 + }, + "id": 22, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_netstat_Tcp_InSegs{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "legendFormat": "TCP received", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_netstat_Tcp_OutSegs{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "legendFormat": "TCP transmitted", + "refId": "B" + } + ], + "title": "TCP segments", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Rate of TCP errors.", + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 30, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never" + }, + "decimals": 1, + "noValue": "No errors", + "unit": "err/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 91 + }, + "id": 23, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_netstat_TcpExt_ListenOverflows{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "legendFormat": "TCP overflow", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_netstat_TcpExt_ListenDrops{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "legendFormat": "TCP ListenDrops - SYNs to LISTEN sockets ignored", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_netstat_TcpExt_TCPSynRetrans{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "legendFormat": "TCP SYN rentransmits", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_netstat_Tcp_RetransSegs{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "legendFormat": "TCP retransmitted segments, containing one or more previously transmitted octets", + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_netstat_Tcp_InErrs{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "legendFormat": "TCP received with errors", + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_netstat_Tcp_OutRsts{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "legendFormat": "TCP segments sent with RST flag", + "refId": "F" + } + ], + "title": "TCP errors rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Rate of UDP datagrams received and transmitted.", + "fieldConfig": { + "defaults": { + "custom": { + "axisCenteredZero": false, + "axisLabel": "out(-) | in(+)", + "fillOpacity": 30, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never" + }, + "decimals": 1, + "unit": "dat/s" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/transmit|tx|out/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 99 + }, + "id": 24, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_netstat_Udp_InDatagrams{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "legendFormat": "UDP received", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_netstat_Udp_OutDatagrams{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "legendFormat": "UDP transmitted", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_netstat_Udp6_InDatagrams{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "legendFormat": "UDP6 received", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_netstat_Udp6_OutDatagrams{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "legendFormat": "UDP6 transmitted", + "refId": "D" + } + ], + "title": "UDP datagrams", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Rate of UDP errors.", + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 30, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never" + }, + "decimals": 1, + "noValue": "No errors", + "unit": "err/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 99 + }, + "id": 25, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_netstat_UdpLite_InErrors{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "legendFormat": "UDPLite InErrors", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_netstat_Udp_InErrors{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "legendFormat": "UDP InErrors", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_netstat_Udp6_InErrors{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "legendFormat": "UDP6 InErrors", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_netstat_Udp_NoPorts{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "legendFormat": "UDP NoPorts", + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_netstat_Udp6_NoPorts{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "legendFormat": "UDP6 NoPorts", + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_netstat_Udp_RcvbufErrors{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "legendFormat": "UDP receive buffer errors", + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_netstat_Udp6_RcvbufErrors{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "legendFormat": "UDP6 receive buffer errors", + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_netstat_Udp_SndbufErrors{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "legendFormat": "UDP transmit buffer errors", + "refId": "H" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_netstat_Udp6_SndbufErrors{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "legendFormat": "UDP6 transmit buffer errors", + "refId": "I" + } + ], + "title": "UDP errors rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Rate of ICMP messages, like 'ping', received and transmitted.", + "fieldConfig": { + "defaults": { + "custom": { + "axisCenteredZero": false, + "axisLabel": "out(-) | in(+)", + "fillOpacity": 30, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never" + }, + "decimals": 1, + "unit": "msg/s" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/transmit|tx|out/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 107 + }, + "id": 26, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_netstat_Icmp_InMsgs{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "legendFormat": "ICMP received", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_netstat_Icmp_OutMsgs{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "legendFormat": "ICMP transmitted", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_netstat_Icmp6_InMsgs{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "legendFormat": "ICMP6 received", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_netstat_Icmp6_OutMsgs{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "legendFormat": "ICMP6 transmitted", + "refId": "D" + } + ], + "title": "ICMP messages", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Rate of ICMP messages received and transmitted with errors.", + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 30, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never" + }, + "decimals": 1, + "noValue": "No errors", + "unit": "err/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 107 + }, + "id": 27, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_netstat_Icmp_InErrors{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "legendFormat": "ICMP6 errors", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_netstat_Icmp6_InErrors{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "legendFormat": "ICMP6 errors", + "refId": "B" + } + ], + "title": "ICMP errors rate", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "30s", + "schemaVersion": 40, + "tags": [ + "linux-node-integration" + ], + "templating": { + "list": [ + { + "current": { + "text": "VictoriaMetrics", + "value": "victoriametrics", + "selected": true + }, + "label": "Data source", + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "(?!grafanacloud-usage|grafanacloud-ml-metrics).+", + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "includeAll": true, + "label": "Cluster", + "multi": true, + "name": "cluster", + "options": [], + "query": "label_values(node_uname_info{job=~\"integrations/(node_exporter|unix)\"}, cluster)", + "refresh": 2, + "sort": 1, + "type": "query" + }, + { + "allValue": ".+", + "current": { + "text": "All", + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "includeAll": true, + "label": "Job", + "multi": true, + "name": "job", + "options": [], + "query": "label_values(node_uname_info{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\"}, job)", + "refresh": 2, + "sort": 1, + "type": "query" + }, + { + "allValue": ".+", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "includeAll": false, + "label": "Instance", + "name": "instance", + "options": [], + "query": "label_values(node_uname_info{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\"}, instance)", + "refresh": 2, + "sort": 1, + "type": "query" + }, + { + "current": { + "text": "loki", + "value": "de87nbxyse41sb" + }, + "hide": 2, + "label": "Loki data source", + "name": "loki_datasource", + "options": [], + "query": "loki", + "refresh": 1, + "regex": "(?!grafanacloud.+usage-insights|grafanacloud.+alert-state-history).+", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "default", + "title": "Linux node / network", + "uid": "node-network", + "version": 1, + "weekStart": "" +} diff --git a/ansible/roles/core-mon/monitoring/files/dashboards/vm-node-overview.json b/ansible/roles/core-mon/monitoring/files/dashboards/vm-node-overview.json new file mode 100644 index 0000000..19de95b --- /dev/null +++ b/ansible/roles/core-mon/monitoring/files/dashboards/vm-node-overview.json @@ -0,0 +1,2077 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": { + "type": "loki", + "uid": "${loki_datasource}" + }, + "enable": true, + "expr": "{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\", transport=\"kernel\", level=\"emerg\"}", + "hide": true, + "iconColor": "light-purple", + "name": "Critical system event", + "tagKeys": "cluster,job,instance,level", + "titleFormat": "Critical system event" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "enable": true, + "expr": "changes(\nsum by (instance) (\n group by (instance,release) (node_uname_info{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"})\n )\n[$__interval:1m] offset -$__interval) > 1\n", + "hide": true, + "iconColor": "light-blue", + "name": "Kernel update", + "step": "5m", + "tagKeys": "cluster,job,instance", + "textFormat": "", + "titleFormat": "Kernel update" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "enable": true, + "expr": "increase(node_vmstat_oom_kill{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__interval])", + "hide": true, + "iconColor": "light-purple", + "name": "OOMkill", + "tagKeys": "cluster,job,instance", + "textFormat": "", + "titleFormat": "OOMkill" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "enable": true, + "expr": "node_boot_time_seconds{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}*1000 > $__from < $__to", + "hide": true, + "iconColor": "light-yellow", + "name": "Reboot", + "tagKeys": "cluster,job,instance", + "titleFormat": "Reboot", + "useValueForTime": "on" + }, + { + "datasource": { + "type": "loki", + "uid": "${loki_datasource}" + }, + "enable": true, + "expr": "{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\", unit=\"init.scope\"} |= \"code=exited, status=1/FAILURE\"", + "hide": true, + "iconColor": "light-orange", + "name": "Service failed", + "tagKeys": "cluster,job,instance,level", + "titleFormat": "Service failed" + }, + { + "datasource": { + "type": "loki", + "uid": "${loki_datasource}" + }, + "enable": true, + "expr": "{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\", unit=\"systemd-logind.service\"}|= \"New session\"", + "hide": true, + "name": "Session closed", + "tagKeys": "cluster,job,instance,level", + "titleFormat": "Session closed" + }, + { + "datasource": { + "type": "loki", + "uid": "${loki_datasource}" + }, + "enable": true, + "expr": "{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\", unit=\"systemd-logind.service\"}|= \"New session\"", + "hide": true, + "name": "Session opened", + "tagKeys": "cluster,job,instance,level", + "titleFormat": "Session opened" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 8, + "links": [ + { + "keepTime": true, + "title": "Back to Linux node / fleet", + "type": "link", + "url": "/d/node-fleet" + }, + { + "asDropdown": true, + "includeVars": true, + "keepTime": true, + "tags": [ + "linux-node-integration" + ], + "title": "All Linux node / dashboards", + "type": "dashboards" + } + ], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "Overview", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The duration of time that has passed since the last reboot or system start.", + "fieldConfig": { + "defaults": { + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "orange", + "value": null + }, + { + "color": "text", + "value": 600 + } + ] + }, + "unit": "dtdurations" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "time() - node_boot_time_seconds{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "refId": "A" + } + ], + "title": "Uptime", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "System's hostname.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "text" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "fixed", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^nodename$/", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_uname_info{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "format": "table", + "refId": "A" + } + ], + "title": "Hostname", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Kernel version of linux host.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "text" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 4, + "options": { + "colorMode": "fixed", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^release$/", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_uname_info{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "format": "table", + "refId": "A" + } + ], + "title": "Kernel version", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Operating system", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "text" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 5, + "options": { + "colorMode": "fixed", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^pretty_name$/", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_os_info{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}\n", + "format": "table", + "refId": "A" + } + ], + "title": "OS", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "CPU count is the number of processor cores or central processing units (CPUs) in a computer,\ndetermining its processing capability and ability to handle tasks concurrently.\n", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "text" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 6, + "x": 0, + "y": 3 + }, + "id": 6, + "options": { + "colorMode": "fixed", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "count without (cpu) (node_cpu_seconds_total{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\", mode=\"idle\"})", + "legendFormat": "Cores", + "refId": "A" + } + ], + "title": "CPU count", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Amount of random-access memory (RAM) installed.\nIt represents the system's available working memory that applications and the operating system use to perform tasks.\nA higher memory total generally leads to better system performance and the ability to run more demanding applications and processes simultaneously.\n", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "text" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 6, + "x": 6, + "y": 3 + }, + "id": 7, + "options": { + "colorMode": "fixed", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_MemTotal_bytes{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "Memory total", + "refId": "A" + } + ], + "title": "Memory total", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Total swap available.\n\nSwap is a space on a storage device (usually a dedicated swap partition or a swap file) \nused as virtual memory when the physical RAM (random-access memory) is fully utilized.\nSwap space helps prevent memory-related performance issues by temporarily transferring less-used data from RAM to disk,\nfreeing up physical memory for active processes and applications.\n", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "text" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 6, + "x": 12, + "y": 3 + }, + "id": 8, + "options": { + "colorMode": "fixed", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_SwapTotal_bytes{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "refId": "A" + } + ], + "title": "Total swap", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Total capacity on the primary mount point /.\n", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "text" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 6, + "x": 18, + "y": 3 + }, + "id": 9, + "options": { + "colorMode": "fixed", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_filesystem_size_bytes{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\", mountpoint=\"/\", fstype!=\"rootfs\"}", + "refId": "A" + } + ], + "title": "Root mount size", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 10, + "panels": [], + "title": "CPU", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Total CPU utilization percent is a metric that indicates the overall level of central processing unit (CPU) usage in a computer system.\nIt represents the combined load placed on all CPU cores or processors.\n\nFor instance, if the total CPU utilization percent is 50%, it means that,\non average, half of the CPU's processing capacity is being used to execute tasks. A higher percentage indicates that the CPU is working more intensively, potentially leading to system slowdowns if it remains consistently high.\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-BlYlRd" + }, + "decimals": 1, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 6 + }, + "id": 11, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "(((count by (instance) (count(node_cpu_seconds_total{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}) by (cpu, instance))) \n- \navg by (instance) (sum by (instance, mode)(irate(node_cpu_seconds_total{mode='idle',job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])))) * 100) \n/ \ncount by(instance) (count(node_cpu_seconds_total{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}) by (cpu, instance))\n", + "legendFormat": "CPU usage", + "refId": "A" + } + ], + "title": "CPU usage", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Total CPU utilization percent is a metric that indicates the overall level of central processing unit (CPU) usage in a computer system.\nIt represents the combined load placed on all CPU cores or processors.\n\nFor instance, if the total CPU utilization percent is 50%, it means that,\non average, half of the CPU's processing capacity is being used to execute tasks. A higher percentage indicates that the CPU is working more intensively, potentially leading to system slowdowns if it remains consistently high.\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-BlYlRd" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "scheme", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 6, + "y": 6 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "(\n (1 - sum without (mode) (rate(node_cpu_seconds_total{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\", mode=~\"idle|iowait|steal\"}[$__rate_interval])))\n/ ignoring(cpu) group_left\n count without (cpu, mode) (node_cpu_seconds_total{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\", mode=\"idle\"})\n) * 100\n", + "legendFormat": "CPU {{cpu}}", + "refId": "A" + } + ], + "title": "CPU usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "System load average over the previous 1, 5, and 15 minute ranges.\n\nA measurement of how many processes are waiting for CPU cycles. The maximum number is the number of CPU cores for the node.\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "Cores" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-orange", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 6 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_load1{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "1m", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_load5{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "5m", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_load15{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "15m", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "count without (cpu) (node_cpu_seconds_total{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\", mode=\"idle\"})", + "legendFormat": "Cores", + "refId": "D" + } + ], + "title": "Load average", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 14, + "panels": [], + "title": "Memory", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "RAM (random-access memory) currently in use by the operating system and running applications, in percent.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-BlYlRd" + }, + "decimals": 1, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 13 + }, + "id": 15, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "100 -\n(\n avg by (instance) (node_memory_MemAvailable_bytes{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}) /\n avg by (instance) (node_memory_MemTotal_bytes{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"})\n* 100\n)\n", + "refId": "A" + } + ], + "title": "Memory usage", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "- Used: The amount of physical memory currently in use by the system.\n- Cached: The amount of physical memory used for caching data from disk. The Linux kernel uses available memory to cache data that is read from or written to disk. This helps speed up disk access times.\n- Free: The amount of physical memory that is currently not in use.\n- Buffers: The amount of physical memory used for temporary storage of data being transferred between devices or applications.\n- Available: The amount of physical memory that is available for use by applications. This takes into account memory that is currently being used for caching but can be freed up if needed.\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*(T|t)otal.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-orange", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + } + ] + }, + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "Memory total", + "Memory used" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 18, + "x": 6, + "y": 13 + }, + "id": 16, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "(\n node_memory_MemTotal_bytes{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}\n-\n node_memory_MemFree_bytes{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}\n-\n node_memory_Buffers_bytes{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}\n-\n node_memory_Cached_bytes{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}\n)\n", + "legendFormat": "Memory used", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_Cached_bytes{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "Memory cached", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_MemAvailable_bytes{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "Memory available", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_Buffers_bytes{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "Memory buffers", + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_MemFree_bytes{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "Memory free", + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_MemTotal_bytes{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "Memory total", + "refId": "F" + } + ], + "title": "Memory usage", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 17, + "panels": [], + "title": "Disk", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Disk read/writes in bytes per second.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 1, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/time|used|busy|util/" + }, + "properties": [ + { + "id": "custom.axisSoftMax", + "value": 100 + }, + { + "id": "custom.drawStyle", + "value": "points" + }, + { + "id": "unit", + "value": "percent" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 20 + }, + "id": 18, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_disk_read_bytes_total{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\", device!=\"\"}[$__rate_interval])", + "legendFormat": "{{ device }} read", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_disk_written_bytes_total{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\", device!=\"\"}[$__rate_interval])", + "legendFormat": "{{ device }} written", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_disk_io_time_seconds_total{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\", device!=\"\"}[$__rate_interval])", + "legendFormat": "{{ device }} io util", + "refId": "C" + } + ], + "title": "Disk reads/writes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Disk utilisation in percent, by mountpoint. Some duplication can occur if the same filesystem is mounted in multiple locations.", + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "light-blue", + "value": null + }, + { + "color": "light-yellow", + "value": 0.8 + }, + { + "color": "light-red", + "value": 0.9 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Mounted on" + }, + "properties": [ + { + "id": "custom.width", + "value": "260" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Size" + }, + "properties": [ + { + "id": "custom.width", + "value": "80" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Used" + }, + "properties": [ + { + "id": "custom.width", + "value": "80" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Available" + }, + "properties": [ + { + "id": "custom.width", + "value": "80" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Used, %" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "mode": "basic", + "type": "gauge" + } + }, + { + "id": "max", + "value": 1 + }, + { + "id": "min", + "value": 0 + }, + { + "id": "unit", + "value": "percentunit" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 20 + }, + "id": 19, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_filesystem_size_bytes{fstype!=\"\", mountpoint!=\"\", job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "format": "table", + "instant": true, + "refId": "TOTAL" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_filesystem_avail_bytes{fstype!=\"\", mountpoint!=\"\", job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "format": "table", + "instant": true, + "legendFormat": "{{ mountpoint }} free", + "refId": "FREE" + } + ], + "title": "Disk space usage", + "transformations": [ + { + "id": "groupBy", + "options": { + "fields": { + "Value #FREE": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "Value #TOTAL": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "mountpoint": { + "aggregations": [], + "operation": "groupby" + } + } + } + }, + { + "id": "merge", + "options": {} + }, + { + "id": "calculateField", + "options": { + "alias": "Used", + "binary": { + "left": "Value #TOTAL (lastNotNull)", + "operator": "-", + "reducer": "sum", + "right": "Value #FREE (lastNotNull)" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "calculateField", + "options": { + "alias": "Used, %", + "binary": { + "left": "Used", + "operator": "/", + "reducer": "sum", + "right": "Value #TOTAL (lastNotNull)" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": { + "Used": 3, + "Used, %": 4, + "Value #FREE (lastNotNull)": 2, + "Value #TOTAL (lastNotNull)": 1, + "mountpoint": 0 + }, + "renameByName": { + "Value #FREE (lastNotNull)": "Available", + "Value #TOTAL (lastNotNull)": "Size", + "mountpoint": "Mounted on" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "desc": false, + "field": "Mounted on" + } + ] + } + } + ], + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 28 + }, + "id": 20, + "panels": [], + "title": "Network", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Network traffic (bits per sec) measures data transmitted and received.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "out(-) | in(+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/transmit|tx|out/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 29 + }, + "id": 21, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_network_receive_bytes_total{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])*8\n# only show interfaces that had traffic change at least once during selected dashboard interval:\nand\nincrease(\n node_network_receive_bytes_total{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__range]\n ) > 0\n", + "legendFormat": "{{ device }} received", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_network_transmit_bytes_total{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])*8\n# only show interfaces that had traffic change at least once during selected dashboard interval:\nand\nincrease(\n node_network_transmit_bytes_total{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__range]\n ) > 0\n", + "legendFormat": "{{ device }} transmitted", + "refId": "B" + } + ], + "title": "Network traffic", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "**Network errors**:\n\nNetwork errors refer to issues that occur during the transmission of data across a network. \n\nThese errors can result from various factors, including physical issues, jitter, collisions, noise and interference.\n\nMonitoring network errors is essential for diagnosing and resolving issues, as they can indicate problems with network hardware or environmental factors affecting network quality.\n\n**Dropped packets**:\n\nDropped packets occur when data packets traveling through a network are intentionally discarded or lost due to congestion, resource limitations, or network configuration issues. \n\nCommon causes include network congestion, buffer overflows, QoS settings, and network errors, as corrupted or incomplete packets may be discarded by receiving devices.\n\nDropped packets can impact network performance and lead to issues such as degraded voice or video quality in real-time applications.\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "out(-) | in(+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "noValue": "No errors", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/transmit|tx|out/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 29 + }, + "id": 22, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_network_transmit_errs_total{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])>0", + "legendFormat": "{{ device }} errors transmitted", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_network_receive_errs_total{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])>0", + "legendFormat": "{{ device }} errors received", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_network_transmit_drop_total{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])>0", + "legendFormat": "{{ device }} transmitted dropped", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_network_receive_drop_total{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])>0", + "legendFormat": "{{ device }} received dropped", + "refId": "D" + } + ], + "title": "Network errors and dropped packets", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "30s", + "schemaVersion": 40, + "tags": [ + "linux-node-integration" + ], + "templating": { + "list": [ + { + "current": { + "text": "VictoriaMetrics", + "value": "victoriametrics", + "selected": true + }, + "label": "Data source", + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "(?!grafanacloud-usage|grafanacloud-ml-metrics).+", + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "includeAll": true, + "label": "Cluster", + "multi": true, + "name": "cluster", + "options": [], + "query": "label_values(node_uname_info{job=~\"integrations/(node_exporter|unix)\"}, cluster)", + "refresh": 2, + "sort": 1, + "type": "query" + }, + { + "allValue": ".+", + "current": { + "text": "All", + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "includeAll": true, + "label": "Job", + "multi": true, + "name": "job", + "options": [], + "query": "label_values(node_uname_info{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\"}, job)", + "refresh": 2, + "sort": 1, + "type": "query" + }, + { + "allValue": ".+", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "includeAll": false, + "label": "Instance", + "name": "instance", + "options": [], + "query": "label_values(node_uname_info{job=~\"integrations/(node_exporter|unix)\",cluster=~\"$cluster\",job=~\"$job\"}, instance)", + "refresh": 2, + "sort": 1, + "type": "query" + }, + { + "current": { + "text": "loki", + "value": "de87nbxyse41sb" + }, + "hide": 2, + "label": "Loki data source", + "name": "loki_datasource", + "options": [], + "query": "loki", + "refresh": 1, + "regex": "(?!grafanacloud.+usage-insights|grafanacloud.+alert-state-history).+", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "default", + "title": "Linux node / overview", + "uid": "nodes", + "version": 1, + "weekStart": "" +} diff --git a/ansible/roles/core-mon/monitoring/files/dashboards/vm-remnawave-dashboard.json b/ansible/roles/core-mon/monitoring/files/dashboards/vm-remnawave-dashboard.json new file mode 100644 index 0000000..ea000aa --- /dev/null +++ b/ansible/roles/core-mon/monitoring/files/dashboards/vm-remnawave-dashboard.json @@ -0,0 +1,2479 @@ +{ + "__inputs": [ + { + "name": "DS_VICTORIAMETRICS", + "label": "victoriametrics", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "panel", + "id": "bargauge", + "name": "Bar gauge", + "version": "" + }, + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "12.2.0-17027759091" + }, + { + "type": "panel", + "id": "piechart", + "name": "Pie chart", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "🌍 Общий обзор системы", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.2.0-17027759091", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(remnawave_users_total)", + "refId": "A" + } + ], + "title": "👥 Всего пользователей", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "#EAB839", + "value": 500 + }, + { + "color": "semi-dark-orange", + "value": 1000 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.2.0-17027759091", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(remnawave_users_online_stats{metricType=\"onlineNow\"})", + "refId": "A" + } + ], + "title": "🟢 Пользователей онлайн", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "index": 0, + "text": "Отключен" + }, + "1": { + "color": "green", + "index": 1, + "text": "Активен" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": 0 + }, + { + "color": "green", + "value": 1 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 1 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.2.0-17027759091", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "count(remnawave_node_status == 1)", + "refId": "A" + } + ], + "title": "🖥️ Активных узлов", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "#EAB839", + "value": 70 + }, + { + "color": "red", + "value": 85 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 1 + }, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.2.0-17027759091", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "avg(remnawave_nodejs_cpu_usage_percent)", + "refId": "A" + } + ], + "title": "💻 CPU загрузка", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "#EAB839", + "value": 70 + }, + { + "color": "red", + "value": 85 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 1 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.2.0-17027759091", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "avg(remnawave_nodejs_heap_usage_percent)", + "refId": "A" + } + ], + "title": "🧠 Heap память", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "#EAB839", + "value": 500 + }, + { + "color": "red", + "value": 1000 + } + ] + }, + "unit": "reqpm" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 1 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.2.0-17027759091", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "avg(remnawave_nodejs_http_req_rate)", + "refId": "A" + } + ], + "title": "📊 Запросов/мин", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 8, + "panels": [], + "title": "👥 Статистика пользователей", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "ACTIVE" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "DISABLED" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "LIMITED" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "EXPIRED" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 6 + }, + "id": 9, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "sort": "desc", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.2.0-17027759091", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "remnawave_users_status", + "legendFormat": "{{status}}", + "refId": "A" + } + ], + "title": "📊 Статусы пользователей", + "type": "piechart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "mappings": [ + { + "options": { + "lastDay": { + "color": "super-light-green", + "index": 0, + "text": "Вчера" + }, + "lastWeek": { + "color": "dark-green", + "index": 1, + "text": "На прошлой неделе" + }, + "neverOnline": { + "color": "red", + "index": 2, + "text": "Не были в сети" + }, + "onlineNow": { + "color": "green", + "index": 3, + "text": "Онлайн сейчас" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 10, + "x": 6, + "y": 6 + }, + "id": 10, + "options": { + "displayMode": "basic", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "12.2.0-17027759091", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "remnawave_users_online_stats", + "legendFormat": "{{metricType}}", + "refId": "A" + } + ], + "title": "📈 Активность пользователей", + "transformations": [ + { + "id": "reduce", + "options": { + "labelsToFields": false, + "reducers": [ + "lastNotNull" + ] + } + } + ], + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 6 + }, + "id": 11, + "options": { + "legend": { + "calcs": [ + "mean", + "last" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.0-17027759091", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "remnawave_users_online_stats{metricType=\"onlineNow\"}", + "legendFormat": "Онлайн сейчас", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "remnawave_users_total", + "legendFormat": "Всего пользователей", + "refId": "B" + } + ], + "title": "📊 Динамика пользователей", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 12, + "panels": [], + "title": "🌍 Статистика узлов", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Страна" + }, + "properties": [ + { + "id": "custom.width", + "value": 80 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Статус" + }, + "properties": [ + { + "id": "custom.width", + "value": 100 + }, + { + "id": "mappings", + "value": [ + { + "options": { + "0": { + "color": "red", + "index": 0, + "text": "❌ Offline" + }, + "1": { + "color": "green", + "index": 1, + "text": "✅ Online" + } + }, + "type": "value" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Пользователей" + }, + "properties": [ + { + "id": "custom.width", + "value": 120 + }, + { + "id": "color", + "value": { + "mode": "continuous-GrYlRd" + } + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "color-background" + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 10, + "x": 0, + "y": 15 + }, + "id": 13, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Статус" + } + ] + }, + "pluginVersion": "12.2.0-17027759091", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "max by (node_uuid, node_name, node_country_emoji) (remnawave_node_online_users)", + "format": "table", + "instant": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "max by (node_uuid, node_name, node_country_emoji) (remnawave_node_status)", + "format": "table", + "instant": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "group by (node_uuid) (topk by (node_uuid) (1, remnawave_node_online_users{provider_name!=\"unknown\"}))", + "format": "table", + "instant": true, + "refId": "C" + } + ], + "title": "🖥️ Статус узлов", + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value #C": true, + "__name__": true, + "app": true, + "job": true, + "node_uuid": true + }, + "indexByName": { + "Value #A": 3, + "Value #B": 2, + "node_country_emoji": 0, + "node_name": 1, + "provider_name": 4 + }, + "renameByName": { + "Value #A": "Пользователей", + "Value #B": "Статус", + "node_country_emoji": "Страна", + "node_name": "Имя узла", + "provider_name": "Провайдер" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "right", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 10, + "y": 15 + }, + "id": 25, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.2.0-17027759091", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "xray_proxy_latency_ms", + "legendFormat": "{{name}}", + "range": true, + "refId": "A" + } + ], + "title": "XRAY LATENCY", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "dark-red", + "index": 0, + "text": "❌ Offline" + }, + "1": { + "color": "dark-green", + "index": 1, + "text": "✅ Online" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 0 + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 18, + "y": 15 + }, + "id": 26, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": { + "percentSize": 1 + }, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "12.2.0-17027759091", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "exemplar": false, + "expr": "xray_proxy_status", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{name}}", + "range": true, + "refId": "A" + } + ], + "title": "XRAY STATUS", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 14, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.2.0-17027759091", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "remnawave_node_online_users", + "legendFormat": "{{node_country_emoji}} {{node_name}}", + "refId": "A" + } + ], + "title": "📈 Онлайн пользователи по узлам", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 15, + "panels": [], + "title": "📊 Сетевой трафик", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 35 + }, + "id": 16, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.2.0-17027759091", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(remnawave_node_inbound_upload_bytes[5m])) by (node_name, node_country_emoji)", + "legendFormat": "{{node_country_emoji}} {{node_name}} Upload", + "refId": "A" + } + ], + "title": "⬆️ Входящий трафик (Upload)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 35 + }, + "id": 17, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.2.0-17027759091", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(remnawave_node_inbound_download_bytes[5m])) by (node_name, node_country_emoji)", + "legendFormat": "{{node_country_emoji}} {{node_name}} Download", + "refId": "A" + } + ], + "title": "⬇️ Входящий трафик (Download)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 13, + "w": 12, + "x": 0, + "y": 43 + }, + "id": 18, + "options": { + "displayMode": "gradient", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 10, + "minVizWidth": 0, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "showUnfilled": true, + "sizing": "auto", + "text": {}, + "valueMode": "color" + }, + "pluginVersion": "12.2.0-17027759091", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (tag) (\n increase(remnawave_node_inbound_upload_bytes[$__range])\n +\n increase(remnawave_node_inbound_download_bytes[$__range])\n)", + "legendFormat": "{{tag}}", + "range": true, + "refId": "A" + } + ], + "title": "🔐 Трафик по инбаундам", + "transformations": [ + { + "id": "reduce", + "options": { + "labelsToFields": false, + "reducers": [ + "lastNotNull" + ] + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "desc": true, + "field": "Last *" + } + ] + } + } + ], + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 13, + "w": 12, + "x": 12, + "y": 43 + }, + "id": 19, + "options": { + "displayMode": "basic", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 10, + "minVizWidth": 0, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "showUnfilled": true, + "sizing": "auto", + "text": {}, + "valueMode": "color" + }, + "pluginVersion": "12.2.0-17027759091", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "topk(100,\n sum by (node_name, node_country_emoji) (\n increase(remnawave_node_inbound_upload_bytes[$__range])\n )\n +\n sum by (node_name, node_country_emoji) (\n increase(remnawave_node_inbound_download_bytes[$__range])\n )\n +\n sum by (node_name, node_country_emoji) (\n increase(remnawave_node_outbound_upload_bytes[$__range])\n )\n +\n sum by (node_name, node_country_emoji) (\n increase(remnawave_node_outbound_download_bytes[$__range])\n )\n)", + "legendFormat": "{{node_country_emoji}} {{node_name}}", + "range": true, + "refId": "A" + } + ], + "title": "🏆 Топ нод по трафику", + "transformations": [ + { + "id": "reduce", + "options": { + "labelsToFields": false, + "reducers": [ + "lastNotNull" + ] + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "desc": true, + "field": "Last *" + } + ] + } + } + ], + "type": "bargauge" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 56 + }, + "id": 20, + "panels": [], + "title": "⚡ Производительность Node.js", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 57 + }, + "id": 21, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.0-17027759091", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "remnawave_nodejs_cpu_usage_percent{instance_name=~\"$instance\"}", + "legendFormat": "{{instance_name}} - CPU", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "remnawave_nodejs_heap_usage_percent{instance_name=~\"$instance\"}", + "legendFormat": "{{instance_name}} - Heap", + "refId": "B" + } + ], + "title": "💻 CPU и Heap использование", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 1000 + } + ] + }, + "unit": "ms" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "HTTP Latency p95" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "HTTP Latency p50" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Event Loop p95" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Event Loop p50" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 57 + }, + "id": 22, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.0-17027759091", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "remnawave_nodejs_http_req_latency_p95{instance_name=~\"$instance\"}", + "legendFormat": "{{instance_name}} - HTTP p95", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "remnawave_nodejs_http_req_latency_p50{instance_name=~\"$instance\"}", + "legendFormat": "{{instance_name}} - HTTP p50", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "remnawave_nodejs_event_loop_latency_p95{instance_name=~\"$instance\"}", + "legendFormat": "{{instance_name}} - Event Loop p95", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "remnawave_nodejs_event_loop_latency_p50{instance_name=~\"$instance\"}", + "legendFormat": "{{instance_name}} - Event Loop p50", + "refId": "D" + } + ], + "title": "⏱️ Latency метрики", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Used Heap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total Heap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Memory Usage" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 65 + }, + "id": 23, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.0-17027759091", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "remnawave_nodejs_heap_used_bytes{instance_name=~\"$instance\"}", + "legendFormat": "{{instance_name}} - Used Heap", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "remnawave_nodejs_heap_total_bytes{instance_name=~\"$instance\"}", + "legendFormat": "{{instance_name}} - Total Heap", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "remnawave_nodejs_memory_usage_bytes{instance_name=~\"$instance\"}", + "legendFormat": "{{instance_name}} - Memory", + "refId": "C" + } + ], + "title": "🧠 Использование памяти", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "HTTP Requests/min" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "left" + }, + { + "id": "unit", + "value": "reqpm" + }, + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Active Handlers" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Active Requests" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 65 + }, + "id": 24, + "options": { + "legend": { + "calcs": [ + "mean", + "last" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.0-17027759091", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "remnawave_nodejs_http_req_rate{instance_name=~\"$instance\"}", + "legendFormat": "{{instance_name}} - Requests/min", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "remnawave_nodejs_active_handlers{instance_name=~\"$instance\"}", + "legendFormat": "{{instance_name}} - Handlers", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "remnawave_nodejs_active_requests{instance_name=~\"$instance\"}", + "legendFormat": "{{instance_name}} - Requests", + "refId": "C" + } + ], + "title": "📊 Активность приложения", + "type": "timeseries" + } + ], + "refresh": "30s", + "schemaVersion": 41, + "tags": [ + "remnawave", + "prometheus", + "monitoring" + ], + "templating": { + "list": [ + { + "current": { + "text": "VictoriaMetrics", + "value": "victoriametrics", + "selected": true + }, + "includeAll": false, + "label": "Data Source", + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(remnawave_nodejs_cpu_usage_percent, instance_name)", + "includeAll": true, + "label": "Node.js Instance", + "multi": true, + "name": "instance", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(remnawave_nodejs_cpu_usage_percent, instance_name)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Remnawave Monitoring Dashboard", + "uid": "remnawave-monitoring", + "version": 1, + "weekStart": "" +} diff --git a/ansible/roles/core-mon/monitoring/tasks/main.yml b/ansible/roles/core-mon/monitoring/tasks/main.yml index eab771f..2cfd1fa 100644 --- a/ansible/roles/core-mon/monitoring/tasks/main.yml +++ b/ansible/roles/core-mon/monitoring/tasks/main.yml @@ -19,6 +19,9 @@ - { path: "{{ monitoring_dir }}/grafana/provisioning/dashboards", owner: "root", group: "root", mode: "0755" } - { path: "{{ monitoring_dir }}/ntfy_data", owner: "root", group: "root", mode: "0777" } - { path: "{{ monitoring_dir }}/alertmanager-ntfy", owner: "root", group: "root", mode: "0755" } + - { path: "{{ monitoring_dir }}/victoria-metrics-data", owner: "65534", group: "65534", mode: "0775" } + - { path: "{{ monitoring_dir }}/vmagent", owner: "root", group: "root", mode: "0755" } + - { path: "{{ monitoring_dir }}/vmagent/conf.d", owner: "root", group: "root", mode: "0755" } - name: Сбросить список целей scrape node-exporter ansible.builtin.set_fact: @@ -82,13 +85,14 @@ group: root mode: "0644" -- name: Установить источник данных Prometheus в Grafana +- name: Установить источники данных Grafana ansible.builtin.template: src: grafana-datasources.yml.j2 dest: "{{ monitoring_dir }}/grafana/provisioning/datasources/prometheus.yml" owner: root group: root mode: "0644" + register: grafana_datasources - name: Установить провайдер дашбордов Grafana ansible.builtin.template: @@ -106,6 +110,51 @@ group: root mode: "0644" +- name: Конфиг vmagent на core-mon + ansible.builtin.template: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + owner: root + group: root + mode: "0644" + loop: + - { src: vmagent-scrape.yml.j2, dest: "{{ monitoring_dir }}/vmagent/scrape.yml" } + - { src: vmagent-nodeexporter.yml.j2, dest: "{{ monitoring_dir }}/vmagent/conf.d/nodeexporter.yml" } + - { src: vmagent-cadvisor.yml.j2, dest: "{{ monitoring_dir }}/vmagent/conf.d/cadvisor.yml" } + +- name: Убрать scrape Xray Checker без ссылки подписки + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: + - "{{ monitoring_dir }}/vmagent/conf.d/xraychecker.yml" + - "{{ monitoring_dir }}/xray-checker.env" + when: xray_checker_subscription_url | length == 0 + +- name: Ссылки подписки для Xray Checker нет + ansible.builtin.debug: + msg: "Ссылки Xray Checker нет, контейнер не запускается." + when: xray_checker_subscription_url | length == 0 + +- name: Файл окружения Xray Checker + ansible.builtin.template: + src: xray-checker.env.j2 + dest: "{{ monitoring_dir }}/xray-checker.env" + owner: root + group: root + mode: "0600" + no_log: true + when: xray_checker_subscription_url | length > 0 + +- name: Scrape Xray Checker + ansible.builtin.template: + src: vmagent-xray.yml.j2 + dest: "{{ monitoring_dir }}/vmagent/conf.d/xraychecker.yml" + owner: root + group: root + mode: "0644" + when: xray_checker_subscription_url | length > 0 + - name: Установить compose.yaml для стека мониторинга ansible.builtin.template: src: compose.yaml.j2 @@ -151,6 +200,20 @@ failed_when: false no_log: true +- name: Дождаться VictoriaMetrics + ansible.builtin.uri: + url: "http://{{ ansible_host }}:8428/health" + status_code: 200 + register: victoriametrics_health + retries: 20 + delay: 2 + until: victoriametrics_health.status == 200 + +- name: Перезапустить Grafana, чтобы подхватить datasource + ansible.builtin.command: docker compose -f {{ monitoring_dir }}/compose.yaml restart grafana + when: grafana_datasources is changed + changed_when: true + - name: Дождаться готовности Grafana ansible.builtin.uri: url: "http://127.0.0.1:3000/api/health" diff --git a/ansible/roles/core-mon/monitoring/templates/compose.yaml.j2 b/ansible/roles/core-mon/monitoring/templates/compose.yaml.j2 index c64af36..ff96675 100644 --- a/ansible/roles/core-mon/monitoring/templates/compose.yaml.j2 +++ b/ansible/roles/core-mon/monitoring/templates/compose.yaml.j2 @@ -78,4 +78,64 @@ services: depends_on: - prometheus + vmsingle: + image: {{ victoriametrics_image }} + container_name: monitoring-vmsingle + hostname: vmsingle + restart: unless-stopped + command: + - --storageDataPath=/victoria-metrics-data + volumes: + - ./victoria-metrics-data:/victoria-metrics-data + ports: + - "{{ ansible_host }}:8428:8428" + + cadvisor: + image: {{ cadvisor_image }} + container_name: monitoring-cadvisor + restart: unless-stopped + privileged: true + command: + - --listen_ip=0.0.0.0 + - --port=9101 + - --docker_only=true + volumes: + - /:/rootfs:ro + - /var/run:/var/run:ro + - /sys:/sys:ro + - /var/lib/docker:/var/lib/docker:ro + - /dev/disk:/dev/disk:ro + ports: + - "127.0.0.1:9101:9101" +{% if xray_checker_subscription_url | length > 0 %} + + xray-checker: + image: {{ xray_checker_image }} + container_name: monitoring-xray-checker + hostname: xraychecker + restart: unless-stopped + env_file: + - ./xray-checker.env + ports: + - "127.0.0.1:2112:2112" +{% endif %} + + vmagent: + image: {{ vmagent_image }} + container_name: monitoring-vmagent + restart: unless-stopped + command: + - -httpListenAddr=0.0.0.0:8429 + - -promscrape.config=/etc/vmagent/scrape.yml + - -promscrape.configCheckInterval=60s + - -remoteWrite.url=http://vmsingle:8428/api/v1/write + volumes: + - ./vmagent/scrape.yml:/etc/vmagent/scrape.yml:ro + - ./vmagent/conf.d:/etc/vmagent/conf.d:ro + ports: + - "127.0.0.1:8429:8429" + depends_on: + - vmsingle + - cadvisor + diff --git a/ansible/roles/core-mon/monitoring/templates/grafana-datasources.yml.j2 b/ansible/roles/core-mon/monitoring/templates/grafana-datasources.yml.j2 index 74c2341..2d16332 100644 --- a/ansible/roles/core-mon/monitoring/templates/grafana-datasources.yml.j2 +++ b/ansible/roles/core-mon/monitoring/templates/grafana-datasources.yml.j2 @@ -10,3 +10,12 @@ datasources: jsonData: timeInterval: 15s + - name: VictoriaMetrics + uid: victoriametrics + type: prometheus + access: proxy + url: http://vmsingle:8428 + isDefault: false + jsonData: + timeInterval: 15s + diff --git a/ansible/roles/core-mon/monitoring/templates/vmagent-cadvisor.yml.j2 b/ansible/roles/core-mon/monitoring/templates/vmagent-cadvisor.yml.j2 new file mode 100644 index 0000000..25ce6fd --- /dev/null +++ b/ansible/roles/core-mon/monitoring/templates/vmagent-cadvisor.yml.j2 @@ -0,0 +1,7 @@ +- job_name: integrations/cAdvisor + scrape_interval: 15s + static_configs: + - targets: ["cadvisor:9101"] + labels: + instance: "{{ inventory_hostname }}" + cluster: "{{ vmagent_cluster }}" diff --git a/ansible/roles/core-mon/monitoring/templates/vmagent-nodeexporter.yml.j2 b/ansible/roles/core-mon/monitoring/templates/vmagent-nodeexporter.yml.j2 new file mode 100644 index 0000000..2001b92 --- /dev/null +++ b/ansible/roles/core-mon/monitoring/templates/vmagent-nodeexporter.yml.j2 @@ -0,0 +1,7 @@ +- job_name: integrations/node_exporter + scrape_interval: 15s + static_configs: + - targets: ["{{ ansible_host }}:9100"] + labels: + instance: "{{ inventory_hostname }}" + cluster: "{{ vmagent_cluster }}" diff --git a/ansible/roles/core-mon/monitoring/templates/vmagent-scrape.yml.j2 b/ansible/roles/core-mon/monitoring/templates/vmagent-scrape.yml.j2 new file mode 100644 index 0000000..81638d8 --- /dev/null +++ b/ansible/roles/core-mon/monitoring/templates/vmagent-scrape.yml.j2 @@ -0,0 +1,5 @@ +scrape_config_files: + - "/etc/vmagent/conf.d/*.yml" + +global: + scrape_interval: 15s diff --git a/ansible/roles/core-mon/monitoring/templates/vmagent-xray.yml.j2 b/ansible/roles/core-mon/monitoring/templates/vmagent-xray.yml.j2 new file mode 100644 index 0000000..62e68a6 --- /dev/null +++ b/ansible/roles/core-mon/monitoring/templates/vmagent-xray.yml.j2 @@ -0,0 +1,7 @@ +- job_name: xray_checker + scrape_interval: 15s + static_configs: + - targets: ["xray-checker:2112"] + labels: + instance: "{{ inventory_hostname }}" + cluster: "{{ vmagent_cluster }}" diff --git a/ansible/roles/core-mon/monitoring/templates/xray-checker.env.j2 b/ansible/roles/core-mon/monitoring/templates/xray-checker.env.j2 new file mode 100644 index 0000000..5b3d65c --- /dev/null +++ b/ansible/roles/core-mon/monitoring/templates/xray-checker.env.j2 @@ -0,0 +1,3 @@ +SUBSCRIPTION_URL={{ xray_checker_subscription_url }} +PROXY_STATUS_CHECK_URL=http://google.com/generate_204 +PROXY_CHECK_INTERVAL=60 diff --git a/ansible/roles/core-net/netbird/tasks/main.yml b/ansible/roles/core-net/netbird/tasks/main.yml new file mode 100644 index 0000000..adcb9ea --- /dev/null +++ b/ansible/roles/core-net/netbird/tasks/main.yml @@ -0,0 +1,4 @@ +--- +- name: Клиент NetBird на панели + ansible.builtin.include_role: + name: netbird_client diff --git a/ansible/roles/core-net/remnawave/defaults/main.yml b/ansible/roles/core-net/remnawave/defaults/main.yml index 7bd932f..0fe6878 100644 --- a/ansible/roles/core-net/remnawave/defaults/main.yml +++ b/ansible/roles/core-net/remnawave/defaults/main.yml @@ -29,6 +29,9 @@ remnawave_api_token: "{{ lookup('env', 'REMNAWAVE_API_TOKEN', default='') }}" # Нода Ger-nod-02 / SelfSteal — создаётся через API панели remnawave_node_name: "Ger-nod-02" remnawave_node_address: "150.241.64.209" +remnawave_node_netbird_hostname: "ger-nod-02" +remnawave_netbird_peers_url: "https://central.gunihome.ru/api/peers" +remnawave_xray_checker_user: "homelab" remnawave_node_port: 2222 remnawave_node_country: "DE" remnawave_selfsteal_domain: "yali.gunis.top" diff --git a/ansible/roles/core-net/remnawave/tasks/main.yml b/ansible/roles/core-net/remnawave/tasks/main.yml index b5af694..10e2d41 100644 --- a/ansible/roles/core-net/remnawave/tasks/main.yml +++ b/ansible/roles/core-net/remnawave/tasks/main.yml @@ -186,6 +186,37 @@ group: root mode: "0600" +- name: Узнать NetBird IP ноды + ansible.builtin.shell: | + set -euo pipefail + if [ -z "${NB_API_TOKEN:-}" ]; then + exit 0 + fi + curl -fsS --max-time 20 -H "Authorization: Token ${NB_API_TOKEN}" \ + "{{ remnawave_netbird_peers_url }}" \ + | jq -r --arg h "$NB_HOST" ' + (if type == "array" then . elif type == "object" then (.items // .peers // []) else [] end) + | map(select((.hostname // "") == $h or (.name // "") == $h or (.dns_label // "") == $h)) + | .[0].ip // empty' + args: + executable: /bin/bash + environment: + NB_API_TOKEN: "{{ lookup('env', 'NETBIRD_API_TOKEN', default='') }}" + NB_HOST: "{{ remnawave_node_netbird_hostname }}" + register: remnawave_node_nb + changed_when: false + failed_when: false + +- name: Подставить NetBird-адрес ноды + ansible.builtin.set_fact: + remnawave_node_address: "{{ remnawave_node_nb.stdout | trim }}" + when: remnawave_node_nb.stdout | default('') | trim is match('^[0-9.]+$') + +- name: Peer ноды ещё не в NetBird + ansible.builtin.debug: + msg: "Peer {{ remnawave_node_netbird_hostname }} ещё не в сети, адрес ноды остаётся {{ remnawave_node_address }}" + when: remnawave_node_nb.stdout | default('') | trim is not match('^[0-9.]+$') + - name: Инициализировать админа, API-токен, профиль DE-Yali, ноду и хост ansible.builtin.shell: | set -euo pipefail @@ -203,6 +234,8 @@ RW_NODE_NAME: "{{ remnawave_node_name }}" RW_NODE_ADDR: "{{ remnawave_node_address }}" RW_NODE_PORT: "{{ remnawave_node_port }}" + RW_SUB_DOMAIN: "{{ remnawave_sub_domain }}" + RW_XRAY_USER: "{{ remnawave_xray_checker_user }}" RW_NODE_CC: "{{ remnawave_node_country }}" RW_SELFSTEAL: "{{ remnawave_selfsteal_domain }}" RW_XHTTP_PATH: "{{ remnawave_xhttp_path }}" @@ -231,6 +264,7 @@ ansible.builtin.set_fact: remnawave_api_token: "{{ (remnawave_bootstrap_secrets.content | b64decode | from_json).api_token }}" remnawave_node_secret: "{{ (remnawave_bootstrap_secrets.content | b64decode | from_json).node_secret }}" + remnawave_xray_checker_url: "{{ (remnawave_bootstrap_secrets.content | b64decode | from_json).xray_checker_subscription_url | default('') }}" no_log: true - name: Проверить, что SECRET_KEY ноды получен @@ -250,14 +284,20 @@ - name: Сохранить API-токен и SECRET_KEY ноды в Infisical ansible.builtin.shell: | set -euo pipefail + args=( + "REMNAWAVE_API_TOKEN=${RW_API_TOKEN}" + "REMNANODE_SECRET_KEY=${RW_NODE_SECRET}" + ) + if [ -n "${RW_XRAY_URL:-}" ]; then + args+=("XRAY_CHECKER_SUBSCRIPTION_URL=${RW_XRAY_URL}") + fi infisical secrets set \ --domain="${INFISICAL_DOMAIN}" \ --token="${INFISICAL_TOKEN}" \ --env=prod \ --projectId="${INFISICAL_PROJECT_ID}" \ --path="/services/remnawave" \ - "REMNAWAVE_API_TOKEN=${RW_API_TOKEN}" \ - "REMNANODE_SECRET_KEY=${RW_NODE_SECRET}" \ + "${args[@]}" \ --silent >/dev/null args: executable: /bin/bash @@ -269,6 +309,7 @@ INFISICAL_PROJECT_ID: "{{ lookup('env', 'INFISICAL_PROJECT_ID') }}" RW_API_TOKEN: "{{ remnawave_api_token }}" RW_NODE_SECRET: "{{ remnawave_node_secret | default(lookup('env', 'REMNANODE_SECRET_KEY', default='')) }}" + RW_XRAY_URL: "{{ remnawave_xray_checker_url | default('') }}" when: remnawave_api_token | default('') | length > 0 changed_when: true no_log: true @@ -321,5 +362,6 @@ - "Админ создан в этом прогоне: {{ remnawave_bootstrap.admin_created | default(false) }}" - "API-токен обновлён в Infisical: {{ remnawave_bootstrap.token_updated | default(false) }}" - "Профиль {{ remnawave_profile_name }}: {{ remnawave_bootstrap.profile_uuid | default('n/a') }}" - - "Нода {{ remnawave_node_name }}: {{ remnawave_bootstrap.node_uuid | default('n/a') }}" + - "Нода {{ remnawave_node_name }}: {{ remnawave_bootstrap.node_uuid | default('n/a') }} адрес {{ remnawave_bootstrap.node_address | default(remnawave_node_address) }}" + - "Ссылка Xray Checker записана в Infisical: {{ remnawave_bootstrap.xray_checker_url_saved | default(false) }}" - "Host {{ remnawave_selfsteal_domain }}: {{ remnawave_bootstrap.host_uuid | default('n/a') }}" diff --git a/ansible/roles/core-net/remnawave/templates/bootstrap-panel.sh.j2 b/ansible/roles/core-net/remnawave/templates/bootstrap-panel.sh.j2 index 985ed11..3f2ea86 100644 --- a/ansible/roles/core-net/remnawave/templates/bootstrap-panel.sh.j2 +++ b/ansible/roles/core-net/remnawave/templates/bootstrap-panel.sh.j2 @@ -10,6 +10,7 @@ admin_created=0 token_updated=0 profile_created=0 node_created=0 +node_address_updated=0 host_created=0 node_secret="" active_token="${RW_API_TOKEN:-}" @@ -20,7 +21,7 @@ redact_json() { jq 'walk( if type == "object" then with_entries( - if (.key | test("(?i)privateKey|privKey|secretKey|SECRET_KEY|accessToken|apiToken|password|^token$|vlessUuid|trojanPassword|ssPassword|shortUuid|subscriptionUrl")) + if (.key | test("(?i)privateKey|privKey|secretKey|SECRET_KEY|accessToken|apiToken|password|^token$|vlessUuid|trojanPassword|ssPassword|shortUuid|subscriptionUrl|setupKey|setup_key|METRICS_PASS")) then .value = "[REDACTED]" else . end @@ -223,20 +224,29 @@ if [ -z "${node_uuid}" ]; then node_created=1 fi +current_addr=$(printf '%s' "${nodes_json}" | jq -r --arg n "$RW_NODE_NAME" '(.response // []) | map(select(.name == $n)) | .[0].address // empty') +if [ -n "${RW_NODE_ADDR}" ] && [ "${current_addr}" != "${RW_NODE_ADDR}" ]; then + node_address_updated=1 +fi node_update=$(jq -n \ --arg u "$node_uuid" \ --arg pu "$profile_uuid" \ --arg iu "$inbound_uuid" \ + --arg addr "$RW_NODE_ADDR" \ + --argjson port "$RW_NODE_PORT" \ '{ uuid: $u, + address: $addr, + port: $port, configProfile: { activeConfigProfileUuid: $pu, activeInbounds: [$iu] } }') -api PATCH /nodes/ -d "${node_update}" >/dev/null 2>&1 \ - || api PUT /nodes/ -d "${node_update}" >/dev/null 2>&1 \ - || true +if ! api PATCH /nodes/ -d "${node_update}" >/dev/null \ + && ! api PUT /nodes/ -d "${node_update}" >/dev/null; then + echo "WARN: не удалось обновить адрес ноды ${RW_NODE_NAME}" >&2 +fi hosts_json=$(api GET /hosts/) host_uuid=$(printf '%s' "${hosts_json}" | jq -r --arg a "$RW_SELFSTEAL" ' @@ -420,11 +430,33 @@ if [ -n "${RW_USERS_FILE:-}" ] && [ -f "${RW_USERS_FILE}" ]; then done < <(printf '%s' "${existing}" | jq -r '.response.users[]? | "\(.username)\t\(.uuid)"') fi +xray_url="" +xray_user="${RW_XRAY_USER:-homelab}" +if [ -n "${xray_user}" ]; then + users_now=$(api GET "/users/?size=500" || api GET /users/ || echo '{}') + xray_url=$(printf '%s' "${users_now}" | jq -r --arg u "${xray_user}" ' + .response.users[]? | select(.username == $u) | .subscriptionUrl // empty' | head -1) + if [ -z "${xray_url}" ]; then + short=$(printf '%s' "${users_now}" | jq -r --arg u "${xray_user}" ' + .response.users[]? | select(.username == $u) | .shortUuid // empty' | head -1) + if [ -n "${short}" ] && [ -n "${RW_SUB_DOMAIN:-}" ]; then + xray_url="https://${RW_SUB_DOMAIN}/${short}" + fi + fi +fi +xray_saved=0 +if [ -n "${xray_url}" ]; then + xray_saved=1 +fi + # Секреты только в файл 0600. В stdout — статус без значений ключей. secrets_file="${RW_SECRETS_FILE:-/opt/remnawave/.bootstrap-secrets.json}" umask 077 -jq -n --arg api_token "${active_token}" --arg node_secret "${node_secret}" \ - '{api_token: $api_token, node_secret: $node_secret}' > "${secrets_file}" +jq -n \ + --arg api_token "${active_token}" \ + --arg node_secret "${node_secret}" \ + --arg xray_url "${xray_url}" \ + '{api_token: $api_token, node_secret: $node_secret, xray_checker_subscription_url: $xray_url}' > "${secrets_file}" chmod 600 "${secrets_file}" jq -n \ @@ -432,22 +464,28 @@ jq -n \ --arg node_uuid "${node_uuid}" \ --arg host_uuid "${host_uuid}" \ --arg reality_pub "${pub}" \ + --arg node_address "${RW_NODE_ADDR}" \ --argjson admin_created "${admin_created}" \ --argjson token_updated "${token_updated}" \ --argjson profile_created "${profile_created}" \ --argjson node_created "${node_created}" \ + --argjson node_address_updated "${node_address_updated}" \ --argjson host_created "${host_created}" \ --argjson users_changed "${users_changed}" \ + --argjson xray_saved "${xray_saved}" \ '{ profile_uuid: $profile_uuid, node_uuid: $node_uuid, host_uuid: $host_uuid, + node_address: $node_address, reality_public_key: $reality_pub, admin_created: ($admin_created == 1), token_updated: ($token_updated == 1), profile_created: ($profile_created == 1), node_created: ($node_created == 1), + node_address_updated: ($node_address_updated == 1), host_created: ($host_created == 1), users_changed: ($users_changed == 1), - changed: (($admin_created + $token_updated + $profile_created + $node_created + $host_created + $users_changed) > 0) + xray_checker_url_saved: ($xray_saved == 1), + changed: (($admin_created + $token_updated + $profile_created + $node_created + $node_address_updated + $host_created + $users_changed + $xray_saved) > 0) }' diff --git a/ansible/roles/core-net/vmagent/tasks/main.yml b/ansible/roles/core-net/vmagent/tasks/main.yml new file mode 100644 index 0000000..909621d --- /dev/null +++ b/ansible/roles/core-net/vmagent/tasks/main.yml @@ -0,0 +1,6 @@ +--- +- name: vmagent панели + ansible.builtin.include_role: + name: vmagent + vars: + vmagent_scrape_remnawave: true diff --git a/ansible/roles/ger-nod-02/netbird/tasks/main.yml b/ansible/roles/ger-nod-02/netbird/tasks/main.yml new file mode 100644 index 0000000..e83c6f7 --- /dev/null +++ b/ansible/roles/ger-nod-02/netbird/tasks/main.yml @@ -0,0 +1,4 @@ +--- +- name: Клиент NetBird на ноде + ansible.builtin.include_role: + name: netbird_client diff --git a/ansible/roles/ger-nod-02/remnanode/tasks/main.yml b/ansible/roles/ger-nod-02/remnanode/tasks/main.yml index 6ec967a..3338b8e 100644 --- a/ansible/roles/ger-nod-02/remnanode/tasks/main.yml +++ b/ansible/roles/ger-nod-02/remnanode/tasks/main.yml @@ -178,12 +178,49 @@ register: ufw_https changed_when: "'added' in (ufw_https.stdout | lower)" +- name: Узнать NetBird IP панели + ansible.builtin.shell: | + set -euo pipefail + if [ -z "${NB_API_TOKEN:-}" ]; then + exit 0 + fi + curl -fsS --max-time 20 -H "Authorization: Token ${NB_API_TOKEN}" \ + "https://central.gunihome.ru/api/peers" \ + | jq -r --arg h "core-net" ' + (if type == "array" then . elif type == "object" then (.items // .peers // []) else [] end) + | map(select((.hostname // "") == $h or (.name // "") == $h or (.dns_label // "") == $h)) + | .[0].ip // empty' + args: + executable: /bin/bash + environment: + NB_API_TOKEN: "{{ lookup('env', 'NETBIRD_API_TOKEN', default='') }}" + register: remnanode_panel_nb + changed_when: false + failed_when: false + +- name: Разрешить NODE_PORT с NetBird IP панели + ansible.builtin.command: > + ufw allow from {{ remnanode_panel_nb.stdout | trim }} to any port {{ remnanode_node_port }} proto tcp + register: ufw_node_nb + changed_when: "'added' in (ufw_node_nb.stdout | lower)" + when: remnanode_panel_nb.stdout | default('') | trim is match('^[0-9.]+$') + +- name: Убрать WAN-доступ к NODE_PORT + ansible.builtin.command: > + ufw --force delete allow from {{ item }} to any port {{ remnanode_node_port }} proto tcp + loop: "{{ remnanode_panel_egress_ips }}" + register: ufw_node_wan_del + changed_when: "'deleted' in (stdout | default('') | lower)" + failed_when: false + when: remnanode_panel_nb.stdout | default('') | trim is match('^[0-9.]+$') + - name: Разрешить NODE_PORT только с egress панели (оба провайдера) ansible.builtin.command: > ufw allow from {{ item }} to any port {{ remnanode_node_port }} proto tcp loop: "{{ remnanode_panel_egress_ips }}" register: ufw_node - changed_when: "'added' in (ufw_node.stdout | lower)" + changed_when: "'added' in (stdout | default('') | lower)" + when: remnanode_panel_nb.stdout | default('') | trim is not match('^[0-9.]+$') - name: Включить firewall ansible.builtin.command: ufw --force enable @@ -238,5 +275,10 @@ ansible.builtin.debug: msg: - "SelfSteal: https://{{ remnanode_selfsteal_domain }}" - - "NODE_PORT {{ remnanode_node_port }} только с {{ remnanode_panel_egress_ips | join(', ') }}" + - >- + {{ + 'NODE_PORT ' ~ remnanode_node_port ~ ' только с NetBird панели ' ~ (remnanode_panel_nb.stdout | trim) + if (remnanode_panel_nb.stdout | default('') | trim is match('^[0-9.]+$')) + else 'Peer core-net ещё не в сети, NODE_PORT ' ~ remnanode_node_port ~ ' остаётся с WAN egress ' ~ (remnanode_panel_egress_ips | join(', ')) + }} - "Compose: {{ remnanode_dir }}/compose.yaml" diff --git a/ansible/roles/ger-nod-02/vmagent/tasks/main.yml b/ansible/roles/ger-nod-02/vmagent/tasks/main.yml new file mode 100644 index 0000000..19b5be7 --- /dev/null +++ b/ansible/roles/ger-nod-02/vmagent/tasks/main.yml @@ -0,0 +1,7 @@ +--- +- name: vmagent ноды + ansible.builtin.include_role: + name: vmagent + vars: + monitoring_node_exporter_public: false + monitoring_node_exporter_listen: "127.0.0.1:9100" diff --git a/ansible/roles/host/defaults/main.yml b/ansible/roles/host/defaults/main.yml index cf21dc2..cbc70da 100644 --- a/ansible/roles/host/defaults/main.yml +++ b/ansible/roles/host/defaults/main.yml @@ -1,4 +1,7 @@ --- +monitoring_node_exporter_public: true +monitoring_node_exporter_listen: "0.0.0.0:9100" + host_authorized_keys: - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF/yxhAs022l/8FlPlhyi7NeF3CvVGsev3B1HtKICO7M root_from_windows" - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEpwqB40k8QQq3k8160gfMKDaCJCX9qzvmuijfe3mO4I gitea-runner" diff --git a/ansible/roles/host/tasks/main.yml b/ansible/roles/host/tasks/main.yml index a13c914..bc98a88 100644 --- a/ansible/roles/host/tasks/main.yml +++ b/ansible/roles/host/tasks/main.yml @@ -91,11 +91,22 @@ failed_when: false changed_when: rm_legacy_exporter.rc == 0 +- name: Слушать node-exporter только локально + ansible.builtin.copy: + dest: /etc/default/prometheus-node-exporter + content: | + ARGS="--web.listen-address={{ monitoring_node_exporter_listen }}" + owner: root + group: root + mode: "0644" + register: node_exporter_args + when: not (monitoring_node_exporter_public | bool) + - name: Включить prometheus-node-exporter ansible.builtin.systemd: name: prometheus-node-exporter enabled: true - state: started + state: "{{ 'restarted' if (node_exporter_args is changed) else 'started' }}" - name: Разрешить SSH ansible.builtin.command: ufw allow OpenSSH @@ -107,7 +118,19 @@ ansible.builtin.command: ufw allow 9100/tcp register: ufw_metrics changed_when: "'added' in (ufw_metrics.stdout | lower)" - when: ansible_virtualization_type | default('') != 'lxc' + when: + - ansible_virtualization_type | default('') != 'lxc' + - monitoring_node_exporter_public | bool + +- name: Закрыть порт метрик node-exporter снаружи + ansible.builtin.command: ufw --force delete allow 9100/tcp + register: ufw_metrics_del + changed_when: "'deleted' in (stdout | default('') | lower)" + failed_when: false + loop: [1, 2] + when: + - ansible_virtualization_type | default('') != 'lxc' + - not (monitoring_node_exporter_public | bool) - name: Проверить состояние firewall ansible.builtin.command: ufw status diff --git a/ansible/roles/netbird_client/defaults/main.yml b/ansible/roles/netbird_client/defaults/main.yml new file mode 100644 index 0000000..4a52e5e --- /dev/null +++ b/ansible/roles/netbird_client/defaults/main.yml @@ -0,0 +1,4 @@ +--- +netbird_client_management_url: "https://central.gunihome.ru" +netbird_client_setup_key: "{{ lookup('env', 'NETBIRD_REMNAWAVE_SETUP_KEY', default='') }}" +netbird_client_hostname: "{{ inventory_hostname }}" diff --git a/ansible/roles/netbird_client/tasks/main.yml b/ansible/roles/netbird_client/tasks/main.yml new file mode 100644 index 0000000..bd58152 --- /dev/null +++ b/ansible/roles/netbird_client/tasks/main.yml @@ -0,0 +1,86 @@ +--- +- name: Проверить ключ установки Remnawave + ansible.builtin.assert: + that: + - netbird_client_setup_key | length > 0 + fail_msg: "Нет NETBIRD_REMNAWAVE_SETUP_KEY. Сначала прогоните ansible-rus-central." + +- name: Проверить /dev/net/tun + ansible.builtin.stat: + path: /dev/net/tun + register: netbird_tun + +- name: Нет TUN для NetBird + ansible.builtin.fail: + msg: "Нет /dev/net/tun. В LXC для NetBird нужны nesting и устройство /dev/net/tun." + when: not netbird_tun.stat.exists + +- name: Поставить зависимости NetBird + ansible.builtin.apt: + name: + - ca-certificates + - curl + - gnupg + state: present + update_cache: true + +- name: Ключ репозитория NetBird + ansible.builtin.shell: | + set -euo pipefail + install -d -m 0755 /usr/share/keyrings + curl -fsSL https://pkgs.netbird.io/debian/public.key \ + | gpg --dearmor --yes -o /usr/share/keyrings/netbird-archive-keyring.gpg + chmod 0644 /usr/share/keyrings/netbird-archive-keyring.gpg + args: + executable: /bin/bash + creates: /usr/share/keyrings/netbird-archive-keyring.gpg + +- name: Репозиторий NetBird + ansible.builtin.copy: + dest: /etc/apt/sources.list.d/netbird.list + content: | + deb [signed-by=/usr/share/keyrings/netbird-archive-keyring.gpg] https://pkgs.netbird.io/debian stable main + owner: root + group: root + mode: "0644" + +- name: Поставить клиент NetBird + ansible.builtin.apt: + name: netbird + state: present + update_cache: true + +- name: Включить службу NetBird + ansible.builtin.systemd: + name: netbird + enabled: true + state: started + +- name: Узнать, подключён ли NetBird + ansible.builtin.command: netbird status + register: netbird_status_before + changed_when: false + failed_when: false + +- name: Подключить NetBird к central.gunihome.ru + ansible.builtin.command: > + netbird up + --management-url {{ netbird_client_management_url }} + --setup-key {{ netbird_client_setup_key }} + --hostname {{ netbird_client_hostname }} + no_log: true + when: "'Management: Connected' not in (netbird_status_before.stdout | default(''))" + register: netbird_up + changed_when: true + +- name: Дождаться Management Connected + ansible.builtin.command: netbird status + register: netbird_status + changed_when: false + retries: 10 + delay: 3 + until: "'Management: Connected' in netbird_status.stdout" + +- name: Показать состояние NetBird + ansible.builtin.debug: + msg: "{{ netbird_status.stdout_lines | select('search', 'Management:|NetBird IP:|FQDN:|Peers:') | list }}" diff --git a/ansible/roles/rus-central/netbird/defaults/main.yml b/ansible/roles/rus-central/netbird/defaults/main.yml index 7f7a6e9..833b5ab 100644 --- a/ansible/roles/rus-central/netbird/defaults/main.yml +++ b/ansible/roles/rus-central/netbird/defaults/main.yml @@ -19,5 +19,8 @@ netbird_setup_key: "{{ lookup('env', 'NETBIRD_SETUP_KEY') }}" netbird_router_group_name: "homelab-routers" netbird_setup_key_name: "homelab-routers-key" +netbird_remnawave_group_name: "remnawave" +netbird_remnawave_setup_key_name: "remnawave-peers" +netbird_remnawave_setup_key: "{{ lookup('env', 'NETBIRD_REMNAWAVE_SETUP_KEY') }}" netbird_lan_network_id: "homelab-lan" netbird_lan_subnet: "192.168.1.0/24" diff --git a/ansible/roles/rus-central/netbird/tasks/main.yml b/ansible/roles/rus-central/netbird/tasks/main.yml index 6251dd7..3d740da 100644 --- a/ansible/roles/rus-central/netbird/tasks/main.yml +++ b/ansible/roles/rus-central/netbird/tasks/main.yml @@ -228,6 +228,20 @@ router_group_id=$(printf '%s' "${grp_resp}" | jq -r '.id') fi + remna_group_id=$(printf '%s' "${groups_json}" | jq -r --arg g "$NB_REMNAWAVE_GROUP" '.[] | select(.name == $g) | .id' | head -n 1) + if [ -z "${remna_group_id}" ]; then + grp_payload=$(jq -n --arg name "$NB_REMNAWAVE_GROUP" '{name: $name, peers: []}') + grp_resp=$(curl "${CURL_OPTS[@]}" -X POST "${API_BASE}/groups" \ + -H "Authorization: Token ${active_token}" \ + -H "Content-Type: application/json" \ + -d "${grp_payload}") + remna_group_id=$(printf '%s' "${grp_resp}" | jq -r '.id // empty') + if [ -z "${remna_group_id}" ]; then + echo "Не удалось создать группу NetBird для Remnawave" >&2 + exit 1 + fi + fi + routes_json=$(curl "${CURL_OPTS[@]}" -H "Authorization: Token ${active_token}" "${API_BASE}/routes") existing_route_id=$(printf '%s' "${routes_json}" | jq -r --arg nid "$NB_LAN_NET_ID" --arg net "$NB_LAN_SUBNET" '.[] | select(.network_id == $nid or .network == $net) | .id' | head -n 1) @@ -288,21 +302,57 @@ setup_key_updated=1 fi + active_remna_key="${NB_REMNAWAVE_SETUP_KEY:-}" + remna_key_updated=0 + valid_remna_id=$(printf '%s' "${keys_json}" | jq -r --arg kname "$NB_REMNAWAVE_KEY_NAME" '.[] | select(.name == $kname and .valid == true and .revoked == false) | .id' | head -n 1) + if [ -z "${active_remna_key}" ] || [ -z "${valid_remna_id}" ]; then + fresh_keys=$(curl "${CURL_OPTS[@]}" -H "Authorization: Token ${active_token}" "${API_BASE}/setup-keys") + for old_kid in $(printf '%s' "${fresh_keys}" | jq -r --arg kname "$NB_REMNAWAVE_KEY_NAME" '.[] | select(.name == $kname) | .id'); do + curl "${CURL_OPTS[@]}" -X DELETE -H "Authorization: Token ${active_token}" "${API_BASE}/setup-keys/${old_kid}" >/dev/null || true + done + key_payload=$(jq -n \ + --arg name "$NB_REMNAWAVE_KEY_NAME" \ + --arg gid "$remna_group_id" \ + '{ + name: $name, + type: "reusable", + expires_in: 31536000, + revoked: false, + auto_groups: [$gid], + usage_limit: 0, + ephemeral: false + }') + key_resp=$(curl "${CURL_OPTS[@]}" -X POST "${API_BASE}/setup-keys" \ + -H "Authorization: Token ${active_token}" \ + -H "Content-Type: application/json" \ + -d "${key_payload}") + active_remna_key=$(printf '%s' "${key_resp}" | jq -r '.key // empty') + if [ -z "${active_remna_key}" ]; then + echo "Не удалось создать ключ установки remnawave-peers" >&2 + exit 1 + fi + remna_key_updated=1 + fi + jq -n \ --arg api_token "${active_token}" \ --arg setup_key "${active_setup_key}" \ + --arg remna_key "${active_remna_key}" \ --argjson admin_created "${admin_created}" \ --argjson token_updated "${token_updated}" \ --argjson setup_key_updated "${setup_key_updated}" \ + --argjson remna_key_updated "${remna_key_updated}" \ --argjson route_created "${route_created}" \ '{ api_token: $api_token, setup_key: $setup_key, + remnawave_setup_key: $remna_key, admin_created: ($admin_created == 1), token_updated: ($token_updated == 1), setup_key_updated: ($setup_key_updated == 1), + remnawave_setup_key_updated: ($remna_key_updated == 1), route_created: ($route_created == 1), - changed: (($admin_created + $token_updated + $setup_key_updated + $route_created) > 0) + changed: (($admin_created + $token_updated + $setup_key_updated + $remna_key_updated + $route_created) > 0) }' args: executable: /bin/bash @@ -314,6 +364,9 @@ NB_SETUP_KEY: "{{ netbird_setup_key }}" NB_ROUTER_GROUP: "{{ netbird_router_group_name }}" NB_SETUP_KEY_NAME: "{{ netbird_setup_key_name }}" + NB_REMNAWAVE_GROUP: "{{ netbird_remnawave_group_name }}" + NB_REMNAWAVE_KEY_NAME: "{{ netbird_remnawave_setup_key_name }}" + NB_REMNAWAVE_SETUP_KEY: "{{ netbird_remnawave_setup_key }}" NB_LAN_NET_ID: "{{ netbird_lan_network_id }}" NB_LAN_SUBNET: "{{ netbird_lan_subnet }}" register: netbird_bootstrap_raw @@ -331,6 +384,7 @@ --path="/services/netbird" \ "NETBIRD_API_TOKEN=${NEW_API_TOKEN}" \ "NETBIRD_SETUP_KEY=${NEW_SETUP_KEY}" \ + "NETBIRD_REMNAWAVE_SETUP_KEY=${NEW_REMNAWAVE_SETUP_KEY}" \ --silent args: executable: /bin/bash @@ -339,7 +393,11 @@ environment: NEW_API_TOKEN: "{{ (netbird_bootstrap_raw.stdout | from_json).api_token }}" NEW_SETUP_KEY: "{{ (netbird_bootstrap_raw.stdout | from_json).setup_key }}" - when: (netbird_bootstrap_raw.stdout | from_json).token_updated or (netbird_bootstrap_raw.stdout | from_json).setup_key_updated + NEW_REMNAWAVE_SETUP_KEY: "{{ (netbird_bootstrap_raw.stdout | from_json).remnawave_setup_key }}" + when: >- + (netbird_bootstrap_raw.stdout | from_json).token_updated or + (netbird_bootstrap_raw.stdout | from_json).setup_key_updated or + (netbird_bootstrap_raw.stdout | from_json).remnawave_setup_key_updated changed_when: true no_log: true @@ -352,4 +410,5 @@ - "Суперпользователь создан в этом прогоне: {{ nb_state.admin_created }}" - "API-токен обновлен в Infisical: {{ nb_state.token_updated }}" - "Setup-ключ обновлен в Infisical: {{ nb_state.setup_key_updated }}" + - "Ключ remnawave-peers обновлен в Infisical: {{ nb_state.remnawave_setup_key_updated }}" - "Маршрут {{ netbird_lan_subnet }} создан в этом прогоне: {{ nb_state.route_created }}" diff --git a/ansible/roles/vmagent/defaults/main.yml b/ansible/roles/vmagent/defaults/main.yml new file mode 100644 index 0000000..75b00d7 --- /dev/null +++ b/ansible/roles/vmagent/defaults/main.yml @@ -0,0 +1,16 @@ +--- +vmagent_dir: /opt/vmagent +vmagent_image: victoriametrics/vmagent:v1.153.0 +# Публичный образ cAdvisor заканчивается на v0.49.2. Бинарника v0.60.6 из гайда в registry нет. +cadvisor_image: gcr.io/cadvisor/cadvisor:v0.49.2 +vmagent_remote_host: "192.168.1.108" +vmagent_remote_write_url: "http://{{ vmagent_remote_host }}:8428/api/v1/write" +vmagent_remote_health_url: "http://{{ vmagent_remote_host }}:8428/health" +vmagent_instance: "{{ inventory_hostname }}" +vmagent_cluster: homelab +vmagent_scrape_remnawave: false +vmagent_metrics_user: admin +vmagent_metrics_pass: "{{ lookup('env', 'REMNAWAVE_METRICS_PASS', default='') }}" +vmagent_metrics_port: 3001 +monitoring_node_exporter_public: true +monitoring_node_exporter_listen: "0.0.0.0:9100" diff --git a/ansible/roles/vmagent/tasks/main.yml b/ansible/roles/vmagent/tasks/main.yml new file mode 100644 index 0000000..3d0eb49 --- /dev/null +++ b/ansible/roles/vmagent/tasks/main.yml @@ -0,0 +1,106 @@ +--- +- name: Проверить пароль метрик панели + ansible.builtin.assert: + that: + - vmagent_metrics_pass | length > 0 + fail_msg: "Нет REMNAWAVE_METRICS_PASS для scrape панели." + when: vmagent_scrape_remnawave | bool + +- name: Слушать node-exporter локально, если порт не публичный + ansible.builtin.copy: + dest: /etc/default/prometheus-node-exporter + content: | + ARGS="--web.listen-address={{ monitoring_node_exporter_listen }}" + owner: root + group: root + mode: "0644" + register: vmagent_exporter_env + when: not (monitoring_node_exporter_public | bool) + +- name: Перезапустить node-exporter после смены адреса + ansible.builtin.systemd: + name: prometheus-node-exporter + state: restarted + enabled: true + when: vmagent_exporter_env is changed + +- name: Дождаться VictoriaMetrics + ansible.builtin.command: curl -fsS --max-time 5 {{ vmagent_remote_health_url }} + register: vmagent_vm_health + changed_when: false + retries: 12 + delay: 5 + until: vmagent_vm_health.rc == 0 + failed_when: false + +- name: VictoriaMetrics недоступна + ansible.builtin.fail: + msg: >- + {{ vmagent_remote_health_url }} не отвечает. + Сначала поднимите VictoriaMetrics на core-mon. + С ger-nod-02 этот адрес открывается маршрутом homelab-lan через OPNsense. + when: vmagent_vm_health.rc != 0 + +- name: Каталоги vmagent + ansible.builtin.file: + path: "{{ item }}" + state: directory + owner: root + group: root + mode: "0755" + loop: + - "{{ vmagent_dir }}" + - "{{ vmagent_dir }}/conf.d" + +- name: Конфиг scrape vmagent + ansible.builtin.template: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + owner: root + group: root + mode: "{{ item.mode }}" + loop: + - { src: scrape.yml.j2, dest: "{{ vmagent_dir }}/scrape.yml", mode: "0644" } + - { src: nodeexporter.yml.j2, dest: "{{ vmagent_dir }}/conf.d/nodeexporter.yml", mode: "0644" } + - { src: cadvisor.yml.j2, dest: "{{ vmagent_dir }}/conf.d/cadvisor.yml", mode: "0644" } + - { src: compose.yaml.j2, dest: "{{ vmagent_dir }}/compose.yaml", mode: "0644" } + +- name: Убрать scrape панели, если он не нужен + ansible.builtin.file: + path: "{{ vmagent_dir }}/conf.d/remnawave.yml" + state: absent + when: not (vmagent_scrape_remnawave | bool) + +- name: Scrape метрик панели + ansible.builtin.template: + src: remnawave.yml.j2 + dest: "{{ vmagent_dir }}/conf.d/remnawave.yml" + owner: root + group: root + mode: "0600" + no_log: true + when: vmagent_scrape_remnawave | bool + +- name: Запустить vmagent и cAdvisor + ansible.builtin.command: docker compose -f {{ vmagent_dir }}/compose.yaml up -d --remove-orphans + register: vmagent_up + changed_when: >- + 'Started' in vmagent_up.stderr or + 'Created' in vmagent_up.stderr or + 'Recreated' in vmagent_up.stderr or + 'Pulled' in vmagent_up.stderr + +- name: Дождаться локального vmagent + ansible.builtin.uri: + url: http://127.0.0.1:8429/health + status_code: 200 + register: vmagent_local + retries: 15 + delay: 2 + until: vmagent_local.status == 200 + +- name: Итог vmagent + ansible.builtin.debug: + msg: + - "vmagent {{ inventory_hostname }} пишет в {{ vmagent_remote_write_url }}" + - "instance={{ vmagent_instance }} scrape панели={{ vmagent_scrape_remnawave | bool }}" diff --git a/ansible/roles/vmagent/templates/cadvisor.yml.j2 b/ansible/roles/vmagent/templates/cadvisor.yml.j2 new file mode 100644 index 0000000..77fec48 --- /dev/null +++ b/ansible/roles/vmagent/templates/cadvisor.yml.j2 @@ -0,0 +1,7 @@ +- job_name: integrations/cAdvisor + scrape_interval: 15s + static_configs: + - targets: ["127.0.0.1:9101"] + labels: + instance: "{{ vmagent_instance }}" + cluster: "{{ vmagent_cluster }}" diff --git a/ansible/roles/vmagent/templates/compose.yaml.j2 b/ansible/roles/vmagent/templates/compose.yaml.j2 new file mode 100644 index 0000000..2f66ba7 --- /dev/null +++ b/ansible/roles/vmagent/templates/compose.yaml.j2 @@ -0,0 +1,33 @@ +services: + cadvisor: + image: {{ cadvisor_image }} + container_name: vmagent-cadvisor + network_mode: host + privileged: true + restart: unless-stopped + command: + - --listen_ip=127.0.0.1 + - --port=9101 + - --docker_only=true + volumes: + - /:/rootfs:ro + - /var/run:/var/run:ro + - /sys:/sys:ro + - /var/lib/docker:/var/lib/docker:ro + - /dev/disk:/dev/disk:ro + + vmagent: + image: {{ vmagent_image }} + container_name: vmagent + network_mode: host + restart: unless-stopped + command: + - -httpListenAddr=127.0.0.1:8429 + - -promscrape.config=/etc/vmagent/scrape.yml + - -promscrape.configCheckInterval=60s + - -remoteWrite.url={{ vmagent_remote_write_url }} + volumes: + - ./scrape.yml:/etc/vmagent/scrape.yml:ro + - ./conf.d:/etc/vmagent/conf.d:ro + depends_on: + - cadvisor diff --git a/ansible/roles/vmagent/templates/nodeexporter.yml.j2 b/ansible/roles/vmagent/templates/nodeexporter.yml.j2 new file mode 100644 index 0000000..6feaa63 --- /dev/null +++ b/ansible/roles/vmagent/templates/nodeexporter.yml.j2 @@ -0,0 +1,7 @@ +- job_name: integrations/node_exporter + scrape_interval: 15s + static_configs: + - targets: ["127.0.0.1:9100"] + labels: + instance: "{{ vmagent_instance }}" + cluster: "{{ vmagent_cluster }}" diff --git a/ansible/roles/vmagent/templates/remnawave.yml.j2 b/ansible/roles/vmagent/templates/remnawave.yml.j2 new file mode 100644 index 0000000..4f7bc21 --- /dev/null +++ b/ansible/roles/vmagent/templates/remnawave.yml.j2 @@ -0,0 +1,10 @@ +- job_name: remnawave + scrape_interval: 15s + static_configs: + - targets: ["127.0.0.1:{{ vmagent_metrics_port }}"] + labels: + instance: "{{ vmagent_instance }}" + cluster: "{{ vmagent_cluster }}" + basic_auth: + username: "{{ vmagent_metrics_user }}" + password: "{{ vmagent_metrics_pass }}" diff --git a/ansible/roles/vmagent/templates/scrape.yml.j2 b/ansible/roles/vmagent/templates/scrape.yml.j2 new file mode 100644 index 0000000..81638d8 --- /dev/null +++ b/ansible/roles/vmagent/templates/scrape.yml.j2 @@ -0,0 +1,5 @@ +scrape_config_files: + - "/etc/vmagent/conf.d/*.yml" + +global: + scrape_interval: 15s