feat(monitoring): add VictoriaLogs, Grafana plugin, and Vector log collectors
ansible-deploy / plan (push) Successful in 1s
ansible-lint / lint (push) Successful in 3s
ansible-deploy / deploy (core-mon, ) (push) Successful in 1m12s
ansible-deploy / deploy (core-net, ) (push) Successful in 1m8s
ansible-deploy / deploy (core-secrets, ) (push) Successful in 36s
ansible-deploy / deploy (rus-central, ) (push) Canceled after 0s
ansible-deploy / deploy (ger-nod-02, ) (push) Canceled after 41s
ansible-deploy / plan (push) Successful in 1s
ansible-lint / lint (push) Successful in 3s
ansible-deploy / deploy (core-mon, ) (push) Successful in 1m12s
ansible-deploy / deploy (core-net, ) (push) Successful in 1m8s
ansible-deploy / deploy (core-secrets, ) (push) Successful in 36s
ansible-deploy / deploy (rus-central, ) (push) Canceled after 0s
ansible-deploy / deploy (ger-nod-02, ) (push) Canceled after 41s
This commit is contained in:
1 parent
9734f5b9e8
commit
200c56b881
15 files changed
+170
No files matched your search
@@ -81,6 +81,11 @@ if [ "$full" -eq 0 ] && [ -n "$changed" ]; then
|
|||||||
add_tag vmagent
|
add_tag vmagent
|
||||||
fi
|
fi
|
||||||
;;
|
;;
|
||||||
|
ansible/roles/vector/*)
|
||||||
|
if grep -q "role: vector" "$playbook"; then
|
||||||
|
add_tag vector
|
||||||
|
fi
|
||||||
|
;;
|
||||||
ansible/roles/"$machine"/.ci-trigger)
|
ansible/roles/"$machine"/.ci-trigger)
|
||||||
;;
|
;;
|
||||||
ansible/roles/"$machine"/*/*)
|
ansible/roles/"$machine"/*/*)
|
||||||
|
|||||||
@@ -5,6 +5,8 @@
|
|||||||
roles:
|
roles:
|
||||||
- role: core-mon/monitoring
|
- role: core-mon/monitoring
|
||||||
tags: [monitoring]
|
tags: [monitoring]
|
||||||
|
- role: vector
|
||||||
|
tags: [vector]
|
||||||
post_tasks:
|
post_tasks:
|
||||||
- name: Зарегистрировать Grafana в Agent Vault
|
- name: Зарегистрировать Grafana в Agent Vault
|
||||||
ansible.builtin.include_role:
|
ansible.builtin.include_role:
|
||||||
|
|||||||
@@ -13,3 +13,5 @@
|
|||||||
tags: [vmagent]
|
tags: [vmagent]
|
||||||
- role: core-net/netbird
|
- role: core-net/netbird
|
||||||
tags: [netbird]
|
tags: [netbird]
|
||||||
|
- role: vector
|
||||||
|
tags: [vector]
|
||||||
@@ -9,6 +9,8 @@
|
|||||||
tags: [vaultwarden]
|
tags: [vaultwarden]
|
||||||
- role: core-secrets/cadvisor
|
- role: core-secrets/cadvisor
|
||||||
tags: [cadvisor]
|
tags: [cadvisor]
|
||||||
|
- role: vector
|
||||||
|
tags: [vector]
|
||||||
post_tasks:
|
post_tasks:
|
||||||
- name: Зарегистрировать Gitea в Agent Vault
|
- name: Зарегистрировать Gitea в Agent Vault
|
||||||
ansible.builtin.include_role:
|
ansible.builtin.include_role:
|
||||||
|
|||||||
@@ -9,3 +9,5 @@
|
|||||||
tags: [remnanode]
|
tags: [remnanode]
|
||||||
- role: vmagent
|
- role: vmagent
|
||||||
tags: [vmagent]
|
tags: [vmagent]
|
||||||
|
- role: vector
|
||||||
|
tags: [vector]
|
||||||
@@ -7,3 +7,5 @@
|
|||||||
tags: [netbird]
|
tags: [netbird]
|
||||||
- role: rus-central/cadvisor
|
- role: rus-central/cadvisor
|
||||||
tags: [cadvisor]
|
tags: [cadvisor]
|
||||||
|
- role: vector
|
||||||
|
tags: [vector]
|
||||||
@@ -10,6 +10,8 @@ monitoring_scrape_exclude_hosts:
|
|||||||
monitoring_node_exporter_port: 9100
|
monitoring_node_exporter_port: 9100
|
||||||
|
|
||||||
victoriametrics_image: victoriametrics/victoria-metrics:v1.153.0
|
victoriametrics_image: victoriametrics/victoria-metrics:v1.153.0
|
||||||
|
victorialogs_image: victoriametrics/victoria-logs:v1.54.0
|
||||||
|
victorialogs_retention: "14d"
|
||||||
vmalert_image: victoriametrics/vmalert:v1.153.0
|
vmalert_image: victoriametrics/vmalert:v1.153.0
|
||||||
vmagent_image: victoriametrics/vmagent:v1.153.0
|
vmagent_image: victoriametrics/vmagent:v1.153.0
|
||||||
# Свой vmagent уже пишет node-exporter этих хостов. Повторный scrape даст двойные сэмплы.
|
# Свой vmagent уже пишет node-exporter этих хостов. Повторный scrape даст двойные сэмплы.
|
||||||
|
|||||||
@@ -34,6 +34,15 @@
|
|||||||
delay: 2
|
delay: 2
|
||||||
until: victoriametrics_health.status == 200
|
until: victoriametrics_health.status == 200
|
||||||
|
|
||||||
|
- name: Дождаться VictoriaLogs
|
||||||
|
ansible.builtin.uri:
|
||||||
|
url: "http://{{ ansible_host }}:9428/health"
|
||||||
|
status_code: 200
|
||||||
|
register: victorialogs_health
|
||||||
|
retries: 20
|
||||||
|
delay: 2
|
||||||
|
until: victorialogs_health.status == 200
|
||||||
|
|
||||||
- name: Дождаться vmalert
|
- name: Дождаться vmalert
|
||||||
ansible.builtin.uri:
|
ansible.builtin.uri:
|
||||||
url: "http://127.0.0.1:8880/health"
|
url: "http://127.0.0.1:8880/health"
|
||||||
|
|||||||
@@ -18,6 +18,7 @@
|
|||||||
- { path: "{{ monitoring_dir }}/ntfy_data", owner: "root", group: "root", mode: "0777" }
|
- { path: "{{ monitoring_dir }}/ntfy_data", owner: "root", group: "root", mode: "0777" }
|
||||||
- { path: "{{ monitoring_dir }}/alertmanager-ntfy", owner: "root", group: "root", mode: "0755" }
|
- { path: "{{ monitoring_dir }}/alertmanager-ntfy", owner: "root", group: "root", mode: "0755" }
|
||||||
- { path: "{{ monitoring_dir }}/victoria-metrics-data", owner: "65534", group: "65534", mode: "0775" }
|
- { path: "{{ monitoring_dir }}/victoria-metrics-data", owner: "65534", group: "65534", mode: "0775" }
|
||||||
|
- { path: "{{ monitoring_dir }}/victoria-logs-data", owner: "65534", group: "65534", mode: "0775" }
|
||||||
- { path: "{{ monitoring_dir }}/vmagent", owner: "root", group: "root", mode: "0755" }
|
- { path: "{{ monitoring_dir }}/vmagent", owner: "root", group: "root", mode: "0755" }
|
||||||
- { path: "{{ monitoring_dir }}/vmagent/conf.d", owner: "root", group: "root", mode: "0755" }
|
- { path: "{{ monitoring_dir }}/vmagent/conf.d", owner: "root", group: "root", mode: "0755" }
|
||||||
- { path: "{{ monitoring_dir }}/vmalert", owner: "root", group: "root", mode: "0755" }
|
- { path: "{{ monitoring_dir }}/vmalert", owner: "root", group: "root", mode: "0755" }
|
||||||
|
|||||||
@@ -53,6 +53,7 @@ services:
|
|||||||
- GF_SECURITY_ADMIN_PASSWORD={{ grafana_admin_password }}
|
- GF_SECURITY_ADMIN_PASSWORD={{ grafana_admin_password }}
|
||||||
- GF_USERS_ALLOW_SIGN_UP=false
|
- GF_USERS_ALLOW_SIGN_UP=false
|
||||||
- GF_SERVER_ROOT_URL={{ grafana_root_url }}
|
- GF_SERVER_ROOT_URL={{ grafana_root_url }}
|
||||||
|
- GF_INSTALL_PLUGINS=victoriametrics-logs-datasource
|
||||||
volumes:
|
volumes:
|
||||||
- ./grafana_data:/var/lib/grafana
|
- ./grafana_data:/var/lib/grafana
|
||||||
- ./grafana/provisioning:/etc/grafana/provisioning:ro
|
- ./grafana/provisioning:/etc/grafana/provisioning:ro
|
||||||
@@ -60,6 +61,7 @@ services:
|
|||||||
- "3000:3000"
|
- "3000:3000"
|
||||||
depends_on:
|
depends_on:
|
||||||
- vmsingle
|
- vmsingle
|
||||||
|
- vlogs
|
||||||
|
|
||||||
vmsingle:
|
vmsingle:
|
||||||
image: {{ victoriametrics_image }}
|
image: {{ victoriametrics_image }}
|
||||||
@@ -74,6 +76,19 @@ services:
|
|||||||
ports:
|
ports:
|
||||||
- "{{ ansible_host }}:8428:8428"
|
- "{{ ansible_host }}:8428:8428"
|
||||||
|
|
||||||
|
vlogs:
|
||||||
|
image: {{ victorialogs_image }}
|
||||||
|
container_name: monitoring-vlogs
|
||||||
|
hostname: vlogs
|
||||||
|
restart: unless-stopped
|
||||||
|
command:
|
||||||
|
- -storageDataPath=/victoria-logs-data
|
||||||
|
- -retentionPeriod={{ victorialogs_retention }}
|
||||||
|
volumes:
|
||||||
|
- ./victoria-logs-data:/victoria-logs-data
|
||||||
|
ports:
|
||||||
|
- "{{ ansible_host }}:9428:9428"
|
||||||
|
|
||||||
cadvisor:
|
cadvisor:
|
||||||
image: {{ cadvisor_image }}
|
image: {{ cadvisor_image }}
|
||||||
container_name: monitoring-cadvisor
|
container_name: monitoring-cadvisor
|
||||||
|
|||||||
@@ -19,3 +19,10 @@ datasources:
|
|||||||
jsonData:
|
jsonData:
|
||||||
timeInterval: 15s
|
timeInterval: 15s
|
||||||
|
|
||||||
|
- name: VictoriaLogs
|
||||||
|
uid: victorialogs
|
||||||
|
type: victoriametrics-logs-datasource
|
||||||
|
access: proxy
|
||||||
|
url: http://vlogs:9428
|
||||||
|
isDefault: false
|
||||||
|
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
---
|
||||||
|
vector_dir: /opt/vector
|
||||||
|
vector_image: "timberio/vector:0.45.0-alpine"
|
||||||
|
vector_vlogs_host: "{{ hostvars['core-mon'].ansible_host }}"
|
||||||
|
vector_vlogs_url: "http://{{ vector_vlogs_host }}:9428/insert/jsonline"
|
||||||
|
vector_host_name: "{{ inventory_hostname }}"
|
||||||
@@ -0,0 +1,43 @@
|
|||||||
|
---
|
||||||
|
- name: Создать каталоги Vector
|
||||||
|
ansible.builtin.file:
|
||||||
|
path: "{{ item }}"
|
||||||
|
state: directory
|
||||||
|
owner: root
|
||||||
|
group: root
|
||||||
|
mode: "0755"
|
||||||
|
loop:
|
||||||
|
- "{{ vector_dir }}"
|
||||||
|
- "{{ vector_dir }}/data"
|
||||||
|
|
||||||
|
- name: Конфиг Vector
|
||||||
|
ansible.builtin.template:
|
||||||
|
src: vector.yaml.j2
|
||||||
|
dest: "{{ vector_dir }}/vector.yaml"
|
||||||
|
owner: root
|
||||||
|
group: root
|
||||||
|
mode: "0644"
|
||||||
|
register: vector_config
|
||||||
|
|
||||||
|
- name: Compose Vector
|
||||||
|
ansible.builtin.template:
|
||||||
|
src: compose.yaml.j2
|
||||||
|
dest: "{{ vector_dir }}/compose.yaml"
|
||||||
|
owner: root
|
||||||
|
group: root
|
||||||
|
mode: "0644"
|
||||||
|
register: vector_compose
|
||||||
|
|
||||||
|
- name: Запустить Vector
|
||||||
|
ansible.builtin.command: docker compose -f {{ vector_dir }}/compose.yaml up -d --remove-orphans
|
||||||
|
register: vector_up
|
||||||
|
changed_when: >-
|
||||||
|
'Started' in vector_up.stderr or
|
||||||
|
'Created' in vector_up.stderr or
|
||||||
|
'Recreated' in vector_up.stderr or
|
||||||
|
'Pulled' in vector_up.stderr
|
||||||
|
|
||||||
|
- name: Перезапустить Vector после изменения конфигурации
|
||||||
|
ansible.builtin.command: docker compose -f {{ vector_dir }}/compose.yaml restart vector
|
||||||
|
when: vector_config is changed and not (vector_compose is changed)
|
||||||
|
changed_when: true
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
name: vector
|
||||||
|
|
||||||
|
services:
|
||||||
|
vector:
|
||||||
|
image: {{ vector_image }}
|
||||||
|
container_name: vector
|
||||||
|
restart: unless-stopped
|
||||||
|
command:
|
||||||
|
- "--config"
|
||||||
|
- "/etc/vector/vector.yaml"
|
||||||
|
volumes:
|
||||||
|
- ./vector.yaml:/etc/vector/vector.yaml:ro
|
||||||
|
- ./data:/var/lib/vector
|
||||||
|
- /var/run/docker.sock:/var/run/docker.sock:ro
|
||||||
|
- /var/log:/var/log:ro
|
||||||
@@ -0,0 +1,57 @@
|
|||||||
|
data_dir: /var/lib/vector
|
||||||
|
|
||||||
|
sources:
|
||||||
|
docker:
|
||||||
|
type: docker_logs
|
||||||
|
exclude_containers:
|
||||||
|
- vector
|
||||||
|
|
||||||
|
host_logs:
|
||||||
|
type: file
|
||||||
|
include:
|
||||||
|
- /var/log/syslog
|
||||||
|
- /var/log/messages
|
||||||
|
- /var/log/auth.log
|
||||||
|
ignore_older_secs: 86400
|
||||||
|
|
||||||
|
transforms:
|
||||||
|
normalize_docker:
|
||||||
|
type: remap
|
||||||
|
inputs:
|
||||||
|
- docker
|
||||||
|
source: |
|
||||||
|
.host = "{{ vector_host_name }}"
|
||||||
|
.source_type = "docker"
|
||||||
|
.service = .container_name
|
||||||
|
if exists(.label."com.docker.compose.service") {
|
||||||
|
.service = string!(.label."com.docker.compose.service")
|
||||||
|
}
|
||||||
|
|
||||||
|
normalize_host:
|
||||||
|
type: remap
|
||||||
|
inputs:
|
||||||
|
- host_logs
|
||||||
|
source: |
|
||||||
|
.host = "{{ vector_host_name }}"
|
||||||
|
.source_type = "host"
|
||||||
|
.service = replace(string!(.file), "/var/log/", "")
|
||||||
|
|
||||||
|
sinks:
|
||||||
|
victorialogs:
|
||||||
|
type: http
|
||||||
|
inputs:
|
||||||
|
- normalize_docker
|
||||||
|
- normalize_host
|
||||||
|
uri: "{{ vector_vlogs_url }}"
|
||||||
|
encoding:
|
||||||
|
codec: json
|
||||||
|
framing:
|
||||||
|
method: newline_delimited
|
||||||
|
compression: gzip
|
||||||
|
request:
|
||||||
|
headers:
|
||||||
|
VL-Stream-Fields: "host,source_type,service"
|
||||||
|
VL-Msg-Field: "message"
|
||||||
|
VL-Time-Field: "timestamp"
|
||||||
|
healthcheck:
|
||||||
|
enabled: false
|
||||||
Reference in new issue
Block a user