diff --git a/ansible/ci/ansible-target.sh b/ansible/ci/ansible-target.sh index 4fd9463..dc29186 100755 --- a/ansible/ci/ansible-target.sh +++ b/ansible/ci/ansible-target.sh @@ -81,6 +81,11 @@ if [ "$full" -eq 0 ] && [ -n "$changed" ]; then add_tag vmagent fi ;; + ansible/roles/vector/*) + if grep -q "role: vector" "$playbook"; then + add_tag vector + fi + ;; ansible/roles/"$machine"/.ci-trigger) ;; ansible/roles/"$machine"/*/*) diff --git a/ansible/playbooks/core-mon.yml b/ansible/playbooks/core-mon.yml index 1374016..59c1fc1 100644 --- a/ansible/playbooks/core-mon.yml +++ b/ansible/playbooks/core-mon.yml @@ -5,6 +5,8 @@ roles: - role: core-mon/monitoring tags: [monitoring] + - role: vector + tags: [vector] post_tasks: - name: Зарегистрировать Grafana в Agent Vault ansible.builtin.include_role: diff --git a/ansible/playbooks/core-net.yml b/ansible/playbooks/core-net.yml index a5741e3..c2ab079 100644 --- a/ansible/playbooks/core-net.yml +++ b/ansible/playbooks/core-net.yml @@ -13,3 +13,5 @@ tags: [vmagent] - role: core-net/netbird tags: [netbird] + - role: vector + tags: [vector] diff --git a/ansible/playbooks/core-secrets.yml b/ansible/playbooks/core-secrets.yml index 610673c..f9981c9 100644 --- a/ansible/playbooks/core-secrets.yml +++ b/ansible/playbooks/core-secrets.yml @@ -9,6 +9,8 @@ tags: [vaultwarden] - role: core-secrets/cadvisor tags: [cadvisor] + - role: vector + tags: [vector] post_tasks: - name: Зарегистрировать Gitea в Agent Vault ansible.builtin.include_role: diff --git a/ansible/playbooks/ger-nod-02.yml b/ansible/playbooks/ger-nod-02.yml index 4aa9fa1..72861ac 100644 --- a/ansible/playbooks/ger-nod-02.yml +++ b/ansible/playbooks/ger-nod-02.yml @@ -9,3 +9,5 @@ tags: [remnanode] - role: vmagent tags: [vmagent] + - role: vector + tags: [vector] diff --git a/ansible/playbooks/rus-central.yml b/ansible/playbooks/rus-central.yml index c0b18a2..28e56e0 100644 --- a/ansible/playbooks/rus-central.yml +++ b/ansible/playbooks/rus-central.yml @@ -7,3 +7,5 @@ tags: [netbird] - role: rus-central/cadvisor tags: [cadvisor] + - role: vector + tags: [vector] diff --git a/ansible/roles/core-mon/monitoring/defaults/main.yml b/ansible/roles/core-mon/monitoring/defaults/main.yml index 7d71956..6baa0fb 100644 --- a/ansible/roles/core-mon/monitoring/defaults/main.yml +++ b/ansible/roles/core-mon/monitoring/defaults/main.yml @@ -10,6 +10,8 @@ monitoring_scrape_exclude_hosts: monitoring_node_exporter_port: 9100 victoriametrics_image: victoriametrics/victoria-metrics:v1.153.0 +victorialogs_image: victoriametrics/victoria-logs:v1.54.0 +victorialogs_retention: "14d" vmalert_image: victoriametrics/vmalert:v1.153.0 vmagent_image: victoriametrics/vmagent:v1.153.0 # Свой vmagent уже пишет node-exporter этих хостов. Повторный scrape даст двойные сэмплы. diff --git a/ansible/roles/core-mon/monitoring/tasks/compose.yml b/ansible/roles/core-mon/monitoring/tasks/compose.yml index b2d3c4a..9100871 100644 --- a/ansible/roles/core-mon/monitoring/tasks/compose.yml +++ b/ansible/roles/core-mon/monitoring/tasks/compose.yml @@ -34,6 +34,15 @@ delay: 2 until: victoriametrics_health.status == 200 +- name: Дождаться VictoriaLogs + ansible.builtin.uri: + url: "http://{{ ansible_host }}:9428/health" + status_code: 200 + register: victorialogs_health + retries: 20 + delay: 2 + until: victorialogs_health.status == 200 + - name: Дождаться vmalert ansible.builtin.uri: url: "http://127.0.0.1:8880/health" diff --git a/ansible/roles/core-mon/monitoring/tasks/dirs.yml b/ansible/roles/core-mon/monitoring/tasks/dirs.yml index 200b5c4..6fbfbb7 100644 --- a/ansible/roles/core-mon/monitoring/tasks/dirs.yml +++ b/ansible/roles/core-mon/monitoring/tasks/dirs.yml @@ -18,6 +18,7 @@ - { path: "{{ monitoring_dir }}/ntfy_data", owner: "root", group: "root", mode: "0777" } - { path: "{{ monitoring_dir }}/alertmanager-ntfy", owner: "root", group: "root", mode: "0755" } - { path: "{{ monitoring_dir }}/victoria-metrics-data", owner: "65534", group: "65534", mode: "0775" } + - { path: "{{ monitoring_dir }}/victoria-logs-data", owner: "65534", group: "65534", mode: "0775" } - { path: "{{ monitoring_dir }}/vmagent", owner: "root", group: "root", mode: "0755" } - { path: "{{ monitoring_dir }}/vmagent/conf.d", owner: "root", group: "root", mode: "0755" } - { path: "{{ monitoring_dir }}/vmalert", owner: "root", group: "root", mode: "0755" } diff --git a/ansible/roles/core-mon/monitoring/templates/compose.yaml.j2 b/ansible/roles/core-mon/monitoring/templates/compose.yaml.j2 index 82592fe..7c3890c 100644 --- a/ansible/roles/core-mon/monitoring/templates/compose.yaml.j2 +++ b/ansible/roles/core-mon/monitoring/templates/compose.yaml.j2 @@ -53,6 +53,7 @@ services: - GF_SECURITY_ADMIN_PASSWORD={{ grafana_admin_password }} - GF_USERS_ALLOW_SIGN_UP=false - GF_SERVER_ROOT_URL={{ grafana_root_url }} + - GF_INSTALL_PLUGINS=victoriametrics-logs-datasource volumes: - ./grafana_data:/var/lib/grafana - ./grafana/provisioning:/etc/grafana/provisioning:ro @@ -60,6 +61,7 @@ services: - "3000:3000" depends_on: - vmsingle + - vlogs vmsingle: image: {{ victoriametrics_image }} @@ -74,6 +76,19 @@ services: ports: - "{{ ansible_host }}:8428:8428" + vlogs: + image: {{ victorialogs_image }} + container_name: monitoring-vlogs + hostname: vlogs + restart: unless-stopped + command: + - -storageDataPath=/victoria-logs-data + - -retentionPeriod={{ victorialogs_retention }} + volumes: + - ./victoria-logs-data:/victoria-logs-data + ports: + - "{{ ansible_host }}:9428:9428" + cadvisor: image: {{ cadvisor_image }} container_name: monitoring-cadvisor diff --git a/ansible/roles/core-mon/monitoring/templates/grafana-datasources.yml.j2 b/ansible/roles/core-mon/monitoring/templates/grafana-datasources.yml.j2 index 69bdca8..8ce0ae3 100644 --- a/ansible/roles/core-mon/monitoring/templates/grafana-datasources.yml.j2 +++ b/ansible/roles/core-mon/monitoring/templates/grafana-datasources.yml.j2 @@ -19,3 +19,10 @@ datasources: jsonData: timeInterval: 15s + - name: VictoriaLogs + uid: victorialogs + type: victoriametrics-logs-datasource + access: proxy + url: http://vlogs:9428 + isDefault: false + diff --git a/ansible/roles/vector/defaults/main.yml b/ansible/roles/vector/defaults/main.yml new file mode 100644 index 0000000..a00da38 --- /dev/null +++ b/ansible/roles/vector/defaults/main.yml @@ -0,0 +1,6 @@ +--- +vector_dir: /opt/vector +vector_image: "timberio/vector:0.45.0-alpine" +vector_vlogs_host: "{{ hostvars['core-mon'].ansible_host }}" +vector_vlogs_url: "http://{{ vector_vlogs_host }}:9428/insert/jsonline" +vector_host_name: "{{ inventory_hostname }}" diff --git a/ansible/roles/vector/tasks/main.yml b/ansible/roles/vector/tasks/main.yml new file mode 100644 index 0000000..e629b6e --- /dev/null +++ b/ansible/roles/vector/tasks/main.yml @@ -0,0 +1,43 @@ +--- +- name: Создать каталоги Vector + ansible.builtin.file: + path: "{{ item }}" + state: directory + owner: root + group: root + mode: "0755" + loop: + - "{{ vector_dir }}" + - "{{ vector_dir }}/data" + +- name: Конфиг Vector + ansible.builtin.template: + src: vector.yaml.j2 + dest: "{{ vector_dir }}/vector.yaml" + owner: root + group: root + mode: "0644" + register: vector_config + +- name: Compose Vector + ansible.builtin.template: + src: compose.yaml.j2 + dest: "{{ vector_dir }}/compose.yaml" + owner: root + group: root + mode: "0644" + register: vector_compose + +- name: Запустить Vector + ansible.builtin.command: docker compose -f {{ vector_dir }}/compose.yaml up -d --remove-orphans + register: vector_up + changed_when: >- + 'Started' in vector_up.stderr or + 'Created' in vector_up.stderr or + 'Recreated' in vector_up.stderr or + 'Pulled' in vector_up.stderr + +- name: Перезапустить Vector после изменения конфигурации + ansible.builtin.command: docker compose -f {{ vector_dir }}/compose.yaml restart vector + when: vector_config is changed and not (vector_compose is changed) + changed_when: true diff --git a/ansible/roles/vector/templates/compose.yaml.j2 b/ansible/roles/vector/templates/compose.yaml.j2 new file mode 100644 index 0000000..3087569 --- /dev/null +++ b/ansible/roles/vector/templates/compose.yaml.j2 @@ -0,0 +1,15 @@ +name: vector + +services: + vector: + image: {{ vector_image }} + container_name: vector + restart: unless-stopped + command: + - "--config" + - "/etc/vector/vector.yaml" + volumes: + - ./vector.yaml:/etc/vector/vector.yaml:ro + - ./data:/var/lib/vector + - /var/run/docker.sock:/var/run/docker.sock:ro + - /var/log:/var/log:ro diff --git a/ansible/roles/vector/templates/vector.yaml.j2 b/ansible/roles/vector/templates/vector.yaml.j2 new file mode 100644 index 0000000..d756c0b --- /dev/null +++ b/ansible/roles/vector/templates/vector.yaml.j2 @@ -0,0 +1,57 @@ +data_dir: /var/lib/vector + +sources: + docker: + type: docker_logs + exclude_containers: + - vector + + host_logs: + type: file + include: + - /var/log/syslog + - /var/log/messages + - /var/log/auth.log + ignore_older_secs: 86400 + +transforms: + normalize_docker: + type: remap + inputs: + - docker + source: | + .host = "{{ vector_host_name }}" + .source_type = "docker" + .service = .container_name + if exists(.label."com.docker.compose.service") { + .service = string!(.label."com.docker.compose.service") + } + + normalize_host: + type: remap + inputs: + - host_logs + source: | + .host = "{{ vector_host_name }}" + .source_type = "host" + .service = replace(string!(.file), "/var/log/", "") + +sinks: + victorialogs: + type: http + inputs: + - normalize_docker + - normalize_host + uri: "{{ vector_vlogs_url }}" + encoding: + codec: json + framing: + method: newline_delimited + compression: gzip + request: + headers: + VL-Stream-Fields: "host,source_type,service" + VL-Msg-Field: "message" + VL-Time-Field: "timestamp" + healthcheck: + enabled: false