diff --git a/ansible/ci/ansible-target.sh b/ansible/ci/ansible-target.sh index dc29186..2ce06fc 100755 --- a/ansible/ci/ansible-target.sh +++ b/ansible/ci/ansible-target.sh @@ -77,7 +77,7 @@ if [ "$full" -eq 0 ] && [ -n "$changed" ]; then fi ;; ansible/roles/vmagent/*) - if grep -q "role: vmagent" "$playbook"; then + if grep -qE "role: (core-net/)?vmagent" "$playbook"; then add_tag vmagent fi ;; diff --git a/ansible/inventories/prod/host_vars/ger-nod-02.yml b/ansible/inventories/prod/host_vars/ger-nod-02.yml index 8bd32c0..8a4c2ce 100644 --- a/ansible/inventories/prod/host_vars/ger-nod-02.yml +++ b/ansible/inventories/prod/host_vars/ger-nod-02.yml @@ -8,6 +8,7 @@ remnanode_panel_egress_ips: - "185.160.137.140" # провайдер 2 monitoring_scrape_enabled: false monitoring_node_exporter_public: false -monitoring_node_exporter_listen: "127.0.0.1:9100" +monitoring_node_exporter_listen: "0.0.0.0:9100" +vmagent_pull_only_target: true # Docker на этой машине не принимает API 1.41, которым говорит cAdvisor 0.49. cadvisor_image: ghcr.io/google/cadvisor:v0.60.6 diff --git a/ansible/roles/netbird_exit_node/tasks/main.yml b/ansible/roles/netbird_exit_node/tasks/main.yml index 5ee13c4..e0569af 100644 --- a/ansible/roles/netbird_exit_node/tasks/main.yml +++ b/ansible/roles/netbird_exit_node/tasks/main.yml @@ -32,13 +32,14 @@ changed_when: "'added' in (ufw_netbird_wg.stdout | lower) or 'deleted' in (ufw_netbird_wg.stdout | lower)" failed_when: false -- name: Ограничить исходящий доступ с exit node в домашнюю LAN (только VictoriaMetrics на core-mon) +- name: Запретить любой исходящий доступ с exit node в домашнюю LAN (defense-in-depth) ansible.builtin.command: "{{ item }}" loop: - - ufw allow out proto tcp from any to 192.168.1.108 port 8428 + - ufw delete allow out proto tcp from any to 192.168.1.108 port 8428 - ufw deny out to 192.168.1.0/24 register: ufw_lan_restrict - changed_when: "'added' in (ufw_lan_restrict.stdout | lower)" + changed_when: "'added' in (ufw_lan_restrict.stdout | lower) or 'deleted' in (ufw_lan_restrict.stdout | lower)" + failed_when: false - name: Клиент NetBird на ноде ansible.builtin.include_role: @@ -116,6 +117,18 @@ fi fi + # Удалить внешнюю ноду из группы remnawave (панель), чтобы у неё не было прав панели + remna_grp_obj=$(printf '%s' "${groups_json}" | jq -c '.[] | select(.name == "remnawave")' | head -n 1) + if [ -n "${remna_grp_obj}" ]; then + remna_grp_id=$(printf '%s' "${remna_grp_obj}" | jq -r '.id') + in_remna=$(printf '%s' "${remna_grp_obj}" | jq -r --arg pid "${ger_peer_id}" '[.peers[]?.id] | index($pid) != null') + if [ "${in_remna}" = "true" ]; then + remna_upd=$(printf '%s' "${remna_grp_obj}" | jq -c --arg pid "${ger_peer_id}" '{name: .name, peers: ([.peers[]?.id | select(. != $pid)] | unique)}') + nb_api_call PUT "/groups/${remna_grp_id}" "${remna_upd}" >/dev/null + exit_group_updated=1 + fi + fi + exit_clients_group_id=$(printf '%s' "${groups_json}" | jq -r --arg g "${NB_EXIT_CLIENTS_GROUP}" '.[] | select(.name == $g) | .id' | head -n 1) if [ -n "${exit_clients_group_id}" ]; then routes_json=$(curl -fsS --max-time 20 -H "Authorization: Token ${NB_API_TOKEN}" "${API_BASE}/routes") diff --git a/ansible/roles/opnsense/netbird/tasks/main.yml b/ansible/roles/opnsense/netbird/tasks/main.yml index 4f9960f..7f65bd5 100644 --- a/ansible/roles/opnsense/netbird/tasks/main.yml +++ b/ansible/roles/opnsense/netbird/tasks/main.yml @@ -58,10 +58,10 @@ (opnsense_nb_current_settings.general.enable | string) != '1' or (opnsense_nb_current_settings.general.wireguardPort | string) != (opnsense_netbird_wg_port | string) or (opnsense_nb_current_settings.firewall.allowConfig | string) != '1' or - (opnsense_nb_current_settings.firewall.blockInboundConnection | string) != '0' or + (opnsense_nb_current_settings.firewall.blockInboundConnection | string) != '1' or (opnsense_nb_current_settings.ssh.enable | string) != (opnsense_netbird_ssh_enable | string) or (opnsense_nb_current_settings.dns.enable | string) != '1' or - (opnsense_nb_current_settings.routing.accessLan | string) != '1' or + (opnsense_nb_current_settings.routing.accessLan | string) != '0' or (opnsense_nb_current_settings.routing.acceptClientRoutes | string) != '1' or (opnsense_nb_current_settings.routing.acceptServerRoutes | string) != '1' }} @@ -80,13 +80,13 @@ wireguardPort: "{{ opnsense_netbird_wg_port }}" firewall: allowConfig: "1" - blockInboundConnection: "0" + blockInboundConnection: "1" ssh: enable: "{{ opnsense_netbird_ssh_enable }}" dns: enable: "1" routing: - accessLan: "1" + accessLan: "0" acceptClientRoutes: "1" acceptServerRoutes: "1" postquantum: diff --git a/ansible/roles/rus-central/netbird/files/access.yml b/ansible/roles/rus-central/netbird/files/access.yml index a0ca63b..9e90ae5 100644 --- a/ansible/roles/rus-central/netbird/files/access.yml +++ b/ansible/roles/rus-central/netbird/files/access.yml @@ -1,12 +1,14 @@ -# Декларативное описание клиентского доступа NetBird. +# Декларативное описание клиентского доступа и политик NetBird. # Применяется задачей rus-central/netbird (tasks/access.yml → sync_access.py). # # Секреты setup-ключей сохраняются в Infisical (/services/netbird) автоматически # и не попадают в Git или логи CI. -# Группы клиентов +# Управляемые группы пиров client_groups: - exit-clients + - remnawave + - exit-ger-nod-02 # Выходные шлюзы (Exit Nodes) exit_nodes: @@ -35,8 +37,6 @@ dns_nameservers: enabled: false # Ключи подключения клиентов (секреты сохраняются в Infisical /services/netbird). -# Чтобы выпустить отдельный ключ — добавьте запись с новым name и secret. -# Чтобы отозвать ключ — поставьте enabled: false или удалите запись. setup_keys: - name: exit-clients-key secret: NETBIRD_EXIT_CLIENTS_SETUP_KEY @@ -48,11 +48,75 @@ setup_keys: - exit-clients enabled: true -# Управление подключёнными клиентскими устройствами (пирами). -# Новые пиры, подключившиеся по setup-ключу, сохраняют свои группы автоматически. -# Здесь можно явно зафиксировать группы пира или удалить/отключить устройство (state: absent). +# Управление подключёнными пирами (жёсткое разделение: внешние ноды не входят в remnawave). peers: - name: "A55 пользователя Vacheslav" groups: - exit-clients state: present + - name: "core-net" + groups: + - remnawave + state: present + - name: "ger-nod-02" + groups: + - exit-ger-nod-02 + state: present + +# Zero-Trust политики доступа (заменяют дефолтную политику Default All <-> All). +# Инициировать соединения в сторону внешних нод могут только панель (core-net) и клиенты (exit-clients). +# Сами внешние ноды не могут открыть ни одного соединения внутрь домашней сети или к пирам. +policies: + - name: "remnawave-panel-to-exit-nodes" + description: "One-way access from Remnawave panel (core-net) to Exit Nodes (NODE_PORT + metrics)" + enabled: true + rules: + - name: "panel-to-node-tcp" + description: "Allow TCP 2222 (remnanode), 9100 (node-exporter), 9101 (cAdvisor) from core-net" + enabled: true + action: accept + protocol: tcp + bidirectional: false + ports: + - "2222" + - "9100" + - "9101" + sources: + - remnawave + destinations: + - exit-ger-nod-02 + + - name: "exit-clients-to-exit-nodes" + description: "One-way traffic from Exit Node clients to Exit Nodes" + enabled: true + rules: + - name: "exit-clients-tcp" + description: "Allow outbound TCP from exit-clients via exit node" + enabled: true + action: accept + protocol: tcp + bidirectional: false + sources: + - exit-clients + destinations: + - exit-ger-nod-02 + - name: "exit-clients-udp" + description: "Allow outbound UDP from exit-clients via exit node" + enabled: true + action: accept + protocol: udp + bidirectional: false + sources: + - exit-clients + destinations: + - exit-ger-nod-02 + - name: "exit-clients-icmp" + description: "Allow ICMP for exit-clients via exit node" + enabled: true + action: accept + protocol: icmp + bidirectional: true + sources: + - exit-clients + destinations: + - exit-ger-nod-02 diff --git a/ansible/roles/rus-central/netbird/files/sync_access.py b/ansible/roles/rus-central/netbird/files/sync_access.py index b12e240..57560da 100644 --- a/ansible/roles/rus-central/netbird/files/sync_access.py +++ b/ansible/roles/rus-central/netbird/files/sync_access.py @@ -309,6 +309,88 @@ def sync(cfg: dict[str, Any], api: NetBirdAPI) -> dict[str, Any]: updated_secrets[secret_var] = plain_key changes.append(f"setup_key.reissue:{name}") + # 6. Политики доступа (Zero-Trust ACL) + if "policies" in cfg: + policies_raw = api.get("/policies") or [] + existing_by_name = {p.get("name"): p for p in policies_raw if p.get("name")} + desired_policy_names: set[str] = set() + + def _extract_group_ids(items: list[Any]) -> list[str]: + out: list[str] = [] + for x in items or []: + if isinstance(x, dict) and x.get("id"): + out.append(str(x["id"])) + elif isinstance(x, str): + out.append(x) + return sorted(out) + + for p_cfg in cfg.get("policies") or []: + p_name = p_cfg["name"] + desired_policy_names.add(p_name) + rules_payload: list[dict[str, Any]] = [] + for r_cfg in p_cfg.get("rules") or []: + src_ids = sorted(ensure_group(g)["id"] for g in (r_cfg.get("sources") or [])) + dst_ids = sorted(ensure_group(g)["id"] for g in (r_cfg.get("destinations") or [])) + rule_obj: dict[str, Any] = { + "name": r_cfg["name"], + "description": r_cfg.get("description", ""), + "enabled": bool(r_cfg.get("enabled", True)), + "action": r_cfg.get("action", "accept"), + "protocol": r_cfg.get("protocol", "tcp"), + "bidirectional": bool(r_cfg.get("bidirectional", False)), + "sources": src_ids, + "destinations": dst_ids, + } + if r_cfg.get("ports"): + rule_obj["ports"] = [str(p) for p in r_cfg["ports"]] + rules_payload.append(rule_obj) + + desired_pol = { + "name": p_name, + "description": p_cfg.get("description", ""), + "enabled": bool(p_cfg.get("enabled", True)), + "rules": rules_payload, + } + + ex_pol = existing_by_name.get(p_name) + if ex_pol is None: + api.post("/policies", desired_pol) + changes.append(f"policy.create:{p_name}") + else: + ex_rules = ex_pol.get("rules") or [] + norm_ex_rules = [ + { + "name": r.get("name", ""), + "description": r.get("description", ""), + "enabled": bool(r.get("enabled", True)), + "action": r.get("action", "accept"), + "protocol": r.get("protocol", "tcp"), + "bidirectional": bool(r.get("bidirectional", False)), + "sources": _extract_group_ids(r.get("sources") or []), + "destinations": _extract_group_ids(r.get("destinations") or []), + **( + {"ports": [str(p) for p in r["ports"]]} + if r.get("ports") + else {} + ), + } + for r in ex_rules + ] + needs_pol_update = ( + (ex_pol.get("description") or "") != desired_pol["description"] + or bool(ex_pol.get("enabled", True)) != desired_pol["enabled"] + or norm_ex_rules != rules_payload + ) + if needs_pol_update: + api.put(f"/policies/{ex_pol['id']}", desired_pol) + changes.append(f"policy.update:{p_name}") + + # Удаляем дефолтную (All <-> All) и любые лишние политики после создания целевых + for ex_name, ex_pol in existing_by_name.items(): + if ex_name not in desired_policy_names and ex_pol.get("id"): + api.delete(f"/policies/{ex_pol['id']}") + changes.append(f"policy.delete:{ex_name}") + return { "changed": bool(changes or updated_secrets), "changes": changes, diff --git a/ansible/roles/rus-central/netbird/tasks/bootstrap.yml b/ansible/roles/rus-central/netbird/tasks/bootstrap.yml index d9625ad..c26dba0 100644 --- a/ansible/roles/rus-central/netbird/tasks/bootstrap.yml +++ b/ansible/roles/rus-central/netbird/tasks/bootstrap.yml @@ -145,40 +145,9 @@ routes_json=$(curl "${CURL_OPTS[@]}" -H "Authorization: Token ${active_token}" "${API_BASE}/routes") existing_route=$(printf '%s' "${routes_json}" | jq -c --arg nid "$NB_LAN_NET_ID" --arg net "$NB_LAN_SUBNET" '.[] | select(.network_id == $nid or .network == $net)' | head -n 1) existing_route_id=$(printf '%s' "${existing_route}" | jq -r '.id // empty' 2>/dev/null || true) - needs_lan_route_update="false" if [ -n "${existing_route_id}" ]; then - needs_lan_route_update=$(printf '%s' "${existing_route}" | jq -r --arg rmg "$remna_group_id" '((.groups // []) | sort) != ([$rmg] | sort)') - fi - - if [ -z "${existing_route_id}" ] || [ "${needs_lan_route_update}" = "true" ]; then - route_payload=$(jq -n \ - --arg desc "Homelab LAN (${NB_LAN_SUBNET} via OPNsense)" \ - --arg nid "$NB_LAN_NET_ID" \ - --arg net "$NB_LAN_SUBNET" \ - --arg rg "$router_group_id" \ - --arg rmg "$remna_group_id" \ - '{ - description: $desc, - network_id: $nid, - enabled: true, - peer_groups: [$rg], - network: $net, - metric: 100, - masquerade: true, - groups: [$rmg], - keep_route: true - }') - if [ -z "${existing_route_id}" ]; then - curl "${CURL_OPTS[@]}" -X POST "${API_BASE}/routes" \ - -H "Authorization: Token ${active_token}" \ - -H "Content-Type: application/json" \ - -d "${route_payload}" >/dev/null - else - curl "${CURL_OPTS[@]}" -X PUT "${API_BASE}/routes/${existing_route_id}" \ - -H "Authorization: Token ${active_token}" \ - -H "Content-Type: application/json" \ - -d "${route_payload}" >/dev/null - fi + curl "${CURL_OPTS[@]}" -X DELETE "${API_BASE}/routes/${existing_route_id}" \ + -H "Authorization: Token ${active_token}" >/dev/null || true route_created=1 fi diff --git a/ansible/roles/vmagent/defaults/main.yml b/ansible/roles/vmagent/defaults/main.yml index 75b00d7..1e75183 100644 --- a/ansible/roles/vmagent/defaults/main.yml +++ b/ansible/roles/vmagent/defaults/main.yml @@ -9,6 +9,8 @@ vmagent_remote_health_url: "http://{{ vmagent_remote_host }}:8428/health" vmagent_instance: "{{ inventory_hostname }}" vmagent_cluster: homelab vmagent_scrape_remnawave: false +vmagent_pull_only_target: false +vmagent_cadvisor_listen_ip: "{{ '0.0.0.0' if (vmagent_pull_only_target | bool) else '127.0.0.1' }}" vmagent_metrics_user: admin vmagent_metrics_pass: "{{ lookup('env', 'REMNAWAVE_METRICS_PASS', default='') }}" vmagent_metrics_port: 3001 diff --git a/ansible/roles/vmagent/tasks/main.yml b/ansible/roles/vmagent/tasks/main.yml index 3d0eb49..dea7150 100644 --- a/ansible/roles/vmagent/tasks/main.yml +++ b/ansible/roles/vmagent/tasks/main.yml @@ -24,6 +24,52 @@ enabled: true when: vmagent_exporter_env is changed +- name: Узнать NetBird IP панели (core-net) для pull-доступа к метрикам + ansible.builtin.shell: | + set -euo pipefail + netbird status --json | jq -r '.peers.details[]? | select(.fqdn | startswith("core-net.")) | .netbirdIp' | head -n 1 + args: + executable: /bin/bash + register: vmagent_panel_nb_ip + changed_when: false + failed_when: false + when: vmagent_pull_only_target | bool + +- name: Разрешить pull метрик (9100, 9101) только с NetBird IP панели (core-net) + ansible.builtin.command: > + ufw allow from {{ vmagent_panel_nb_ip.stdout | trim }} to any port {{ item }} proto tcp + loop: + - 9100 + - 9101 + register: ufw_pull_metrics + changed_when: "'added' in (ufw_pull_metrics.stdout | default('') | lower)" + when: + - vmagent_pull_only_target | bool + - vmagent_panel_nb_ip.stdout | default('') | trim is match('^[0-9.]+$') + +- name: Собрать NetBird IP выходных нод (exit_nodes) для pull с core-net + ansible.builtin.shell: | + set -euo pipefail + nb_json=$(netbird status --json) + jq -c -n --argjson nb "${nb_json}" --argjson hosts '{{ (groups["exit_nodes"] | default([])) | to_json }}' ' + [ + $hosts[] as $h + | ($nb.peers.details[]? | select(.fqdn | startswith($h + ".")) | .netbirdIp) as $ip + | select($ip != null and $ip != "") + | {name: $h, ip: $ip} + ] + ' + args: + executable: /bin/bash + register: vmagent_exit_targets_raw + changed_when: false + when: vmagent_scrape_remnawave | bool + +- name: Задать список NetBird pull-целей для vmagent + ansible.builtin.set_fact: + vmagent_netbird_exit_targets: "{{ vmagent_exit_targets_raw.stdout | from_json }}" + when: vmagent_scrape_remnawave | bool + - name: Дождаться VictoriaMetrics ansible.builtin.command: curl -fsS --max-time 5 {{ vmagent_remote_health_url }} register: vmagent_vm_health @@ -32,14 +78,16 @@ delay: 5 until: vmagent_vm_health.rc == 0 failed_when: false + when: not (vmagent_pull_only_target | bool) - name: VictoriaMetrics недоступна ansible.builtin.fail: msg: >- {{ vmagent_remote_health_url }} не отвечает. Сначала поднимите VictoriaMetrics на core-mon. - С ger-nod-02 этот адрес открывается маршрутом homelab-lan через OPNsense. - when: vmagent_vm_health.rc != 0 + when: + - not (vmagent_pull_only_target | bool) + - vmagent_vm_health.rc != 0 - name: Каталоги vmagent ansible.builtin.file: @@ -64,6 +112,7 @@ - { src: nodeexporter.yml.j2, dest: "{{ vmagent_dir }}/conf.d/nodeexporter.yml", mode: "0644" } - { src: cadvisor.yml.j2, dest: "{{ vmagent_dir }}/conf.d/cadvisor.yml", mode: "0644" } - { src: compose.yaml.j2, dest: "{{ vmagent_dir }}/compose.yaml", mode: "0644" } + register: vmagent_configs - name: Убрать scrape панели, если он не нужен ansible.builtin.file: @@ -81,14 +130,31 @@ no_log: true when: vmagent_scrape_remnawave | bool -- name: Запустить vmagent и cAdvisor +- name: Запустить контейнеры мониторинга (vmagent / cAdvisor) ansible.builtin.command: docker compose -f {{ vmagent_dir }}/compose.yaml up -d --remove-orphans register: vmagent_up changed_when: >- 'Started' in vmagent_up.stderr or 'Created' in vmagent_up.stderr or 'Recreated' in vmagent_up.stderr or - 'Pulled' in vmagent_up.stderr + 'Pulled' in vmagent_up.stderr or + 'Removed' in vmagent_up.stderr + +- name: Перезапустить vmagent при обновлении целей scrape + ansible.builtin.command: docker compose -f {{ vmagent_dir }}/compose.yaml restart vmagent + when: + - not (vmagent_pull_only_target | bool) + - vmagent_configs is changed + changed_when: true + +- name: Дождаться локального cAdvisor + ansible.builtin.uri: + url: http://127.0.0.1:9101/healthz + status_code: 200 + register: vmagent_cadvisor_local + retries: 15 + delay: 2 + until: vmagent_cadvisor_local.status == 200 - name: Дождаться локального vmagent ansible.builtin.uri: @@ -98,9 +164,14 @@ retries: 15 delay: 2 until: vmagent_local.status == 200 + when: not (vmagent_pull_only_target | bool) - name: Итог vmagent ansible.builtin.debug: msg: - - "vmagent {{ inventory_hostname }} пишет в {{ vmagent_remote_write_url }}" - - "instance={{ vmagent_instance }} scrape панели={{ vmagent_scrape_remnawave | bool }}" + - >- + {{ + 'Pull-таргет ' ~ inventory_hostname ~ ': node_exporter (:9100) и cAdvisor (:9101) доступны только по NetBird с core-net' + if (vmagent_pull_only_target | bool) + else 'vmagent ' ~ inventory_hostname ~ ' пишет в ' ~ vmagent_remote_write_url ~ ' (exit_targets=' ~ ((vmagent_netbird_exit_targets | default([])) | length) ~ ')' + }} diff --git a/ansible/roles/vmagent/templates/cadvisor.yml.j2 b/ansible/roles/vmagent/templates/cadvisor.yml.j2 index 77fec48..dbb9928 100644 --- a/ansible/roles/vmagent/templates/cadvisor.yml.j2 +++ b/ansible/roles/vmagent/templates/cadvisor.yml.j2 @@ -5,3 +5,9 @@ labels: instance: "{{ vmagent_instance }}" cluster: "{{ vmagent_cluster }}" +{% for target in (vmagent_netbird_exit_targets | default([])) %} + - targets: ["{{ target.ip }}:9101"] + labels: + instance: "{{ target.name }}" + cluster: "{{ vmagent_cluster }}" +{% endfor %} diff --git a/ansible/roles/vmagent/templates/compose.yaml.j2 b/ansible/roles/vmagent/templates/compose.yaml.j2 index 2f66ba7..d4d01db 100644 --- a/ansible/roles/vmagent/templates/compose.yaml.j2 +++ b/ansible/roles/vmagent/templates/compose.yaml.j2 @@ -6,7 +6,7 @@ services: privileged: true restart: unless-stopped command: - - --listen_ip=127.0.0.1 + - --listen_ip={{ vmagent_cadvisor_listen_ip }} - --port=9101 - --docker_only=true volumes: @@ -15,6 +15,7 @@ services: - /sys:/sys:ro - /var/lib/docker:/var/lib/docker:ro - /dev/disk:/dev/disk:ro +{% if not (vmagent_pull_only_target | bool) %} vmagent: image: {{ vmagent_image }} @@ -31,3 +32,4 @@ services: - ./conf.d:/etc/vmagent/conf.d:ro depends_on: - cadvisor +{% endif %} diff --git a/ansible/roles/vmagent/templates/nodeexporter.yml.j2 b/ansible/roles/vmagent/templates/nodeexporter.yml.j2 index 6feaa63..8653b02 100644 --- a/ansible/roles/vmagent/templates/nodeexporter.yml.j2 +++ b/ansible/roles/vmagent/templates/nodeexporter.yml.j2 @@ -5,3 +5,10 @@ labels: instance: "{{ vmagent_instance }}" cluster: "{{ vmagent_cluster }}" +{% for target in (vmagent_netbird_exit_targets | default([])) %} + - targets: ["{{ target.ip }}:9100"] + labels: + instance: "{{ target.name }}" + host: "{{ target.name }}" + cluster: "{{ vmagent_cluster }}" +{% endfor %}