agent.yml

---
# agent.sh runs the whole thing. A single stage runs via tags:
#   uv run --frozen ansible-playbook agent.yml --tags deploy
- name: Provision and deploy the CompassVPN host
  hosts: localhost
  connection: local
  become: true
  gather_facts: false   # gathered later, once python3 is guaranteed

  vars:
    ansible_python_interpreter: /usr/bin/python3
    project_dir: "{{ playbook_dir }}"

    # agent.sh passes these in from env_file; empty means no cron job.
    auto_update: ""
    redeploy_interval: ""

    # Priority order, and unreachable ones are dropped before we write
    # resolv.conf, so we never swap working DNS for dead entries.
    dns_servers:
      - "127.0.0.53"
      - "1.1.1.2"

    # The metrics ports (29100, 25000, 29550) belong nowhere near this list -
    # they're loopback-only and scraped over loopback.
    required_ports:
      - 80
      - 8080
      - 443
      - 2053
      - 2083
      - 2087
      - 8443
      - 8880
      - 2052
      - 8000

    # Host netns only. Containers don't inherit most of these, so the keys the
    # proxy needs are set again in docker-compose.yml.
    sysctl_conf_path: /etc/sysctl.d/99-compassvpn.conf
    sysctl_settings:
      fs.file-max: "67108864"
      net.core.default_qdisc: "fq"
      net.ipv4.tcp_congestion_control: "bbr"
      net.ipv4.tcp_max_syn_backlog: "10240"
      net.ipv4.tcp_fin_timeout: "25"
      net.core.netdev_max_backlog: "32768"
      net.netfilter.nf_conntrack_max: "1048576"
      net.netfilter.nf_conntrack_buckets: "262144"
      # Default is 5 days, so dead flows pile up. 7440s is just over the 2h
      # keepalive Go uses by default, so live flows keep refreshing.
      net.netfilter.nf_conntrack_tcp_timeout_established: "7440"
      # Conntrack entries only; socket TIME_WAIT is separate.
      net.netfilter.nf_conntrack_tcp_timeout_time_wait: "30"

    compassvpn_log_path: /var/log/compassvpn
    compassvpn_log_files:
      - nginx_access.log
      - nginx_error.log
      - xray_access.log
      - xray_error.log
      - xray.log
      - debug.log

  tasks:
    # ---- prerequisites (every tag) -----------------------------------------
    # Minimal images ship without python3 / python3-apt, and raw is the only
    # module that works before they're there.
    - name: Ensure python3 and python3-apt are present
      ansible.builtin.raw: dpkg -s python3 python3-apt >/dev/null 2>&1 || (apt-get update && apt-get install -y python3 python3-apt)
      changed_when: false
      tags: always

    - name: Gather facts
      ansible.builtin.setup: {}
      tags: always

    # The floor is ansible-core needing Python 3.9+, which rules out Ubuntu 20.04.
    - name: Check supported OS and version (Debian 11+, Ubuntu 22.04+)
      ansible.builtin.assert:
        that:
          - ansible_facts['distribution'] | lower in ['ubuntu', 'debian']
          - >-
            (ansible_facts['distribution'] | lower == 'debian' and
             ansible_facts['distribution_major_version'] | int >= 11)
            or
            (ansible_facts['distribution'] | lower == 'ubuntu' and
             ansible_facts['distribution_major_version'] | int >= 22)
        fail_msg: >-
          Unsupported OS: {{ ansible_facts['distribution'] }} {{ ansible_facts['distribution_version'] }}.
          This playbook supports Debian 11+ and Ubuntu 22.04+ (ansible-core needs Python 3.9+ on the host).
        success_msg: >-
          {{ ansible_facts['distribution'] }} {{ ansible_facts['distribution_version'] }} is supported.
      tags: always

    # ---- prepare: packages, DNS, tuning, firewall, logs --------------------
    - name: Prepare the host
      tags: prepare
      block:
        - name: Detect OpenVZ (its kernel can't take the sysctl tuning)
          ansible.builtin.stat:
            path: /proc/user_beancounters
          register: openvz_file

        - name: Note when OpenVZ is detected
          ansible.builtin.debug:
            msg: "OpenVZ detected, skipping the kernel tuning"
          changed_when: true   # so it still shows when agent.sh hides ok tasks
          when: openvz_file.stat.exists

        - name: Install base packages
          ansible.builtin.apt:
            update_cache: true
            cache_valid_time: 3600
            name:
              - curl
              - wget
              - sudo
              - coreutils
              - iproute2
              - lsof
              - logrotate
            state: present

        # Our own ports show up as docker-proxy, so filter those out or a re-run
        # on a live box fails on itself. Match the socket cgroup too - the
        # process name goes blank while docker restarts, the cgroup never does.
        - name: Fail early if a required port is held by a non-Docker process
          ansible.builtin.shell: |
            set -o pipefail
            ss -tlpneH "sport = :{{ item }}" | grep -v -e docker-proxy -e 'cgroup:/system.slice/docker.service' || true
          args:
            executable: /bin/bash
          register: port_check
          changed_when: false
          failed_when: port_check.stdout | trim | length > 0
          retries: 3
          delay: 2
          until: port_check.stdout | trim | length == 0
          loop: "{{ required_ports }}"
          loop_control:
            label: "{{ item }}"

        - name: Ensure hostname entry exists in /etc/hosts
          ansible.builtin.lineinfile:
            path: /etc/hosts
            regexp: '^127\.0\.1\.1\s'
            line: "127.0.1.1 {{ ansible_facts['hostname'] }}"
            state: present

        - name: Probe configured DNS servers
          ansible.builtin.command: "ping -c1 -W1 {{ item }}"
          register: dns_ping
          changed_when: false
          failed_when: false
          loop: "{{ dns_servers }}"
          loop_control:
            label: "{{ item }}"

        - name: Keep only the reachable DNS servers
          ansible.builtin.set_fact:
            reachable_dns: "{{ dns_ping.results | selectattr('rc', 'equalto', 0) | map(attribute='item') | list }}"

        - name: Back up /etc/resolv.conf once
          ansible.builtin.copy:
            src: /etc/resolv.conf
            dest: /etc/resolv.conf.bak
            remote_src: true
            mode: "0644"
            force: false
          when: reachable_dns | length > 0

        - name: Write reachable DNS servers to /etc/resolv.conf
          ansible.builtin.copy:
            dest: /etc/resolv.conf
            mode: "0644"
            content: |
              {% for dns in reachable_dns %}
              nameserver {{ dns }}
              {% endfor %}
          when: reachable_dns | length > 0

        - name: Warn when no DNS server is reachable (leaving resolv.conf untouched)
          ansible.builtin.debug:
            msg: "No DNS server answered ({{ dns_servers | join(', ') }}), leaving /etc/resolv.conf as is"
          changed_when: true   # so it still shows when agent.sh hides ok tasks
          when: reachable_dns | length == 0

        - name: Set timezone to UTC
          community.general.timezone:
            name: Etc/UTC

        - name: Load nf_conntrack (needed by the conntrack sysctls)
          community.general.modprobe:
            name: nf_conntrack
            state: present
          failed_when: false
          when: not openvz_file.stat.exists

        - name: Apply sysctl tuning
          ansible.posix.sysctl:
            name: "{{ item.key }}"
            value: "{{ item.value }}"
            sysctl_file: "{{ sysctl_conf_path }}"
            sysctl_set: true
            reload: false
            ignoreerrors: true   # some kernels reject a key (e.g. conntrack); don't abort
            state: present
          loop: "{{ sysctl_settings | dict2items }}"
          loop_control:
            label: "{{ item.key }}"
          when: not openvz_file.stat.exists

        - name: Reload sysctl from all config files
          ansible.builtin.command: sysctl --system
          changed_when: false
          when: not openvz_file.stat.exists

        - name: Ensure UFW is installed
          ansible.builtin.apt:
            name: ufw
            state: present

        # Otherwise UFW reapplies its own sysctl file and undoes ip_forward,
        # which breaks Docker networking.
        - name: Point UFW at the system sysctl file
          ansible.builtin.replace:
            path: /etc/default/ufw
            regexp: '/etc/ufw/sysctl\.conf'
            replace: '/etc/sysctl.conf'

        - name: Detect the active SSH port
          ansible.builtin.shell: |
            set -o pipefail
            ss -tlnpH | grep -oP '(?<=:)\d+(?= .*sshd)' | head -n1
          args:
            executable: /bin/bash
          register: active_ssh_port
          changed_when: false
          failed_when: false

        - name: Fall back to the configured SSH port (sshd_config + drop-ins)
          ansible.builtin.shell: |
            set -o pipefail
            grep -rhoP '^Port\s+\K\d+' /etc/ssh/sshd_config /etc/ssh/sshd_config.d/ 2>/dev/null | head -n1
          args:
            executable: /bin/bash
          register: configured_ssh_port
          changed_when: false
          failed_when: false
          when: (active_ssh_port.stdout | trim) == ''

        - name: Settle on the SSH port (active -> configured -> 22)
          ansible.builtin.set_fact:
            ssh_port: >-
              {{
                (active_ssh_port.stdout | trim)
                if (active_ssh_port.stdout | trim) != ''
                else ((configured_ssh_port.stdout | default('') | trim) or '22')
              }}

        # Must come before UFW is enabled, or the deploy locks itself out.
        - name: Allow the SSH port in UFW
          community.general.ufw:
            rule: allow
            port: "{{ ssh_port }}"
            proto: tcp

        - name: Default deny incoming
          community.general.ufw:
            direction: incoming
            policy: deny

        - name: Default allow outgoing
          community.general.ufw:
            direction: outgoing
            policy: allow

        - name: Allow the required service ports (tcp + udp)
          community.general.ufw:
            rule: allow
            port: "{{ item.0 }}"
            proto: "{{ item.1 }}"
          loop: "{{ required_ports | product(['tcp', 'udp']) | list }}"
          loop_control:
            label: "{{ item.0 }}/{{ item.1 }}"

        - name: Enable UFW
          community.general.ufw:
            state: enabled

        - name: Ensure CompassVPN log directory exists
          ansible.builtin.file:
            path: "{{ compassvpn_log_path }}"
            state: directory
            owner: root
            group: root
            mode: "0750"

        - name: Ensure CompassVPN log files exist
          ansible.builtin.file:
            path: "{{ compassvpn_log_path }}/{{ item }}"
            state: touch
            owner: root
            group: root
            mode: "0640"
            modification_time: preserve
            access_time: preserve
          loop: "{{ compassvpn_log_files }}"

        - name: Ensure Nginx log directory exists
          ansible.builtin.file:
            path: /var/log/nginx
            state: directory
            mode: "0755"

        - name: Ensure Nginx error log exists
          ansible.builtin.file:
            path: /var/log/nginx/error.log
            state: touch
            owner: root
            group: root
            mode: "0640"
            modification_time: preserve
            access_time: preserve

        - name: Configure CompassVPN logrotate
          ansible.builtin.copy:
            dest: /etc/logrotate.d/compassvpn
            mode: "0644"
            content: |
              /var/log/compassvpn/*.log {
                  size 100M
                  rotate 2
                  compress
                  delaycompress
                  missingok
                  notifempty
                  copytruncate
                  dateext
                  dateformat -%Y%m%d-%s
              }

        - name: Check the size rule hourly, since logrotate.timer only runs daily
          ansible.builtin.cron:
            name: compassvpn-logrotate
            special_time: hourly
            job: "/usr/sbin/logrotate /etc/logrotate.d/compassvpn"

        - name: Enable and start logrotate.timer when available
          ansible.builtin.systemd_service:
            name: logrotate.timer
            enabled: true
            state: started
          failed_when: false

        - name: Validate the logrotate config (dry run, no rotation)
          ansible.builtin.command: logrotate -d /etc/logrotate.d/compassvpn
          changed_when: false
          failed_when: false

    # ---- docker ------------------------------------------------------------
    - name: Install Docker
      tags: docker
      block:
        - name: Check whether Docker is installed
          ansible.builtin.command: docker --version
          register: docker_check
          changed_when: false
          failed_when: false

        - name: Install Docker via the official convenience script
          ansible.builtin.shell: curl -fsSL https://get.docker.com | sh
          args:
            executable: /bin/bash
          when: docker_check.rc != 0

    # ---- docker network: non-conflicting address pools ---------------------
    # Docker's default 172.16.0.0/12 collides with VNets in that block (common on
    # Azure) and wedges networking, so bip moves docker0 and the pools move the
    # compose network, both into a quiet 10.x range. The v6 pool is what
    # enable_ipv6 in docker-compose.yml draws from.
    - name: Point Docker at non-conflicting address pools
      tags: [docker, deploy]   # 'deploy' too, so the daemon is fixed before containers start
      block:
        - name: Write /etc/docker/daemon.json
          ansible.builtin.copy:
            dest: /etc/docker/daemon.json
            mode: "0644"
            content: |
              {
                "bip": "10.69.0.1/24",
                "default-address-pools": [
                  { "base": "10.70.0.0/16", "size": 24 },
                  { "base": "10.169.0.0/16", "size": 24 },
                  { "base": "10.199.0.0/16", "size": 24 },
                  { "base": "fd00:c0:ffee::/48", "size": 64 }
                ]
              }
          register: docker_daemon

        - name: Restart Docker when the config changed
          ansible.builtin.systemd_service:
            name: docker
            state: restarted
          when: docker_daemon.changed

    # ---- identifier: a stable per-node label, generated once ---------------
    # Tagged 'deploy' as well, since the services hard-fail without IDENTIFIER.
    - name: Ensure a unique node identifier
      tags: [identifier, deploy]
      block:
        - name: Check for an existing identifier in env_file
          ansible.builtin.command: grep -q '^IDENTIFIER=' env_file
          args:
            chdir: "{{ project_dir }}"
          register: identifier_check
          changed_when: false
          failed_when: false

        - name: Generate and store a unique identifier
          ansible.builtin.shell: |
            set -o pipefail
            id="$(head -c 4096 /dev/urandom | tr -dc 'a-zA-Z0-9' | head -c 10)"
            [ -n "$id" ] || { echo "failed to generate identifier" >&2; exit 1; }
            printf '\nIDENTIFIER=%s\n' "$id" >> env_file
          args:
            chdir: "{{ project_dir }}"
            executable: /bin/bash
          when: identifier_check.rc != 0

    # ---- cron: renewal + optional auto-update ------------------------------
    - name: Configure cron jobs
      tags: cron
      block:
        - name: Ensure cron is installed
          ansible.builtin.apt:
            name: cron
            state: present

        - name: Ensure the cron service is enabled and running
          ansible.builtin.systemd_service:
            name: cron
            enabled: true
            state: started

        - name: Schedule periodic redeploy / cert renewal
          ansible.builtin.cron:
            name: compassvpn-renewal
            minute: "0"
            hour: "0"
            day: "{{ ('*/' ~ redeploy_interval[:-1]) if redeploy_interval.endswith('d') else '1' }}"
            month: "{{ ('*/' ~ redeploy_interval[:-1]) if redeploy_interval.endswith('m') else '*' }}"
            job: "cd {{ project_dir }} && ./agent.sh start"
          when:
            - redeploy_interval | length > 0
            - redeploy_interval is match('^[0-9]+[dm]$')

        - name: Remove the redeploy job when no valid interval is set
          ansible.builtin.cron:
            name: compassvpn-renewal
            state: absent
          when: (redeploy_interval | length == 0) or not (redeploy_interval is match('^[0-9]+[dm]$'))

        - name: Schedule the hourly auto-update check
          ansible.builtin.cron:
            name: compassvpn-auto-update
            minute: "0"
            job: "cd {{ project_dir }} && ./agent.sh update"
          when: auto_update | lower == 'on'

        - name: Remove the auto-update check when disabled
          ansible.builtin.cron:
            name: compassvpn-auto-update
            state: absent
          when: auto_update | lower != 'on'

    # ---- deploy: build + (re)start the stack -------------------------------
    - name: Deploy the stack
      tags: deploy
      block:
        # Quiet, or the whole build log lands in agent.sh's output. Errors still print.
        - name: Build and (re)start all services
          ansible.builtin.command:
            cmd: docker compose --progress quiet up -d --build --force-recreate --remove-orphans
            chdir: "{{ project_dir }}"
          changed_when: true

        - name: Prune dangling images
          ansible.builtin.command: docker image prune -f
          changed_when: false