verify.yml #1

---
# Verify: the monitoring probes are deployed, scheduled, and actually emit
# Prometheus metrics against the live p4d.
- name: Verify
  hosts: all
  become: true
  tasks:
    - name: "Probe scripts are installed and executable"
      ansible.builtin.stat:
        path: "/p4/common/bin/{{ item }}"
      loop:
        - p4_healthcheck.sh
        - p4_disk_space.sh
        - p4_network_latency.sh
      register: probe_scripts

    - name: "Assert all probe scripts exist and are executable"
      ansible.builtin.assert:
        that:
          - item.stat.exists
          - item.stat.executable
        fail_msg: "probe script missing or not executable: {{ item.item }}"
      loop: "{{ probe_scripts.results }}"
      loop_control:
        label: "{{ item.item }}"

    - name: "Read the perforce crontab"
      become: true
      become_user: "{{ perforce_user }}"
      ansible.builtin.command: "crontab -l"
      register: crontab_out
      changed_when: false

    - name: "Assert the monitoring cron jobs are scheduled"
      ansible.builtin.assert:
        that:
          - "'p4 health check' in crontab_out.stdout"
          - "'p4 disk space check' in crontab_out.stdout"
          - "'p4 network latency check' in crontab_out.stdout"
        fail_msg: "monitoring cron jobs missing. crontab:\n{{ crontab_out.stdout }}"

    - name: "Confirm p4prometheus location file was rendered"
      ansible.builtin.stat:
        path: "{{ p4prometheus_metrics_dir }}/location.prom"
      register: location_prom

    - name: "Assert location.prom exists"
      ansible.builtin.assert:
        that: location_prom.stat.exists
        fail_msg: "{{ p4prometheus_metrics_dir }}/location.prom not found"

    - name: "Run the health-check probe by hand"
      become: true
      become_user: "{{ perforce_user }}"
      ansible.builtin.command: "/p4/common/bin/p4_healthcheck.sh"
      changed_when: false

    - name: "Read the health-check metric file"
      ansible.builtin.slurp:
        src: "{{ p4prometheus_metrics_dir }}/p4_healthcheck.prom"
      register: healthcheck_prom

    - name: "Assert the probe emitted p4_up=1 against the live server"
      ansible.builtin.assert:
        that: "'p4_up{instance=\"1\"} 1' in (healthcheck_prom.content | b64decode)"
        fail_msg: "health-check did not report the server up. Metric file:\n{{ healthcheck_prom.content | b64decode }}"