{"id":376,"sha1":"a3a89ea25732a4ac051d8c1090a2ca8f8b384d4c","playbook":{"id":4,"items":{"plays":13,"tasks":64,"results":194,"hosts":17,"files":46,"records":0},"arguments":{"version":null,"verbosity":0,"private_key_file":null,"remote_user":null,"connection":"openstack.osa.ssh","timeout":null,"ssh_common_args":null,"sftp_extra_args":null,"scp_extra_args":null,"ssh_extra_args":null,"ask_pass":false,"connection_password_file":null,"force_handlers":true,"flush_cache":false,"become":false,"become_method":"sudo","become_user":null,"become_ask_pass":false,"become_password_file":null,"tags":["all"],"skip_tags":[],"check":false,"diff":false,"inventory":["/home/zuul/src/opendev.org/openstack/openstack-ansible/inventory/dynamic_inventory.py","/home/zuul/src/opendev.org/openstack/openstack-ansible/inventory/inventory.ini","/etc/openstack_deploy/inventory.ini"],"listhosts":false,"subset":null,"extra_vars":"Not saved by ARA as configured by 'ignored_arguments'","vault_ids":[],"ask_vault_pass":false,"vault_password_files":[],"forks":8,"module_path":null,"syntax":false,"listtasks":false,"listtags":false,"step":false,"start_at_task":null,"args":["healthcheck-infrastructure.yml"]},"labels":[{"id":1,"name":"check:False"},{"id":2,"name":"tags:all"}],"started":"2025-12-15T10:16:58.109700Z","ended":"2025-12-15T10:19:03.654506Z","duration":"00:02:05.544806","name":null,"ansible_version":"2.18.6","client_version":"1.7.4","python_version":"3.12.3","server_version":"1.7.4","status":"completed","path":"/home/zuul/src/opendev.org/openstack/openstack-ansible/playbooks/healthcheck-infrastructure.yml","controller":"aio1.openstack.local","user":"root"},"content":"---\n# Copyright 2017, Rackspace US, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# This playbook is meant to run after setup-infrastructure, and expects\n# the infrastructure bits to have properly deployed to succeed.\n\n# Test unbound-install.yml\n# TO BE IMPLEMENTED\n\n# Test repo-install.yml\n- name: Ensure all repo-servers are built and are accessible by hosts.\n  hosts: all_containers[0]:physical_hosts[0]\n  gather_facts: true\n  vars:\n    repo_requirements_file: \"constraints/upper_constraints_cached.txt\"\n  tasks:\n    - name: Setup installation variables\n      ansible.builtin.include_role:\n        name: openstack.osa.install_defaults\n        defaults_from: \"{{ install_method }}\"\n        public: true\n\n    - name: Check the repo sync file on each repo server\n      ansible.builtin.uri:\n        url: \"{{ openstack_repo_protocol }}://{{ hostvars[item]['management_address'] }}:{{ repo_server_port }}/{{ repo_requirements_file }}\"\n      with_inventory_hostnames: \"{{ groups['repo_all'] }}\"\n      when: install_method == 'source'\n  tags:\n    - healthcheck\n    - healthcheck-repo-install\n\n# Test haproxy-install.yml\n- name: Ensuring haproxy runs\n  hosts: haproxy\n  gather_facts: true\n  tasks:\n    - name: Check if host can connect to external keepalived ping IP\n      ansible.builtin.command: \"ping -c 2 {{ keepalived_external_ping_address }}\"\n      changed_when: false\n\n    - name: Check if host can connect to internal keepalived ping IP\n      ansible.builtin.command: \"ping -c 2 {{ keepalived_internal_ping_address }}\"\n      changed_when: false\n\n    - name: Checking if keepalived is running\n      ansible.builtin.command: \"pgrep keepalived\"\n      changed_when: false\n      when: groups['haproxy'] | length > 1\n\n    - name: Install netcat\n      ansible.builtin.package:\n        name: \"{% if ansible_facts['os_family'] | lower == 'redhat' %}nmap-ncat{% else %}netcat-openbsd{% endif %}\"\n        state: present\n\n    # Fails if HAProxy is not running\n    - name: Recording haproxy stats as a way to ensure haproxy runs\n      ansible.builtin.shell: 'echo \"show info;show stat\" | nc -U /var/run/haproxy.stat'\n      changed_when: false\n      register: haproxy_stats\n\n    # Run this playbook with -v and you'll see your DOWN issues\n    - name: Printing the output of haproxy stats\n      ansible.builtin.debug:\n        var: haproxy_stats\n        verbosity: 1\n  tags:\n    - healthcheck\n    - healthcheck-haproxy-install\n\n# Test repo-use.yml\n- name: Ensure all the containers can connect to the repos\n  hosts: all_containers\n  gather_facts: true\n  # By having serial, you ensure that the first three containers are hitting\n  # the load balancer at the same time, which would then cause hitting three\n  # different repos servers.\n  # When this is done, the rest can be done with all the nodes at the same time.\n  serial:\n    - 3\n    - 100%\n  tasks:\n    - name: Run check only for source method\n      when:\n        - install_method == 'source'\n      block:\n        - name: Setup installation variables\n          ansible.builtin.include_role:\n            name: openstack.osa.install_defaults\n            defaults_from: \"{{ install_method }}\"\n            public: true\n\n        # Repo release path points to the internal LB vip\n        - name: Check the presence of upper constraints on your repos and check load balancing\n          ansible.builtin.uri:\n            url: \"{{ openstack_repo_url }}/constraints/upper_constraints_cached.txt\"\n  tags:\n    - healthcheck\n    - healthcheck-repo-use\n\n# Test utility-install.yml\n- name: Ensure the service setup host is ready to run openstack calls\n  hosts: \"{{ openstack_service_setup_host | default('localhost') }}\"\n  gather_facts: false\n  vars:\n    ansible_python_interpreter: \"{{ openstack_service_setup_host_python_interpreter | default(ansible_facts['python']['executable']) }}\"\n  pre_tasks:\n    - name: Setup installation variables\n      ansible.builtin.import_role:\n        name: openstack.osa.install_defaults\n        defaults_from: \"{{ install_method | default('source') }}\"\n  tasks:\n    - name: Get openstack client config\n      openstack.cloud.config:\n    - name: Show openstack client config\n      ansible.builtin.debug:\n        var: openstack.clouds\n        verbosity: 1\n  tags:\n    - healthcheck\n    - healthcheck-utility-install\n\n# Test memcached-install.yml\n- name: Check memcached for keystone\n  hosts: keystone_all\n  gather_facts: false\n  tasks:\n    - name: Set facts about memcached\n      ansible.builtin.setup:\n      delegate_to: \"{{ item }}\"\n      delegate_facts: true\n      with_items: \"{{ groups['memcached'] }}\"\n\n    - name: Install netcat\n      ansible.builtin.package:\n        name: \"{% if ansible_facts['os_family'] | lower == 'redhat' %}nmap-ncat{% else %}netcat-openbsd{% endif %}\"\n        state: present\n\n    - name: Connect to remote memcache servers (full mesh testing)\n      ansible.builtin.shell: \"echo stats | nc -w 3 {{ hostvars[memcached_host]['management_address'] }} {{ memcached_port }}\"\n      changed_when: false\n      register: memcache_stats\n      with_items: \"{{ groups['memcached'] }}\"\n      loop_control:\n        loop_var: memcached_host\n\n    - name: Output memcache stats if in verbose mode\n      ansible.builtin.debug:\n        var: memcache_stats\n        verbosity: 1\n  tags:\n    - healthcheck\n    - healthcheck-memcached-install\n\n# Test galera-install.yml\n- name: Sanity checks for all containers\n  hosts: all_containers:physical_hosts\n  gather_facts: false\n  tasks:\n    - name: Connect to galera port\n      ansible.builtin.wait_for:\n        port: 3306\n        host: \"{{ internal_lb_vip_address }}\"\n        state: started\n  tags:\n    - healthcheck\n    - healthcheck-galera-install\n\n- name: Run functional tests\n  hosts: galera_all\n  user: root\n  gather_facts: true\n  vars:\n    _mariadb_client_binary: \"{{ galera_mariadb_client_binary | default('mariadb') }}\"\n  tasks:\n    - name: Wait for cluster to be ready\n      block:\n        - name: Wait for cluster ready state\n          ansible.builtin.command: |\n            {{ _mariadb_client_binary }} -h {{ management_address }} \\\n                  -u \"{{ galera_root_user | default('root') }}\" \\\n                  -p\"{{ galera_root_password }}\" \\\n                  -e \"show status like 'wsrep_incoming_addresses';\" \\\n                  --silent \\\n                  --skip-column-names\n          register: mysql_instance_ready\n          retries: 20\n          delay: 5\n          changed_when: false\n          until: mysql_instance_ready is success and mysql_instance_ready.stdout.split()[-1].split(',') | length == groups['galera_all'] | length\n      rescue:\n        - name: Restarting weird maria instance\n          ansible.builtin.service:\n            name: mariadb\n            state: restarted\n        - name: Wait for cluster ready state\n          ansible.builtin.command: |\n            {{ _mariadb_client_binary }} -h {{ management_address }} \\\n                  -u \"{{ galera_root_user | default('root') }}\" \\\n                  -p\"{{ galera_root_password }}\" \\\n                  -e \"show status like 'wsrep_incoming_addresses';\" \\\n                  --silent \\\n                  --skip-column-names\n          register: mysql_instance_ready\n          retries: 20\n          delay: 5\n          changed_when: false\n          until: mysql_instance_ready is success and mysql_instance_ready.stdout.split()[-1].split(',') | length == groups['galera_all'] | length\n\n    - name: Check cluster local state\n      ansible.builtin.command: |\n        {{ _mariadb_client_binary }} -h {{ management_address }} \\\n              -u \"{{ galera_root_user | default('root') }}\" \\\n              -p\"{{ galera_root_password }}\" \\\n              -e \"show status like 'wsrep_local_state_comment';\" \\\n              --silent \\\n              --skip-column-names\n      register: wsrep_local_state_comment\n      changed_when: false\n      tags:\n        - skip_ansible_lint\n\n    - name: Check cluster evs state\n      ansible.builtin.command: |\n        {{ _mariadb_client_binary }} -h {{ management_address }} \\\n              -u \"{{ galera_root_user | default('root') }}\" \\\n              -p\"{{ galera_root_password }}\" \\\n              -e \"show status like 'wsrep_evs_state';\" \\\n              --silent \\\n              --skip-column-names\n      register: wsrep_evs_state\n      changed_when: false\n      tags:\n        - skip_ansible_lint\n\n    - name: Check contents\n      ansible.builtin.assert:\n        that:\n          - \"'Synced' in wsrep_local_state_comment.stdout\"\n          - \"'OPERATIONAL' in wsrep_evs_state.stdout\"\n\n    - name: Create DB for service on \"{{ groups['galera_all'][0] }}\"\n      community.mysql.mysql_db:\n        login_user: \"{{ galera_root_user | default('root') }}\"\n        login_password: \"{{ galera_root_password }}\"\n        login_host: \"{{ management_address }}\"\n        name: \"OSA-test\"\n        state: \"present\"\n        check_hostname: true\n      when: inventory_hostname == groups['galera_all'][0]\n      tags:\n        - skip_ansible_lint\n\n    - name: Grant access to the DB on \"{{ groups['galera_all'][-1] }}\"\n      community.mysql.mysql_user:\n        login_user: \"{{ galera_root_user | default('root') }}\"\n        login_password: \"{{ galera_root_password }}\" # noqa no-log-password\n        login_host: \"{{ management_address }}\"\n        name: \"osa-tester\"\n        password: \"tester-secrete\" # noqa no-log-password\n        host: \"{{ item }}\"\n        state: \"present\"\n        priv: \"OSA-test.*:ALL\"\n        check_hostname: true\n      with_items:\n        - \"localhost\"\n        - \"%\"\n      when: inventory_hostname == groups['galera_all'][-1]\n\n    - name: Try to login with user to DB\n      delegate_to: \"{{ groups['utility_all'][0] }}\"\n      ansible.builtin.command: |\n        {{ _mariadb_client_binary }} -h {{ internal_lb_vip_address }} \\\n              -p\"tester-secrete\" \\\n              -u osa-tester \\\n              OSA-test \\\n              -e \"SHOW TABLES;\"\n      changed_when: false\n      when: inventory_hostname == groups['galera_all'][-1]\n\n    - name: Remove created user\n      community.mysql.mysql_user:\n        login_user: \"{{ galera_root_user | default('root') }}\"\n        login_password: \"{{ galera_root_password }}\" # noqa no-log-password\n        login_host: \"{{ management_address }}\"\n        name: \"osa-tester\"\n        state: \"absent\"\n        host: \"{{ item }}\"\n        check_hostname: true\n      with_items:\n        - \"localhost\"\n        - \"%\"\n      when: inventory_hostname == groups['galera_all'][-1]\n\n    - name: Remove created DB\n      community.mysql.mysql_db:\n        login_user: \"{{ galera_root_user | default('root') }}\"\n        login_password: \"{{ galera_root_password }}\" # noqa no-log-password\n        login_host: \"{{ management_address }}\"\n        name: \"OSA-test\"\n        state: \"absent\"\n        check_hostname: true\n      when: inventory_hostname == groups['galera_all'][0]\n      tags:\n        - skip_ansible_lint\n\n\n# Test rabbitmq-install.yml\n- name: Add a user for rabbitmq\n  hosts: rabbitmq_all[0]\n  gather_facts: false\n  tasks:\n    - name: Configure Rabbitmq vhost\n      community.rabbitmq.rabbitmq_vhost:\n        name: \"/testvhost\"\n        state: \"present\"\n\n    - name: Configure Rabbitmq user\n      community.rabbitmq.rabbitmq_user:\n        user: \"testguest\"\n        password: \"secrete\" # noqa no-log-password\n        vhost: \"/testvhost\"\n        configure_priv: \".*\"\n        read_priv: \".*\"\n        write_priv: \".*\"\n        state: \"present\"\n\n  tags:\n    - healthcheck\n    - healthcheck-rabbitmq-install\n\n- name: Ensure all the usual openstack containers can connect to rabbit\n  hosts: all_containers:!galera_all:!memcached:!haproxy:!rabbitmq_all:!unbound:!repo_all\n  gather_facts: false\n  vars:\n    venv_path: /tmp/rabbitmqtest\n  post_tasks:\n    - name: Setup installation variables\n      ansible.builtin.include_role:\n        name: openstack.osa.install_defaults\n        defaults_from: \"{{ install_method }}\"\n        public: true\n    - name: Generate venv for rabbitmq testing\n      ansible.builtin.include_role:\n        name: \"python_venv_build\"\n      vars:\n        venv_install_destination_path: \"{{ venv_path }}\"\n        venv_pip_packages:\n          - pika\n    - name: Copying test script\n      ansible.builtin.copy:\n        src: \"scripts/rabbitmq-test.py\"\n        dest: \"{{ venv_path }}/rabbitmq-test.py\"\n        mode: \"0755\"\n    - name: Connect to rabbitmq\n      ansible.builtin.command: \"{{ venv_path }}/bin/python {{ venv_path }}/rabbitmq-test.py {{ hostvars[groups['rabbitmq_all'][0]]['management_address'] }}\"\n      changed_when: false\n  tags:\n    - healthcheck\n    - healthcheck-rabbitmq-install\n\n- name: Remove guest user for rabbitmq\n  hosts: rabbitmq_all[0]\n  gather_facts: false\n  tasks:\n    - name: Remove test user\n      community.rabbitmq.rabbitmq_user:\n        user: testguest\n        password: secrete\n        vhost: \"/testvhost\"\n        state: absent\n      no_log: true\n    - name: Remove test vhost\n      community.rabbitmq.rabbitmq_vhost:\n        name: \"/testvhost\"\n        state: \"absent\"\n  tags:\n    - healthcheck\n    - healthcheck-rabbitmq-install\n    - healthcheck-teardown\n\n# Test zookeeper-install\n\n- name: Ensure coordination is running and accepting connections\n  hosts: utility_all[0]\n  tasks:\n    - name: Probing TCP connection to zookeeper\n      ansible.builtin.wait_for:\n        host: \"{{ hostvars[item]['management_address'] }}\"\n        port: \"{{ coordination_port | default(2181) }}\"\n      with_items: \"{{ groups[coordination_host_group | default('zookeeper_all')] }}\"\n\n- name: Ensure zookeeper is healthy\n  hosts: \"zookeeper_all\"\n  tasks:\n    - name: Esuring netcat is installed\n      ansible.builtin.package:\n        name: \"{% if ansible_facts['os_family'] | lower == 'redhat' %}nmap-ncat{% else %}netcat-openbsd{% endif %}\"\n        state: present\n\n    - name: Gathering zookeeper state\n      ansible.builtin.shell: \"echo ruok | nc localhost {{ coordination_port | default(2181) }}\"\n      register: zookeeper_ok\n      changed_when: false\n\n    - name: Gathering zookeeper rw/ro\n      ansible.builtin.shell: \"echo isro | nc localhost {{ coordination_port | default(2181) }}\"\n      register: zookeeper_ro\n      changed_when: false\n\n    - name: Check zookeeper results\n      ansible.builtin.assert:\n        that:\n          - \"'imok' in zookeeper_ok.stdout\"\n          - \"'rw' in zookeeper_ro.stdout\"\n\n# TODO: Other playbook's tests.\n","created":"2025-12-15T10:16:58.469412Z","updated":"2025-12-15T10:16:58.469441Z","path":"/etc/ansible/ansible_collections/openstack/osa/playbooks/healthcheck/infrastructure.yml"}