[root@AnsibleServer ~]# ansible-playbook --ask-pass dbs3-postgres-ha.yml
SSH password:
PLAY [install PG HA] ***************************************************************************************************************
TASK [Gathering Facts] *************************************************************************************************************
ok: [dbs04.prodea-int.net]
ok: [dbs03.prodea-int.net]
TASK [disable firewall] ************************************************************************************************************
changed: [dbs03.prodea-int.net]
changed: [dbs04.prodea-int.net]
TASK [postgres-ha6 : debug] ********************************************************************************************************
ok: [dbs03.prodea-int.net] => {
"msg": "MASTER NODE SET TO dbs03.prodea-int.net"
}
TASK [postgres-ha6 : verify postgres_ha_cluster_master_host] ***********************************************************************
skipping: [dbs03.prodea-int.net]
skipping: [dbs04.prodea-int.net]
TASK [postgres-ha6 : identify the OS] **********************************************************************************************
ok: [dbs03.prodea-int.net]
ok: [dbs04.prodea-int.net]
TASK [postgres-ha6 : debug] ********************************************************************************************************
ok: [dbs03.prodea-int.net] => {
"msg": "cluster_members=[u'dbs03.prodea-int.net', u'dbs04.prodea-int.net']"
}
TASK [postgres-ha6 : install cluster pkgs] *****************************************************************************************
changed: [dbs04.prodea-int.net]
changed: [dbs03.prodea-int.net]
TASK [postgres-ha6 : install additional cluster pkgs for centos6] ******************************************************************
changed: [dbs03.prodea-int.net] => (item=[u'pacemaker', u'libselinux-python'])
changed: [dbs04.prodea-int.net] => (item=[u'pacemaker', u'libselinux-python'])
TASK [postgres-ha6 : Build hosts file] *********************************************************************************************
changed: [dbs03.prodea-int.net] => (item=dbs03.prodea-int.net)
changed: [dbs04.prodea-int.net] => (item=dbs03.prodea-int.net)
changed: [dbs03.prodea-int.net] => (item=dbs04.prodea-int.net)
changed: [dbs04.prodea-int.net] => (item=dbs04.prodea-int.net)
TASK [postgres-ha6 : service pcsd start] *******************************************************************************************
changed: [dbs03.prodea-int.net]
changed: [dbs04.prodea-int.net]
TASK [postgres-ha6 : setup hacluster password] *************************************************************************************
changed: [dbs04.prodea-int.net]
changed: [dbs03.prodea-int.net]
TASK [postgres-ha6 : setup cluster auth] *******************************************************************************************
skipping: [dbs03.prodea-int.net]
skipping: [dbs04.prodea-int.net]
TASK [postgres-ha6 : create cluster] ***********************************************************************************************
skipping: [dbs03.prodea-int.net]
TASK [postgres-ha6 : create cluster] ***********************************************************************************************
changed: [dbs04.prodea-int.net]
changed: [dbs03.prodea-int.net]
TASK [postgres-ha6 : join cluster nodes] *******************************************************************************************
skipping: [dbs03.prodea-int.net] => (item=dbs04.prodea-int.net)
TASK [postgres-ha6 : start cluster] ************************************************************************************************
changed: [dbs04.prodea-int.net]
changed: [dbs03.prodea-int.net]
TASK [postgres-ha6 : alter stonith settings] ***************************************************************************************
changed: [dbs03.prodea-int.net]
TASK [postgres-ha6 : alter cluster policy settings] ********************************************************************************
changed: [dbs03.prodea-int.net]
TASK [postgres-ha6 : alter cluster transition settings] ****************************************************************************
changed: [dbs03.prodea-int.net]
TASK [postgres-ha6 : verify cluster configuration] *********************************************************************************
changed: [dbs03.prodea-int.net]
TASK [postgres-ha6 : enable cluster autostart] *************************************************************************************
changed: [dbs03.prodea-int.net]
changed: [dbs04.prodea-int.net]
TASK [postgres-ha6 : create virtual IP resource] ***********************************************************************************
[WARNING]: when statements should not include jinja2 templating delimiters such as {{ }} or {% %}. Found: inventory_hostname ==
"{{ postgres_ha_cluster_master_host }}"
[WARNING]: when statements should not include jinja2 templating delimiters such as {{ }} or {% %}. Found: inventory_hostname ==
"{{ postgres_ha_cluster_master_host }}"
skipping: [dbs04.prodea-int.net]
changed: [dbs03.prodea-int.net]
TASK [postgres-ha6 : import pg96 repo] *********************************************************************************************
changed: [dbs03.prodea-int.net]
changed: [dbs04.prodea-int.net]
TASK [postgres-ha6 : install pg96] *************************************************************************************************
changed: [dbs04.prodea-int.net]
changed: [dbs03.prodea-int.net]
TASK [postgres-ha6 : init DB dir on master if necessary (centos 7)] ****************************************************************
skipping: [dbs03.prodea-int.net]
skipping: [dbs04.prodea-int.net]
TASK [postgres-ha6 : init DB dir on master if necessary (centos 6)] ****************************************************************
skipping: [dbs04.prodea-int.net]
changed: [dbs03.prodea-int.net]
TASK [postgres-ha6 : check if DB was synchronized before] **************************************************************************
ok: [dbs03.prodea-int.net]
ok: [dbs04.prodea-int.net]
TASK [postgres-ha6 : alter clustering-related settings in postgresql.conf] *********************************************************
[WARNING]: when statements should not include jinja2 templating delimiters such as {{ }} or {% %}. Found: inventory_hostname ==
"{{ postgres_ha_cluster_master_host }}"
[WARNING]: when statements should not include jinja2 templating delimiters such as {{ }} or {% %}. Found: inventory_hostname ==
"{{ postgres_ha_cluster_master_host }}"
skipping: [dbs04.prodea-int.net] => (item={'key': u'hot_standby', 'value': u'on'})
skipping: [dbs04.prodea-int.net] => (item={'key': u'listen_addresses', 'value': u"''"})
skipping: [dbs04.prodea-int.net] => (item={'key': u'wal_level', 'value': u'hot_standby'})
skipping: [dbs04.prodea-int.net] => (item={'key': u'wal_log_hints', 'value': u'on'})
skipping: [dbs04.prodea-int.net] => (item={'key': u'max_wal_senders', 'value': u'4'})
skipping: [dbs04.prodea-int.net] => (item={'key': u'max_replication_slots', 'value': u'4'})
changed: [dbs03.prodea-int.net] => (item={'key': u'hot_standby', 'value': u'on'})
changed: [dbs03.prodea-int.net] => (item={'key': u'listen_addresses', 'value': u"''"})
changed: [dbs03.prodea-int.net] => (item={'key': u'wal_level', 'value': u'hot_standby'})
changed: [dbs03.prodea-int.net] => (item={'key': u'wal_log_hints', 'value': u'on'})
changed: [dbs03.prodea-int.net] => (item={'key': u'max_wal_senders', 'value': u'4'})
changed: [dbs03.prodea-int.net] => (item={'key': u'max_replication_slots', 'value': u'4'})
RUNNING HANDLER [postgres-ha6 : restart postgresql] ********************************************************************************
changed: [dbs03.prodea-int.net]
TASK [postgres-ha6 : alter DB ACL in pg_hba.conf] **********************************************************************************
[WARNING]: when statements should not include jinja2 templating delimiters such as {{ }} or {% %}. Found: inventory_hostname ==
"{{ postgres_ha_cluster_master_host }}" or db_prevsync_file.stat.exists
[WARNING]: when statements should not include jinja2 templating delimiters such as {{ }} or {% %}. Found: inventory_hostname ==
"{{ postgres_ha_cluster_master_host }}" or db_prevsync_file.stat.exists
skipping: [dbs04.prodea-int.net] => (item=dbs03.prodea-int.net)
skipping: [dbs04.prodea-int.net] => (item=dbs04.prodea-int.net)
changed: [dbs03.prodea-int.net] => (item=dbs03.prodea-int.net)
changed: [dbs03.prodea-int.net] => (item=dbs04.prodea-int.net)
TASK [postgres-ha6 : alter DB replication ACL in pg_hba.conf] **********************************************************************
[WARNING]: when statements should not include jinja2 templating delimiters such as {{ }} or {% %}. Found: inventory_hostname ==
"{{ postgres_ha_cluster_master_host }}" or db_prevsync_file.stat.exists
[WARNING]: when statements should not include jinja2 templating delimiters such as {{ }} or {% %}. Found: inventory_hostname ==
"{{ postgres_ha_cluster_master_host }}" or db_prevsync_file.stat.exists
skipping: [dbs04.prodea-int.net] => (item=dbs03.prodea-int.net)
skipping: [dbs04.prodea-int.net] => (item=dbs04.prodea-int.net)
changed: [dbs03.prodea-int.net] => (item=dbs03.prodea-int.net)
changed: [dbs03.prodea-int.net] => (item=dbs04.prodea-int.net)
RUNNING HANDLER [postgres-ha6 : reload postgresql] *********************************************************************************
changed: [dbs03.prodea-int.net]
TASK [postgres-ha6 : setup DB cluster auth (master IP)] ****************************************************************************
changed: [dbs03.prodea-int.net]
changed: [dbs04.prodea-int.net]
TASK [postgres-ha6 : setup .pgpass replication auth for master IP] *****************************************************************
changed: [dbs03.prodea-int.net]
changed: [dbs04.prodea-int.net]
TASK [postgres-ha6 : setup .pgpass replication auth for other IPs] *****************************************************************
changed: [dbs04.prodea-int.net] => (item=dbs03.prodea-int.net)
changed: [dbs03.prodea-int.net] => (item=dbs03.prodea-int.net)
changed: [dbs04.prodea-int.net] => (item=dbs04.prodea-int.net)
changed: [dbs03.prodea-int.net] => (item=dbs04.prodea-int.net)
TASK [postgres-ha6 : check if master host "dbs03.prodea-int.net" is really a DB master] ********************************************
[WARNING]: when statements should not include jinja2 templating delimiters such as {{ }} or {% %}. Found: inventory_hostname ==
"{{ postgres_ha_cluster_master_host }}"
[WARNING]: when statements should not include jinja2 templating delimiters such as {{ }} or {% %}. Found: inventory_hostname ==
"{{ postgres_ha_cluster_master_host }}"
skipping: [dbs04.prodea-int.net]
changed: [dbs03.prodea-int.net]
TASK [postgres-ha6 : mark master DB] ***********************************************************************************************
[WARNING]: when statements should not include jinja2 templating delimiters such as {{ }} or {% %}. Found: inventory_hostname ==
"{{ postgres_ha_cluster_master_host }}"
[WARNING]: when statements should not include jinja2 templating delimiters such as {{ }} or {% %}. Found: inventory_hostname ==
"{{ postgres_ha_cluster_master_host }}"
skipping: [dbs04.prodea-int.net]
changed: [dbs03.prodea-int.net]
TASK [postgres-ha6 : check if DB is running (failure is ok)] ***********************************************************************
changed: [dbs03.prodea-int.net]
fatal: [dbs04.prodea-int.net]: FAILED! => {"changed": true, "cmd": "/usr/pgsql-9.6/bin/pg_ctl -D /var/lib/pgsql/9.6/data status", "delta": "0:00:00.042861", "end": "2017-08-28 23:38:32.862478", "failed": true, "rc": 4, "start": "2017-08-28 23:38:32.819617", "stderr": "pg_ctl: directory "/var/lib/pgsql/9.6/data" is not a database cluster directory", "stderr_lines": ["pg_ctl: directory "/var/lib/pgsql/9.6/data" is not a database cluster directory"], "stdout": "", "stdout_lines": []}
...ignoring
TASK [postgres-ha6 : check if DB is running in cluster (failure is OK)] ************************************************************
fatal: [dbs03.prodea-int.net]: FAILED! => {"changed": true, "cmd": "pcs constraint location show resources "postgres-ha" | grep -q Enabled", "delta": "0:00:00.344149", "end": "2017-08-28 23:38:34.271540", "failed": true, "rc": 1, "start": "2017-08-28 23:38:33.927391", "stderr": "", "stderr_lines": [], "stdout": "", "stdout_lines": []}
...ignoring
fatal: [dbs04.prodea-int.net]: FAILED! => {"changed": true, "cmd": "pcs constraint location show resources "postgres-ha" | grep -q Enabled", "delta": "0:00:00.330268", "end": "2017-08-28 23:38:34.298727", "failed": true, "rc": 1, "start": "2017-08-28 23:38:33.968459", "stderr": "", "stderr_lines": [], "stdout": "", "stdout_lines": []}
...ignoring
TASK [postgres-ha6 : start master DB if necessary (without cluster)] ***************************************************************
[WARNING]: when statements should not include jinja2 templating delimiters such as {{ }} or {% %}. Found: (inventory_hostname ==
"{{ postgres_ha_cluster_master_host }}") and (db_resource_exists|failed) and (db_running|failed)
skipping: [dbs03.prodea-int.net]
[WARNING]: when statements should not include jinja2 templating delimiters such as {{ }} or {% %}. Found: (inventory_hostname ==
"{{ postgres_ha_cluster_master_host }}") and (db_resource_exists|failed) and (db_running|failed)
skipping: [dbs04.prodea-int.net]
TASK [postgres-ha6 : start master DB if necessary (in cluster)] ********************************************************************
[WARNING]: when statements should not include jinja2 templating delimiters such as {{ }} or {% %}. Found: (inventory_hostname ==
"{{ postgres_ha_cluster_master_host }}") and (db_resource_exists|succeeded) and (db_running|failed)
skipping: [dbs03.prodea-int.net]
[WARNING]: when statements should not include jinja2 templating delimiters such as {{ }} or {% %}. Found: (inventory_hostname ==
"{{ postgres_ha_cluster_master_host }}") and (db_resource_exists|succeeded) and (db_running|failed)
skipping: [dbs04.prodea-int.net]
TASK [postgres-ha6 : setup DB replication auth] ************************************************************************************
[WARNING]: when statements should not include jinja2 templating delimiters such as {{ }} or {% %}. Found: inventory_hostname ==
"{{ postgres_ha_cluster_master_host }}"
[WARNING]: when statements should not include jinja2 templating delimiters such as {{ }} or {% %}. Found: inventory_hostname ==
"{{ postgres_ha_cluster_master_host }}"
skipping: [dbs04.prodea-int.net]
changed: [dbs03.prodea-int.net]
TASK [postgres-ha6 : check if DB sync is required] *********************************************************************************
ok: [dbs04.prodea-int.net]
ok: [dbs03.prodea-int.net]
TASK [postgres-ha6 : stop slave DB] ************************************************************************************************
skipping: [dbs03.prodea-int.net]
skipping: [dbs04.prodea-int.net]
TASK [postgres-ha6 : remove slave DB datadir before sync] **************************************************************************
skipping: [dbs03.prodea-int.net]
changed: [dbs04.prodea-int.net]
TASK [postgres-ha6 : synchronize slave databases] **********************************************************************************
skipping: [dbs03.prodea-int.net]
changed: [dbs04.prodea-int.net]
TASK [postgres-ha6 : start slave DBs] **********************************************************************************************
[WARNING]: when statements should not include jinja2 templating delimiters such as {{ }} or {% %}. Found: (inventory_hostname !=
"{{ postgres_ha_cluster_master_host }}") and (db_resource_exists|failed)
skipping: [dbs03.prodea-int.net]
[WARNING]: when statements should not include jinja2 templating delimiters such as {{ }} or {% %}. Found: (inventory_hostname !=
"{{ postgres_ha_cluster_master_host }}") and (db_resource_exists|failed)
changed: [dbs04.prodea-int.net]
TASK [postgres-ha6 : check if slaves are connected] ********************************************************************************
[WARNING]: when statements should not include jinja2 templating delimiters such as {{ }} or {% %}. Found: inventory_hostname ==
"{{ postgres_ha_cluster_master_host }}"
[WARNING]: when statements should not include jinja2 templating delimiters such as {{ }} or {% %}. Found: inventory_hostname ==
"{{ postgres_ha_cluster_master_host }}"
skipping: [dbs04.prodea-int.net]
changed: [dbs03.prodea-int.net]
TASK [postgres-ha6 : select proper PAF package (centos6)] **************************************************************************
skipping: [dbs03.prodea-int.net]
skipping: [dbs04.prodea-int.net]
TASK [postgres-ha6 : select proper PAF package (centos6)] **************************************************************************
ok: [dbs03.prodea-int.net]
ok: [dbs04.prodea-int.net]
TASK [postgres-ha6 : copy PAF rpm to hosts] ****************************************************************************************
changed: [dbs03.prodea-int.net]
changed: [dbs04.prodea-int.net]
TASK [postgres-ha6 : install PAF DB failover agent] ********************************************************************************
changed: [dbs04.prodea-int.net]
changed: [dbs03.prodea-int.net]
TASK [postgres-ha6 : prepare DB recovery config] ***********************************************************************************
changed: [dbs03.prodea-int.net]
changed: [dbs04.prodea-int.net]
TASK [postgres-ha6 : stop database for clustering] *********************************************************************************
changed: [dbs04.prodea-int.net]
changed: [dbs03.prodea-int.net]
TASK [postgres-ha6 : create database cluster resource] *****************************************************************************
[WARNING]: when statements should not include jinja2 templating delimiters such as {{ }} or {% %}. Found: inventory_hostname ==
"{{ postgres_ha_cluster_master_host }}"
[WARNING]: when statements should not include jinja2 templating delimiters such as {{ }} or {% %}. Found: inventory_hostname ==
"{{ postgres_ha_cluster_master_host }}"
skipping: [dbs04.prodea-int.net]
changed: [dbs03.prodea-int.net]
TASK [postgres-ha6 : create master DB resource] ************************************************************************************
[WARNING]: when statements should not include jinja2 templating delimiters such as {{ }} or {% %}. Found: inventory_hostname ==
"{{ postgres_ha_cluster_master_host }}"
[WARNING]: when statements should not include jinja2 templating delimiters such as {{ }} or {% %}. Found: inventory_hostname ==
"{{ postgres_ha_cluster_master_host }}"
skipping: [dbs04.prodea-int.net]
changed: [dbs03.prodea-int.net]
TASK [postgres-ha6 : test constraints presence] ************************************************************************************
ok: [dbs04.prodea-int.net]
ok: [dbs03.prodea-int.net]
TASK [postgres-ha6 : setting VIP location constraints] *****************************************************************************
[WARNING]: when statements should not include jinja2 templating delimiters such as {{ }} or {% %}. Found: inventory_hostname ==
"{{ postgres_ha_cluster_master_host }}"
[WARNING]: when statements should not include jinja2 templating delimiters such as {{ }} or {% %}. Found: inventory_hostname ==
"{{ postgres_ha_cluster_master_host }}"
skipping: [dbs04.prodea-int.net]
changed: [dbs03.prodea-int.net]
TASK [postgres-ha6 : setting DB location constraints] ******************************************************************************
[WARNING]: when statements should not include jinja2 templating delimiters such as {{ }} or {% %}. Found: inventory_hostname ==
"{{ postgres_ha_cluster_master_host }}"
[WARNING]: when statements should not include jinja2 templating delimiters such as {{ }} or {% %}. Found: inventory_hostname ==
"{{ postgres_ha_cluster_master_host }}"
skipping: [dbs04.prodea-int.net]
changed: [dbs03.prodea-int.net]
TASK [postgres-ha6 : setting resources colocation group 1] *************************************************************************
[WARNING]: when statements should not include jinja2 templating delimiters such as {{ }} or {% %}. Found: inventory_hostname ==
"{{ postgres_ha_cluster_master_host }}"
[WARNING]: when statements should not include jinja2 templating delimiters such as {{ }} or {% %}. Found: inventory_hostname ==
"{{ postgres_ha_cluster_master_host }}"
skipping: [dbs04.prodea-int.net]
changed: [dbs03.prodea-int.net]
TASK [postgres-ha6 : setting resources start order] ********************************************************************************
[WARNING]: when statements should not include jinja2 templating delimiters such as {{ }} or {% %}. Found: inventory_hostname ==
"{{ postgres_ha_cluster_master_host }}"
[WARNING]: when statements should not include jinja2 templating delimiters such as {{ }} or {% %}. Found: inventory_hostname ==
"{{ postgres_ha_cluster_master_host }}"
skipping: [dbs04.prodea-int.net]
changed: [dbs03.prodea-int.net]
TASK [postgres-ha6 : setting resources stop order] *********************************************************************************
[WARNING]: when statements should not include jinja2 templating delimiters such as {{ }} or {% %}. Found: inventory_hostname ==
"{{ postgres_ha_cluster_master_host }}"
[WARNING]: when statements should not include jinja2 templating delimiters such as {{ }} or {% %}. Found: inventory_hostname ==
"{{ postgres_ha_cluster_master_host }}"
skipping: [dbs04.prodea-int.net]
changed: [dbs03.prodea-int.net]
TASK [postgres-ha6 : marking constraints as processed] *****************************************************************************
changed: [dbs03.prodea-int.net]
changed: [dbs04.prodea-int.net]
TASK [postgres-ha6 : enable database cluster resource] *****************************************************************************
[WARNING]: when statements should not include jinja2 templating delimiters such as {{ }} or {% %}. Found: inventory_hostname ==
"{{ postgres_ha_cluster_master_host }}"
[WARNING]: when statements should not include jinja2 templating delimiters such as {{ }} or {% %}. Found: inventory_hostname ==
"{{ postgres_ha_cluster_master_host }}"
skipping: [dbs04.prodea-int.net]
changed: [dbs03.prodea-int.net]
TASK [postgres-ha6 : refresh database cluster resource] ****************************************************************************
[WARNING]: when statements should not include jinja2 templating delimiters such as {{ }} or {% %}. Found: inventory_hostname ==
"{{ postgres_ha_cluster_master_host }}"
[WARNING]: when statements should not include jinja2 templating delimiters such as {{ }} or {% %}. Found: inventory_hostname ==
"{{ postgres_ha_cluster_master_host }}"
skipping: [dbs04.prodea-int.net]
changed: [dbs03.prodea-int.net]
TASK [postgres-ha6 : check if all slaves are connected] ****************************************************************************
[WARNING]: when statements should not include jinja2 templating delimiters such as {{ }} or {% %}. Found: inventory_hostname ==
"{{ postgres_ha_cluster_master_host }}"
[WARNING]: when statements should not include jinja2 templating delimiters such as {{ }} or {% %}. Found: inventory_hostname ==
"{{ postgres_ha_cluster_master_host }}"
skipping: [dbs04.prodea-int.net]
FAILED - RETRYING: check if all slaves are connected (16 retries left).
FAILED - RETRYING: check if all slaves are connected (15 retries left).
FAILED - RETRYING: check if all slaves are connected (14 retries left).
changed: [dbs03.prodea-int.net]
PLAY RECAP *************************************************************************************************************************
dbs03.prodea-int.net : ok=54 changed=46 unreachable=0 failed=0
dbs04.prodea-int.net : ok=30 changed=24 unreachable=0 failed=0
[root@AnsibleServer ~]#
[root@localhost ~]# pcs status
Cluster name: pgcluster
Stack: cman
Current DC: dbs04.prodea-int.net (version 1.1.15-5.el6-e174ec8) - partition with quorum
Last updated: Mon Aug 28 23:50:05 2017 Last change: Mon Aug 28 23:39:18 2017 by root via crm_attribute on dbs03.prodea-int.net
2 nodes and 3 resources configured
Online: [ dbs03.prodea-int.net dbs04.prodea-int.net ]
Full list of resources:
pg-vip (ocf::heartbeat:IPaddr2): Started dbs03.prodea-int.net
Master/Slave Set: postgres-ha [postgres]
Slaves: [ dbs03.prodea-int.net dbs04.prodea-int.net ]
Daemon Status:
cman: active/disabled
corosync: active/disabled
pacemaker: active/enabled
pcsd: active/enabled
[root@localhost ~]#