Site Tools


New release available: 2022-07-31 "Igor". upgrade now! [52.2] (what's this?)
walkthrough_pcs

Pacemaker Corosync

These pages are describing the walkthrough for the configuration of the GxFS HA Admin Server systems. Prerequirement for this walkthrough is the automatically installed base system (either via RHEL kickstart or via xCAT). The last step in the kickstart automated setup already includes the creation. This is the RAW code provided by Marcus: The last step in the pcs setup automated by RHEL kickstart are as followed:

############################################################
# corosysnc and pacemaker
############################################################
/bin/curl -o /etc/corosync/corosync.conf %%%CFGSRV%%%/corosync.conf-%%%CLUSTER%%%
systemctl start corosync
systemctl start pacemaker
systemctl start pcsd
echo necINST. | passwd hacluster --stdin

Check the correctness of /etc/corosync/corosync.conf. From here the manual setup need to take place:

seems to be not needed: echo drbd >/etc/modules-load.d/drbd.conf
seems to be not needed: ssh gxfsadm0-1 "echo drbd >/etc/modules-load.d/drbd.conf"

[root@gxfsadm10 ~]# pcs cluster auth 192.168.2.122 192.168.2.123
Username: hacluster
Password:
192.168.2.123: Authorized
192.168.2.122: Authorized
[root@gxfsadm10 ~]

After that it shall be checked on both nodes that corosysnc and pacemaker services are enabled and started. Checking the status with pcs cluster status:

[root@gxfsadm1-1 ~]# pcs cluster status
Cluster Status:
 Stack: corosync
 Current DC: gxfsadm1-0 (version 1.1.16-12.el7-94ff4df) - partition with quorum
 Last updated: Thu Apr  1 08:18:35 2010
 Last change: Thu Apr  1 08:18:21 2010 by hacluster via crmd on gxfsadm1-0
 2 nodes configured
 0 resources configured

PCSD Status:
  gxfsadm1-1 (192.168.2.123): Online
  gxfsadm1-0 (192.168.2.122): Online
[root@gxfsadm1-1 ~]#
[root@gxfsadm1-0 ~]# systemctl enable corosync
[root@gxfsadm1-0 ~]# systemctl enable pacemaker
[root@gxfsadm1-0 ~]# systemctl enable pcsd
[root@gxfsadm1-0 ~]#
[root@gxfsadm1-0 corosync]# corosync-cfgtool -s
Printing ring status.
Local node ID 1
RING ID 0
	id	= 192.168.2.122
	status	= ring 0 active with no faults
RING ID 1
	id	= 10.11.121.1
	status	= ring 1 active with no faults
[root@gxfsadm1-0 corosync]#

STONITH configuration between the two HA Admin Servers:

pcs stonith create fence_gxfsadm1-0 fence_ipmilan pcmk_host_list="gxfsadm1-0" method="onoff" ipaddr="192.168.2.21" login="ADMIN" passwd="ADMIN" delay=15 op monitor interval=60s
pcs stonith create fence_gxfsadm1-1 fence_ipmilan pcmk_host_list="gxfsadm1-1" method="onoff" ipaddr="192.168.2.22" login="ADMIN" passwd="ADMIN" delay=15 op monitor interval=60s
pcs property set stonith-action=reboot
pcs property set stonith-enabled=true

in case of trouble for newer ipmi interface lanplus may be enabled:
pcs stonith update fence_gxfsadm0 lanplus="1"
pcs stonith update fence_gxfsadm1 lanplus="1"

#### no quorum, otherwise both nodes would go down ####
pcs property set no-quorum-policy=ignore

new syntax with rocky linux 8

pcs stonith create fence_gxfsadm0 fence_ipmilan pcmk_host_list="gxfsadm0" method="onoff" ip="10.200.100.13" username="admin" password="necINST." delay=15 op monitor interval=60s

PCS setup for all volumes including the according VIPs for the containers in that PCS group

DATABASE
========
pcs resource create drbd-database_vol1 ocf:linbit:drbd drbd_resource=database_vol1 op monitor interval=60s
OLD: pcs resource master drbd-database_vol1Clone drbd-database_vol1 master-max=1 master-node-max=1 clone-max=2 clone-node-max=1 notify=true
NEW: pcs resource promotable drbd-database_vol1 promoted-max=1 promoted-node-max=1 clone-max=2 clone-node-max=1 notify=true
pcs resource create database_vol1-fs ocf:heartbeat:Filesystem device="/dev/drbd100" directory="/drbd/database_vol1" fstype="xfs" --group database

pcs resource create VIP_maridb-fdr IPaddr2 ip=10.11.121.110 cidr_netmask=24 op monitor interval=5s --group database

pcs constraint colocation add database with drbd-database_vol1Clone INFINITY with-rsc-role=Master
pcs constraint order promote drbd-database_vol1Clone then start database
pcs resource cleanup drbd-database_vol1Clone
pcs resource cleanup database_vol1-fs

MONITORING
==========
pcs resource create drbd-monitoring_vol1 ocf:linbit:drbd drbd_resource=monitoring_vol1 op monitor interval=60s
pcs resource master drbd-monitoring_vol1Clone drbd-monitoring_vol1 master-max=1 master-node-max=1 clone-max=2 clone-node-max=1 notify=true
pcs resource create monitoring_vol1-fs ocf:heartbeat:Filesystem device="/dev/drbd101" directory="/drbd/monitoring_vol1" fstype="xfs" --group monitoring

pcs resource create VIP_icnga2-fdr IPaddr2 ip=10.11.121.111 cidr_netmask=24 op monitor interval=5s --group monitoring
pcs resource create VIP_icnga2-adm IPaddr2 ip=10.67.122.111 cidr_netmask=24 op monitor interval=5s --group monitoring
pcs resource create VIP_icnga2-mgt IPaddr2 ip=10.78.1.111   cidr_netmask=24 op monitor interval=5s --group monitoring
pcs resource create VIP_icnga2-roc IPaddr2 ip=172.18.3.111  cidr_netmask=24 op monitor interval=5s --group monitoring

pcs constraint colocation add monitoring with drbd-monitoring_vol1Clone INFINITY with-rsc-role=Master
pcs constraint order promote drbd-monitoring_vol1Clone then start monitoring
pcs resource cleanup drbd-monitoring_vol1
pcs resource cleanup monitoring_vol1-fs


WEBAPPS
=======
pcs resource create drbd-webapps_vol1 ocf:linbit:drbd drbd_resource=webapps_vol1 op monitor interval=60s
pcs resource master drbd-webapps_vol1Clone drbd-webapps_vol1 master-max=1 master-node-max=1 clone-max=2 clone-node-max=1 notify=true
pcs resource create webapps_vol1-fs ocf:heartbeat:Filesystem device="/dev/drbd102" directory="/drbd/webapps_vol1" fstype="xfs" --group webapps

pcs resource create VIP_icngw2-fdr IPaddr2 ip=10.11.121.112 cidr_netmask=24 op monitor interval=5s --group webapps
pcs resource create VIP_nginx-ext IPaddr2 ip=192.168.2.83 cidr_netmask=24 op monitor interval=5s --group webapps
pcs resource create VIP_nginx-adm IPaddr2 ip=10.67.122.100 cidr_netmask=24 op monitor interval=5s --group webapps
pcs resource create VIP_nginx-mgt IPaddr2 ip=10.78.1.100 cidr_netmask=24 op monitor interval=5s --group webapps
pcs resource create VIP_nginx-fdr IPaddr2 ip=10.11.121.100 cidr_netmask=24 op monitor interval=5s --group webapps
pcs resource create VIP_websp-fdr IPaddr2 ip=10.11.121.113 cidr_netmask=24 op monitor interval=5s --group webapps
pcs resource create VIP_websp-mgt IPaddr2 ip=10.78.1.113 cidr_netmask=24 op monitor interval=5s --group webapps

pcs constraint colocation add webapps with drbd-webapps_vol1Clone INFINITY with-rsc-role=Master
pcs constraint order promote drbd-webapps_vol1Clone then start webapps
pcs resource cleanup drbd-webapps_vol1
pcs resource cleanup webapps_vol1-fs

HOME
====
pcs resource create drbd-home_vol1 ocf:linbit:drbd drbd_resource=home_vol1 op monitor interval=60s
pcs resource master drbd-home_vol1Clone drbd-home_vol1 master-max=1 master-node-max=1 clone-max=2 clone-node-max=1 notify=true
pcs resource create home_vol1-fs ocf:heartbeat:Filesystem device="/dev/drbd104" directory="/drbd/home_vol1" fstype="xfs" --group homefs

pcs resource create VIP_home-gbe IPaddr2 ip=10.1.17.226 cidr_netmask=24 op monitor interval=5s --group homefs

pcs constraint colocation add homefs with drbd-home_vol1Clone INFINITY with-rsc-role=Master
pcs constraint order promote drbd-home_vol1Clone then start homefs
pcs resource cleanup drbd-home_vol1Clone
pcs resource cleanup home_vol1-fs


PROVISIONING
============
### do NOT group provisioning_vol1-fs to the xcat group! colocation and order for xCAT services will be configured later ###
pcs resource create drbd-provisioning_vol1 ocf:linbit:drbd drbd_resource=provisioning_vol1 op monitor interval=60s
pcs resource master drbd-provisioning_vol1Clone drbd-provisioning_vol1 master-max=1 master-node-max=1 clone-max=2 clone-node-max=1 notify=true
pcs resource create provisioning_vol1-fs ocf:heartbeat:Filesystem device="/dev/drbd103" directory="/drbd/provisioning_vol1" fstype="xfs"
pcs constraint order promote drbd-provisioning_vol1Clone then start provisioning_vol1-fs
pcs constraint colocation add provisioning_vol1-fs with drbd-provisioning_vol1Clone INFINITY with-rsc-role=Master
pcs resource cleanup drbd-provisioning_vol1
pcs resource cleanup provisioning_vol1-fs

### webservices proxy will be attached to icinga2_vol1, but has it's own pcs group ###
pcs resource create docker_websp ocf:heartbeat:docker image="necgxfs/websp_v1.0" op monitor timeout="30s" interval="30s"
pcs resource update docker_websp reuse=1 name=websp
pcs resource group add websp docker_websp
pcs constraint order promote drbd-icinga2_vol1Clone then start websp
pcs constraint colocation add websp with drbd-icinga2_vol1Clone INFINITY with-rsc-role=Master
pcs resource cleanup websp
pcs resource create VIP_icinga2 IPaddr2 ip=192.168.2.155 cidr_netmask=24 op monitor interval=5s --group websp

Now let's install and set up xCAT & dhcpd & named…

### install xCAT packages from the tarballs (either have install host with repo or create local repo) ###
mkdir /tmp/xcat
cd /tmp/xcat
wget https://xcat.org/files/xcat/xcat-core/2.14.x_Linux/xcat-core/xcat-core-2.14.0-linux.tar.bz2
wget https://xcat.org/files/xcat/xcat-dep/2.x_Linux/xcat-dep-2.14.0-linux.tar.bz2
tar xvjpf xcat-core-2.14.0-linux.tar.bz2 xcat-dep-2.14.0-linux.tar.bz2
rsync -av xcat-core/ <target repo dir>/
rsync -av xcat-dep/ <target repo dir>/
cd <target repo dir>/
createrepo .

### create a repo file (example is for install host-based repo) ###
cd /etc/yum.repos.d
cat << EOF > xcat.repo
[gxfsmaster-xcat-core]
name=gxfsmaster-xcat-core
enabled=1
baseurl=http://192.168.2.10/xcat
gpgcheck=0

[gxfsmaster-xcat-dep]
name=gxfsmaster-xcat-dep
enabled=1
baseurl=http://192.168.2.10/xcat/rh7/x86_64
gpgcheck=1
gpgkey=http://192.168.2.10/xcat/rh7/x86_64/repodata/repomd.xml.key
EOF

### install xCAT RPMs ###
yum install xCAT
systemctl stop xcatd
systemctl stop dhcpd
systemctl stop named
systemctl disable xcatd
systemctl disable dhcpd
systemctl disable named

### do these operations on the active xCAT volume owner! ###
cd /drbd/xcat_vol1/
mkdir -p etc/xcat install root/.xcat tftpboot var/lib/dhcpd etc/dhcp var/named
rsync -av /etc/xcat/ /drbd/xcat_vol1/etc/xcat/
rsync -av /root/.xcat/ /drbd/xcat_vol1/root/.xcat/

### now empty the xCAT-related directories on both hosts, but first move the DRBD away ###
rm -rf /etc/xcat/* /install/* /root/.xcat/* /tftpboot/* /var/lib/dhcpd/* /etc/dhcp/* /var/named/*

### copy NEC OCF scripts from the install host ###
rsync -av /usr/lib/ocf/resource.d/nec gxfsadm0-[0,1]:/usr/lib/ocf/resource.d/

### now create xCAT services ###
pcs resource create VIP_xcat IPaddr2 ip=<xCAT IP> cidr_netmask=24 op monitor interval=5s
pcs resource create xcat-etc ocf:nec:Filesystem device="/drbd/tools_vol1/etc/xcat" directory="/etc/xcat" fstype=none options=bind op monitor interval=60 meta resource-stickiness=0
pcs resource create xcat-install ocf:nec:Filesystem device="/drbd/tools_vol1/install" directory="/install" fstype=none options=bind op monitor interval=60 meta resource-stickiness=0
pcs resource create xcat-root ocf:nec:Filesystem device="/drbd/tools_vol1/root/.xcat" directory="/root/.xcat" fstype=none options=bind op monitor interval=60 meta resource-stickiness=0
pcs resource create xcat-tftpboot ocf:nec:Filesystem device="/drbd/tools_vol1/tftpboot" directory="/tftpboot" fstype=none options=bind op monitor interval=60 meta resource-stickiness=0
pcs resource create xcatd systemd:xcatd op monitor interval=60 meta resource-stickiness=0
pcs resource group add xcat VIP_xcat xcat-etc xcat-install xcat-root xcat-tftpboot xcatd apache
pcs constraint colocation add xcat with drbd-tools_vol1Clone INFINITY with-rsc-role=Master
pcs constraint order start tools_vol1-fs then start VIP_xcat
pcs constraint order start tools_vol1-fs then start xcat-etc
pcs constraint order start tools_vol1-fs then start xcat-install
pcs constraint order start tools_vol1-fs then start xcat-root
pcs constraint order start tools_vol1-fs then start xcat-tftpboot
pcs constraint order start xcat-etc then start xcatd
pcs constraint order start xcatd then start apache
pcs resource create dhcp-etc ocf:nec:Filesystem device="/drbd/tools_vol1/etc/dhcp" directory="/etc/dhcp" fstype=none options=bind op monitor interval=60 meta resource-stickiness=0
pcs resource create dhcp-state ocf:nec:Filesystem device="/drbd/tools_vol1/var/lib/dhcpd" directory="/var/lib/dhcpd" fstype=none options=bind op monitor interval=60 meta resource-stickiness=0
pcs resource create dhcpd systemd:dhcpd op monitor interval=60 meta resource-stickiness=0
pcs resource group add dhcp dhcp-etc dhcp-state dhcpd
pcs constraint colocation add dhcp with drbd-tools_vol1Clone INFINITY with-rsc-role=Master
pcs constraint order start tools_vol1-fs then start dhcp-etc
pcs constraint order start tools_vol1-fs then start dhcp-state
pcs constraint order start dhcp-etc then start dhcpd
pcs resource create dns-state ocf:nec:Filesystem device="/drbd/tools_vol1/var/named" directory="/var/named" fstype=none options=bind op monitor interval=60 meta resource-stickiness=0
pcs resource create named systemd:named op monitor interval=60 meta resource-stickiness=0
pcs resource group add dns dns-state named
pcs constraint colocation add dns with drbd-tools_vol1Clone INFINITY with-rsc-role=Master
pcs constraint order start tools_vol1-fs then start dns-state
pcs constraint order start dns-state then start named
walkthrough_pcs.txt · Last modified: 2022/07/10 09:34 by 127.0.0.1