#!/bin/bash
#################################################################################
#                                                                               #
# iscsi-ha - High Availability framework for iSCSI cluster used in conjunction  #
# with XAPI based Xen Virtualization Environment (Xen Cloud Platform/XenServer) #
# Copyright 2021 Salvatore Costantino                                           #
# ha@pulsesupply.com                                                            #
#                                                                               #
#                                                                               #
#    iscsi-ha is free software: you can redistribute it and/or modify           #
#    it under the terms of the GNU General Public License as published by       #
#    the Free Software Foundation, either version 3 of the License, or          #
#    (at your option) any later version.                                        #
#                                                                               #
#    iscsi-ha is distributed in the hope that it will be useful,                #
#    but WITHOUT ANY WARRANTY; without even the implied warranty of             #
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the              #
#    GNU General Public License for more details.                               #
#                                                                               #
#    You should have received a copy of the GNU General Public License          #
#    along with iscsi-ha.  If not, see <http://www.gnu.org/licenses/>.          #
#                                                                               #
#################################################################################

DRBD_CONF=/etc/drbd.conf
VALID_NUMBER='^[0-9]+$'
WARNING="
##########################################
##########################################
#### DRBD SPLIT BRAIN RECOVERY SCRIPT ####
#### DO NOT PROCEED IF YOU ARE NOT    ####
#### SURE OF WHAT YOU ARE DOING       ####
#### !! YOU COULD LOSE YOUR DATA !!   ####
##########################################
##########################################"

echo "$WARNING"
echo "PROCEED? [y|n]"
read PROCEED
if [ "$PROCEED" != "y" ]
then
	exit 0
fi


#######################
# select the resource
#######################
RESOURCES_ARRAY=()
SUCCESS=false
COUNT=1
RESOURCE_LIST=$(cat $DRBD_CONF | grep resource | awk {'print $2'})

if [ $(echo "$RESOURCE_LIST" | wc -l) -gt 1 ]
then
	for resource in ${RESOURCE_LIST[@]}
	do
		RESOURCES_ARRAY[$COUNT]=$resource
		echo "Select [ $COUNT ] for [ $resource ]"
		COUNT=$(($COUNT+1))
	done
	COUNT=$(($COUNT-1))
else
	RESOURCES_ARRAY[$COUNT]=$RESOURCE_LIST
fi

if [ "$COUNT" -gt 1 ]
then
	echo "Select 1 through $COUNT"
	DRBD_RESOURCE_SELECT=0
else
	DRBD_RESOURCE_SELECT=1
	echo "Only one resource found [ ${RESOURCES_ARRAY[$DRBD_RESOURCE_SELECT]} ]"
fi

while [ "$DRBD_RESOURCE_SELECT" -lt "1" -o "$DRBD_RESOURCE_SELECT" -gt "$COUNT" ]
do
	echo "Make a  selection"
	read DRBD_RESOURCE_SELECT
	if ! [[ $DRBD_RESOURCE_SELECT =~ $VALID_NUMBER ]]
	then
		DRBD_RESOURCE_SELECT=0
	fi
done			
		
echo "[ ${RESOURCES_ARRAY[$DRBD_RESOURCE_SELECT]} ] selected"

#######################
# are we the surviving
# data set?
#######################
SUCCESS=false
while [ "$SUCCESS" != "true" ]
do
	echo "Is this host the split brain [s]urvivor or [v]ictim?"
	read ROLE

	#############################
	# This node is the survivor
	#############################
	if [ "$ROLE" = "s" ]
	then
		echo "[ survivor ] selected"
		SUCCESS=true
		MY_STATE=$(drbdadm cstate ${RESOURCES_ARRAY[$DRBD_RESOURCE_SELECT]})
		if [ "$MY_STATE" != "WFConnection" ]
		then
			MY_NEW_STATE=""
			MY_NEW_STATE_COUNT=0
			while [ "$MY_NEW_STATE" != "WFConnection" ]
			do
				drbdadm connect ${RESOURCES_ARRAY[$DRBD_RESOURCE_SELECT]} 2> /dev/null
				MY_NEW_STATE=$(drbdadm cstate ${RESOURCES_ARRAY[$DRBD_RESOURCE_SELECT]})
				sleep 1
				MY_NEW_STATE_COUNT=$(($MY_NEW_STATE_COUNT+1))
				if [ "$MY_NEW_STATE_COUNT" -gt 10 ]
				then
					echo "Failed to assert [ WFConnection ] state"
					exit 1
				fi
			done
		fi
		drbdadm primary ${RESOURCES_ARRAY[$DRBD_RESOURCE_SELECT]}
		RETVAL=$?
		if [ $RETVAL -eq 0 ]
		then
			echo "Resource [ ${RESOURCES_ARRAY[$DRBD_RESOURCE_SELECT]} ] ready."
			echo "Run this script on the victim host to complete the process"
			while [ "$MY_NEW_STATE" != "Connected" ]
			do
				drbdadm connect ${RESOURCES_ARRAY[$DRBD_RESOURCE_SELECT]} 2> /dev/null
				MY_NEW_STATE=$(drbdadm cstate ${RESOURCES_ARRAY[$DRBD_RESOURCE_SELECT]})
				sleep 1
			done			
			echo "Press <enter> to watch live resource status or <control +c> to exit"
			read ENTER
			while :
			do
				clear
				echo "CONTROL + C to exit"
				cat /proc/drbd
				sleep .5
			done
			exit 0
		else
			echo "Failed to disconnect resource [ ${RESOURCES_ARRAY[$DRBD_RESOURCE_SELECT]} ]"
			echo "Try manually recovering"
			exit 1
		fi

	#############################
	# this node is the victim
	#############################
	elif [ "$ROLE" = "v" ]
	then
		echo "[ victim ] selected"
		SUCCESS=true
		echo
		echo "Are you sure you want to proceed?"
		echo "Make sure to run this script on the survivor before proceeding"
		echo "Proceeding will invalidate the data on resource [ ${RESOURCES_ARRAY[$DRBD_RESOURCE_SELECT]} ]"
		echo "PROCEED? [y|n]"
		read PROCEED
		if [ "$PROCEED" != "y" ]
		then
		        exit 0
		fi	
		MY_STATE=$(drbdadm cstate ${RESOURCES_ARRAY[$DRBD_RESOURCE_SELECT]})
		if [ "$MY_STATE" != "StandAlone" ]
		then
			drbdadm disconnect ${RESOURCES_ARRAY[$DRBD_RESOURCE_SELECT]} 2> /dev/null
		fi
		drbdadm secondary ${RESOURCES_ARRAY[$DRBD_RESOURCE_SELECT]} 2> /dev/null
		drbdadm connect --discard-my-data ${RESOURCES_ARRAY[$DRBD_RESOURCE_SELECT]}
		RETVAL=$?
		if [ $RETVAL -eq 0 ]
		then
			while :
			do
				clear
				echo "SUCCESS"
				echo "CONTROL + C to exit"
				cat /proc/drbd
				sleep .5
			done
		else
			echo "Failed to sync resource [ ${RESOURCES_ARRAY[$DRBD_RESOURCE_SELECT]} ]"
			echo "Try manually recovering"
			exit 1
		fi
	fi
done