infra-generale/split-brain_resolve.sh

106 lines
2.0 KiB
Bash
Executable File

#!/bin/bash
count=$1
VMs=$(sudo virsh list --state-running --name)
echo "*** Trying to solve split-brain ***"
for vm in $VMs
do
echo MIGRATING $vm ...
sudo virsh migrate --p2p --live --verbose --undefinesource --persistent --copy-storage-all --abort-on-error $vm qemu+ssh://mother.onlink/system
if [ $? -ne 0 ]
then
echo ERROR OCCURED, CANNOT MIGRATE $vm, ABORTING
exit 1
fi
sudo mv /var/backups/vm/aunt/$vm /var/backups/vm/mother
done
if [ "$(sudo virsh list --state-running --name | wc -l)" -ne "1" ];
then
echo ERROR OCCURED, NOT ALL VM MIGRATED, ABORTING
exit 1
fi
echo "UNMOUNTING SHAREDFS"
sudo umount /opt/sharedfs
if [ $? -ne 0 ]
then
if [ ! -z "$(sudo umount /opt/sharedfs | grep 'non monté')" ]
then
echo ERROR OCCURED, CANNOT UNMOUNT SHAREDFS, ABORTING
exit 1
fi
fi
echo "SWITCHING SECONDARY"
sudo drbdadm secondary all
if [ $? -ne 0 ]
then
echo ERROR OCCURED, CANNOT SWITCH SECONDARY, ABORTING
exit 1
fi
echo "*** Waiting for DRBD to sync ***"
if [ $? -ne 0 ]
then
echo ERROR OCCURED, CANNOT SWITCH SECONDARY, ABORTING
exit 1
fi
if [ ! -z "$(cat /proc/drbd | grep Standalone)" ]
then
sudo drbdadm connect all
if [ $? -ne 0 ]
then
echo ERROR OCCURED, CANNOT CONNECT DRBD, ABORTING
exit 1
fi
fi
if [ ! -z "$(cat /proc/drbd | grep WFConnection)" ]
then
ssh mother 'sudo drbdadm connect all'
fi
while [ -z "$(cat /proc/drbd | grep UpToDate/UpToDate)" ]
do
sleep 10
done
echo "SWICHING BACK PRIMARY"
sudo drbdadm primary all
if [ $? -ne 0 ]
then
echo ERROR OCCURED, CANNOT SWITCH BACK PRIMARY, ABORTING
exit 1
fi
echo "MOUNTING SHAREDFS"
sudo mount /opt/sharedfs
if [ $? -ne 0 ]
then
echo ERROR OCCURED, CANNOT MOUNT SHAREDFS, ABORTING
exit 1
fi
echo "*** Migrating from mother to aunt ***"
ssh mother '/opt/sharedfs/outils/migrate_many_to_aunt.sh 12'
if [ $? -ne 0 ]
then
echo ERROR OCCURED, CANNOT MIGRATE BACK VMs, ABORTING
exit 1
fi
echo "*** Successfully solved split-brain ***"