123 lines
2.8 KiB
Bash
Executable File
123 lines
2.8 KiB
Bash
Executable File
#!/bin/bash
|
|
# Script to solve split brain situations by copying things smartly
|
|
#
|
|
# Copyright (C) 2023 Adrien 'neox' Bourmault <neox@a-lec.org>
|
|
#
|
|
# This program is free software: you can redistribute it and/or modify
|
|
# it under the terms of the GNU Affero General Public License as published by
|
|
# the Free Software Foundation, either version 3 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU Affero General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU Affero General Public License
|
|
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
|
|
count=$1
|
|
VMs=$(sudo virsh list --state-running --name)
|
|
|
|
echo "*** Trying to solve split-brain ***"
|
|
|
|
for vm in $VMs
|
|
do
|
|
echo MIGRATING $vm ...
|
|
sudo virsh migrate --p2p --live --verbose --undefinesource --persistent --copy-storage-all --abort-on-error $vm qemu+ssh://mother.onlink/system
|
|
|
|
if [ $? -ne 0 ]
|
|
then
|
|
echo ERROR OCCURED, CANNOT MIGRATE $vm, ABORTING
|
|
exit 1
|
|
fi
|
|
|
|
sudo mv /var/backups/vm/aunt/$vm /var/backups/vm/mother
|
|
done
|
|
|
|
if [ "$(sudo virsh list --state-running --name | wc -l)" -ne "1" ];
|
|
then
|
|
echo ERROR OCCURED, NOT ALL VM MIGRATED, ABORTING
|
|
exit 1
|
|
fi
|
|
|
|
echo "UNMOUNTING SHAREDFS"
|
|
sudo umount /opt/sharedfs
|
|
|
|
if [ $? -ne 0 ]
|
|
then
|
|
if [ ! -z "$(sudo umount /opt/sharedfs | grep 'non monté')" ]
|
|
then
|
|
echo ERROR OCCURED, CANNOT UNMOUNT SHAREDFS, ABORTING
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
echo "SWITCHING SECONDARY"
|
|
sudo drbdadm secondary all
|
|
|
|
if [ $? -ne 0 ]
|
|
then
|
|
echo ERROR OCCURED, CANNOT SWITCH SECONDARY, ABORTING
|
|
exit 1
|
|
fi
|
|
|
|
echo "*** Waiting for DRBD to sync ***"
|
|
|
|
if [ $? -ne 0 ]
|
|
then
|
|
echo ERROR OCCURED, CANNOT SWITCH SECONDARY, ABORTING
|
|
exit 1
|
|
fi
|
|
|
|
if [ ! -z "$(cat /proc/drbd | grep Standalone)" ]
|
|
then
|
|
sudo drbdadm connect all
|
|
|
|
if [ $? -ne 0 ]
|
|
then
|
|
echo ERROR OCCURED, CANNOT CONNECT DRBD, ABORTING
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
if [ ! -z "$(cat /proc/drbd | grep WFConnection)" ]
|
|
then
|
|
ssh mother 'sudo drbdadm connect all'
|
|
fi
|
|
|
|
|
|
while [ -z "$(cat /proc/drbd | grep UpToDate/UpToDate)" ]
|
|
do
|
|
sleep 10
|
|
done
|
|
|
|
echo "SWICHING BACK PRIMARY"
|
|
sudo drbdadm primary all
|
|
|
|
if [ $? -ne 0 ]
|
|
then
|
|
echo ERROR OCCURED, CANNOT SWITCH BACK PRIMARY, ABORTING
|
|
exit 1
|
|
fi
|
|
|
|
echo "MOUNTING SHAREDFS"
|
|
sudo mount /opt/sharedfs
|
|
|
|
if [ $? -ne 0 ]
|
|
then
|
|
echo ERROR OCCURED, CANNOT MOUNT SHAREDFS, ABORTING
|
|
exit 1
|
|
fi
|
|
|
|
echo "*** Migrating from mother to aunt ***"
|
|
ssh mother '/opt/sharedfs/outils/migrate_many_to_aunt.sh 12'
|
|
|
|
if [ $? -ne 0 ]
|
|
then
|
|
echo ERROR OCCURED, CANNOT MIGRATE BACK VMs, ABORTING
|
|
exit 1
|
|
fi
|
|
|
|
echo "*** Successfully solved split-brain ***"
|