infra-generale/split-brain_resolve.sh

#!/bin/bash
# Script to solve split brain situations by copying things smartly
#
# Copyright (C) 2023 Adrien 'neox' Bourmault <neox@a-lec.org>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <https://www.gnu.org/licenses/>.

count=$1
VMs=$(sudo virsh list --state-running --name)

echo "*** Trying to solve split-brain ***"

for vm in $VMs
do
    echo MIGRATING $vm ...
    sudo virsh migrate --p2p --live --verbose --undefinesource --persistent --copy-storage-all --abort-on-error $vm qemu+ssh://mother.onlink/system

    if [ $? -ne 0 ]
    then
        echo ERROR OCCURED, CANNOT MIGRATE $vm, ABORTING
        exit 1
    fi

    sudo mv /var/backups/vm/aunt/$vm /var/backups/vm/mother
done

if [ "$(sudo virsh list --state-running --name | wc -l)" -ne "1" ];
then
    echo ERROR OCCURED, NOT ALL VM MIGRATED, ABORTING
    exit 1
fi

echo "UNMOUNTING SHAREDFS"
sudo umount /opt/sharedfs

if [ $? -ne 0 ]
then
    if [ ! -z "$(sudo umount /opt/sharedfs | grep 'non monté')" ]
    then
        echo ERROR OCCURED, CANNOT UNMOUNT SHAREDFS, ABORTING
        exit 1
    fi
fi

echo "SWITCHING SECONDARY"
sudo drbdadm secondary all

if [ $? -ne 0 ]
then
    echo ERROR OCCURED, CANNOT SWITCH SECONDARY, ABORTING
    exit 1
fi

echo "*** Waiting for DRBD to sync ***"

if [ $? -ne 0 ]
then
    echo ERROR OCCURED, CANNOT SWITCH SECONDARY, ABORTING
    exit 1
fi

if [ ! -z "$(cat /proc/drbd | grep Standalone)" ]
then
    sudo drbdadm connect all

    if [ $? -ne 0 ]
    then
        echo ERROR OCCURED, CANNOT CONNECT DRBD, ABORTING
        exit 1
    fi
fi

if [ ! -z "$(cat /proc/drbd | grep WFConnection)" ]
then
    ssh mother 'sudo drbdadm connect all'
fi


while [ -z "$(cat /proc/drbd | grep UpToDate/UpToDate)" ]
do
    sleep 10
done

echo "SWICHING BACK PRIMARY"
sudo drbdadm primary all

if [ $? -ne 0 ]
then
    echo ERROR OCCURED, CANNOT SWITCH BACK PRIMARY, ABORTING
    exit 1
fi

echo "MOUNTING SHAREDFS"
sudo mount /opt/sharedfs

if [ $? -ne 0 ]
then
    echo ERROR OCCURED, CANNOT MOUNT SHAREDFS, ABORTING
    exit 1
fi

echo "*** Migrating from mother to aunt ***"
ssh mother '/opt/sharedfs/outils/migrate_many_to_aunt.sh 12'

if [ $? -ne 0 ]
then
    echo ERROR OCCURED, CANNOT MIGRATE BACK VMs, ABORTING
    exit 1
fi

echo "*** Successfully solved split-brain ***"