WIP: detecting errors and criticals

boot_and_check.sh

@@ -17,41 +17,104 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-set -e
 source power.inc
 source logs.inc
 
-buffer=mktemp
+buffer=$(mktemp)
 
 greyblue='\033[38;5;245m'
 color_reset='\033[0m'
 
-## Main program ---------------------------------------------------------------
+CRIT=$(mktemp)
-main()
+ERR=$(mktemp)
+WARN=$(mktemp)
+INFO=$(mktemp)
+
+echo 0 > $CRIT
+echo 0 > $ERR
+echo 0 > $WARN
+echo 0 > $INFO
+
+new_critical()
 {
+	local crit=$(cat $CRIT)
+	echo $((crit+1)) > $CRIT
+	
+	echo Stopping logs...
+	stop_logs
+	
+	echo Halting...
+	halt
+	
+	pkill -P $$
+	exit 0
+}
+
+new_error()
+{
+	local err=$(cat $ERR)
+	echo $((err+1)) > $ERR
+}
+
+new_warning()
+{
+	local warn=$(cat $WARN)
+	echo $((warn+1)) > $WARN
+}
+
+new_info()
+{
+	local info=$(cat $INFO)
+	echo $((info+1)) > $INFO
+}
+
+## Main program ---------------------------------------------------------------
 
 echo 'Welcome to the KGPE-D16 debugging tools'
 echo 'Copyright (C) 2024 Adrien "neox" Bourmault <neox@gnu.org>'
+echo " "
+echo Log file at ${buffer}
+echo " "
 
-	stop_logs
+check_logs_still_on
+
+react_to_info $buffer new_info 'starting' &
+react_to_info $buffer new_info 'payload' &
+react_to_warn $buffer new_warning 'Warning' &
+react_to_warn $buffer new_warning 'AMD_CB_EventNotify' &
+react_to_warn $buffer new_warning 'amd_ht_fixup' &
+react_to_err $buffer new_error 'Error' &
+react_to_critical $buffer new_critical 'fatal' &
+react_to_critical $buffer new_critical 'FAULT' &
 
 echo Starting logs...
-	start_logs buffer
+start_logs $buffer
 
 echo Booting up...
 boot
 
-	wait_for buffer "Debian GNU/Linux 9"
+wait_for $buffer 'Debian GNU/Linux 9'
-	echo System successfully booted!
 
-	wait_for buffer "link becomes ready"
+echo Criticals: $(cat $CRIT)
+echo Errors: $(cat $ERR)
+echo Warnings: $(cat $WARN)
+
+if [ "$(cat $CRIT)" -le 0 ]; then
+	echo System successfully booted!
+else
+	echo Summary: fail to boot.
+	pkill -P $$
+	exit 0
+fi
+
+wait_for $buffer 'link becomes ready'
 echo Network ready
 
-	wait_for buffer "tty"
+wait_for $buffer 'tty'
 echo Login should be ready
 
 echo Shutting down
 shutdown
 echo Successful power cycle!
-}
+pkill -P $$
-main
+exit 0

logs.inc

@@ -22,7 +22,23 @@
 source ssh.inc
 
 # colors
-greyblue='\033[38;5;245m'
+black='\033[0;30m'
+red='\033[0;31m'
+green='\033[0;32m'
+yellow='\033[0;33m'
+blue='\033[0;34m'
+magenta='\033[0;35m'
+cyan='\033[0;36m'
+white='\033[0;37m'
+lightgrey='\033[1;30m'
+lightred='\033[1;31m'
+lightgreen='\033[1;32m'
+lightyellow='\033[1;33m'
+lightblue='\033[1;34m'
+lightmagenta='\033[1;35m'
+lightcyan='\033[1;36m'
+lightwhite='\033[1;37m'
+
 color_reset='\033[0m'
 
 get_logs()
@@ -40,21 +56,61 @@ start_logs()
 	get_logs > ${1} &
 }
 
+check_logs_still_on()
+{
+	if [ $(ssh_bmc top -n 1 | grep microcom | wc -l) -ge 1 ]; then
+		echo Stopping logs before starting...
+		stop_logs
+	fi
+}
+
 find_in_logs()
 {
-	
+	keyword=$1
-	keyword=$@
+	while IFS= read -r line
-	while read line
 	do
 		echo "$line" | grep "$keyword" &> /dev/null
 		if [ $? -eq 0 ]; then
-			echo "${greyblue}Found : ${line}${color_reset}"
+			echo -e "${lightgreen}Found : ${line}${color_reset}"
 			break
   		fi
 	done
 }
 
+find_print_and_do()
+{
+	keyword=${1}
+	while IFS= read -r line
+	do
+		echo "$line" | grep "$keyword" &> /dev/null
+		if [ $? -eq 0 ]; then
+			echo -e "${2}${4} : ${line}${color_reset}"
+			${3}
+  		fi
+	done
+}
+
 wait_for()
 {
-	tail -f ${1} | find_in_logs $@
+	tail -F ${1} | find_in_logs ${2}
+}
+
+react_to_info()
+{
+	tail -F ${1} | find_print_and_do ${3} ${cyan} ${2} INFO
+}
+
+react_to_warn()
+{
+	tail -F ${1} | find_print_and_do ${3} ${yellow} ${2} WARN
+}
+
+react_to_err()
+{
+	tail -F ${1} | find_print_and_do ${3} ${red} ${2} ERR
+}
+
+react_to_critical()
+{
+	tail -F ${1} | find_print_and_do ${3} ${lightred} ${2} CRIT
 }

power.inc

@@ -23,7 +23,7 @@ source ssh.inc
 
 boot()
 {
-	ssh_bmc /usr/local/bin/asus_power.sh on
+	ssh_bmc /usr/local/bin/asus_power.sh on &
 }
 
 halt()