281 lines · bash
1#!/bin/bash2# SPDX-License-Identifier: GPL-2.0+3#4# Run a series of tests on remote systems under KVM.5#6# Usage: kvm-remote.sh "systems" [ <kvm.sh args> ]7# kvm-remote.sh "systems" /path/to/old/run [ <kvm-again.sh args> ]8#9# Copyright (C) 2021 Facebook, Inc.10#11# Authors: Paul E. McKenney <paulmck@kernel.org>12 13scriptname=$014args="$*"15 16if ! test -d tools/testing/selftests/rcutorture/bin17then18 echo $scriptname must be run from top-level directory of kernel source tree.19 exit 120fi21 22RCUTORTURE="`pwd`/tools/testing/selftests/rcutorture"; export RCUTORTURE23PATH=${RCUTORTURE}/bin:$PATH; export PATH24. functions.sh25 26starttime="`get_starttime`"27 28systems="$1"29if test -z "$systems"30then31 echo $scriptname: Empty list of systems will go nowhere good, giving up.32 exit 133fi34shift35 36# Pathnames:37# T: /tmp/kvm-remote.sh.NNNNNN where "NNNNNN" is set by mktemp38# resdir: /tmp/kvm-remote.sh.NNNNNN/res39# rundir: /tmp/kvm-remote.sh.NNNNNN/res/$ds ("-remote" suffix)40# oldrun: `pwd`/tools/testing/.../res/$otherds41#42# Pathname segments:43# TD: kvm-remote.sh.NNNNNN44# ds: yyyy.mm.dd-hh.mm.ss-remote45 46T="`mktemp -d ${TMPDIR-/tmp}/kvm-remote.sh.XXXXXX`"47trap 'rm -rf $T' 048TD="`basename "$T"`"49 50resdir="$T/res"51ds=`date +%Y.%m.%d-%H.%M.%S`-remote52rundir=$resdir/$ds53echo Results directory: $rundir54echo $scriptname $args55if echo $1 | grep -q '^--'56then57 # Fresh build. Create a datestamp unless the caller supplied one.58 datestamp="`echo "$@" | awk -v ds="$ds" '{59 for (i = 1; i < NF; i++) {60 if ($i == "--datestamp") {61 ds = "";62 break;63 }64 }65 if (ds != "")66 print "--datestamp " ds;67 }'`"68 kvm.sh --remote "$@" $datestamp --buildonly > $T/kvm.sh.out 2>&169 ret=$?70 if test "$ret" -ne 071 then72 echo $scriptname: kvm.sh failed exit code $?73 cat $T/kvm.sh.out74 exit 275 fi76 oldrun="`grep -m 1 "^Results directory: " $T/kvm.sh.out | awk '{ print $3 }'`"77 touch "$oldrun/remote-log"78 echo $scriptname $args >> "$oldrun/remote-log"79 echo | tee -a "$oldrun/remote-log"80 echo " ----" kvm.sh output: "(`date`)" | tee -a "$oldrun/remote-log"81 cat $T/kvm.sh.out | tee -a "$oldrun/remote-log"82 # We are going to run this, so remove the buildonly files.83 rm -f "$oldrun"/*/buildonly84 kvm-again.sh $oldrun --dryrun --remote --rundir "$rundir" > $T/kvm-again.sh.out 2>&185 ret=$?86 if test "$ret" -ne 087 then88 echo $scriptname: kvm-again.sh failed exit code $? | tee -a "$oldrun/remote-log"89 cat $T/kvm-again.sh.out | tee -a "$oldrun/remote-log"90 exit 291 fi92else93 # Re-use old run.94 oldrun="$1"95 if ! echo $oldrun | grep -q '^/'96 then97 oldrun="`pwd`/$oldrun"98 fi99 shift100 touch "$oldrun/remote-log"101 echo $scriptname $args >> "$oldrun/remote-log"102 kvm-again.sh "$oldrun" "$@" --dryrun --remote --rundir "$rundir" > $T/kvm-again.sh.out 2>&1103 ret=$?104 if test "$ret" -ne 0105 then106 echo $scriptname: kvm-again.sh failed exit code $? | tee -a "$oldrun/remote-log"107 cat $T/kvm-again.sh.out | tee -a "$oldrun/remote-log"108 exit 2109 fi110 cp -a "$rundir" "$RCUTORTURE/res/"111 oldrun="$RCUTORTURE/res/$ds"112fi113echo | tee -a "$oldrun/remote-log"114echo " ----" kvm-again.sh output: "(`date`)" | tee -a "$oldrun/remote-log"115cat $T/kvm-again.sh.out116echo | tee -a "$oldrun/remote-log"117echo Remote run directory: $rundir | tee -a "$oldrun/remote-log"118echo Local build-side run directory: $oldrun | tee -a "$oldrun/remote-log"119 120# Create the kvm-remote-N.sh scripts in the bin directory.121awk < "$rundir"/scenarios -v dest="$T/bin" -v rundir="$rundir" '122{123 n = $1;124 sub(/\./, "", n);125 fn = dest "/kvm-remote-" n ".sh"126 print "kvm-remote-noreap.sh " rundir " &" > fn;127 scenarios = "";128 for (i = 2; i <= NF; i++)129 scenarios = scenarios " " $i;130 print "kvm-test-1-run-batch.sh" scenarios >> fn;131 print "sync" >> fn;132 print "rm " rundir "/remote.run" >> fn;133}'134chmod +x $T/bin/kvm-remote-*.sh135( cd "`dirname $T`"; tar -chzf $T/binres.tgz "$TD/bin" "$TD/res" )136 137# Check first to avoid the need for cleanup for system-name typos138for i in $systems139do140 ssh -o BatchMode=yes $i getconf _NPROCESSORS_ONLN > $T/ssh.stdout 2> $T/ssh.stderr141 ret=$?142 if test "$ret" -ne 0143 then144 echo "System $i unreachable ($ret), giving up." | tee -a "$oldrun/remote-log"145 echo ' --- ssh stdout: vvv' | tee -a "$oldrun/remote-log"146 cat $T/ssh.stdout | tee -a "$oldrun/remote-log"147 echo ' --- ssh stdout: ^^^' | tee -a "$oldrun/remote-log"148 echo ' --- ssh stderr: vvv' | tee -a "$oldrun/remote-log"149 cat $T/ssh.stderr | tee -a "$oldrun/remote-log"150 echo ' --- ssh stderr: ^^^' | tee -a "$oldrun/remote-log"151 exit 4152 fi153 echo $i: `cat $T/ssh.stdout` CPUs " " `date` | tee -a "$oldrun/remote-log"154done155 156# Download and expand the tarball on all systems.157echo Build-products tarball: `du -h $T/binres.tgz` | tee -a "$oldrun/remote-log"158for i in $systems159do160 echo Downloading tarball to $i `date` | tee -a "$oldrun/remote-log"161 cat $T/binres.tgz | ssh -o BatchMode=yes $i "cd /tmp; tar -xzf -"162 ret=$?163 tries=0164 while test "$ret" -ne 0165 do166 echo Unable to download $T/binres.tgz to system $i, waiting and then retrying. $tries prior retries. | tee -a "$oldrun/remote-log"167 sleep 60168 cat $T/binres.tgz | ssh -o BatchMode=yes $i "cd /tmp; tar -xzf -"169 ret=$?170 if test "$ret" -ne 0171 then172 if test "$tries" > 5173 then174 echo Unable to download $T/binres.tgz to system $i, giving up. | tee -a "$oldrun/remote-log"175 exit 10176 fi177 fi178 tries=$((tries+1))179 done180done181 182# Function to check for presence of a file on the specified system.183# Complain if the system cannot be reached, and retry after a wait.184# Currently just waits forever if a machine disappears.185#186# Usage: checkremotefile system pathname187checkremotefile () {188 local ret189 local sleeptime=60190 191 while :192 do193 ssh -o BatchMode=yes $1 "test -f \"$2\""194 ret=$?195 if test "$ret" -eq 255196 then197 echo " ---" ssh failure to $1 checking for file $2, retry after $sleeptime seconds. `date` | tee -a "$oldrun/remote-log"198 elif test "$ret" -eq 0199 then200 return 0201 elif test "$ret" -eq 1202 then203 echo " ---" File \"$2\" not found: ssh $1 test -f \"$2\" | tee -a "$oldrun/remote-log"204 return 1205 else206 echo " ---" Exit code $ret: ssh $1 test -f \"$2\", retry after $sleeptime seconds. `date` | tee -a "$oldrun/remote-log"207 return $ret208 fi209 sleep $sleeptime210 done211}212 213# Function to start batches on idle remote $systems214#215# Usage: startbatches curbatch nbatches216#217# Batches are numbered starting at 1. Returns the next batch to start.218# Be careful to redirect all debug output to FD 2 (stderr).219startbatches () {220 local curbatch="$1"221 local nbatches="$2"222 local ret223 224 # Each pass through the following loop examines one system.225 for i in $systems226 do227 if test "$curbatch" -gt "$nbatches"228 then229 echo $((nbatches + 1))230 return 0231 fi232 if checkremotefile "$i" "$resdir/$ds/remote.run" 1>&2233 then234 continue # System still running last test, skip.235 fi236 ssh -o BatchMode=yes "$i" "cd \"$resdir/$ds\"; touch remote.run; PATH=\"$T/bin:$PATH\" nohup kvm-remote-$curbatch.sh > kvm-remote-$curbatch.sh.out 2>&1 &" 1>&2237 ret=$?238 if test "$ret" -ne 0239 then240 echo ssh $i failed: exitcode $ret 1>&2241 exit 11242 fi243 echo " ----" System $i Batch `head -n $curbatch < "$rundir"/scenarios | tail -1` `date` 1>&2244 curbatch=$((curbatch + 1))245 done246 echo $curbatch247}248 249# Launch all the scenarios.250nbatches="`wc -l "$rundir"/scenarios | awk '{ print $1 }'`"251curbatch=1252while test "$curbatch" -le "$nbatches"253do254 startbatches $curbatch $nbatches > $T/curbatch 2> $T/startbatches.stderr255 curbatch="`cat $T/curbatch`"256 if test -s "$T/startbatches.stderr"257 then258 cat "$T/startbatches.stderr" | tee -a "$oldrun/remote-log"259 fi260 if test "$curbatch" -le "$nbatches"261 then262 sleep 30263 fi264done265echo All batches started. `date` | tee -a "$oldrun/remote-log"266 267# Wait for all remaining scenarios to complete and collect results.268for i in $systems269do270 echo " ---" Waiting for $i `date` | tee -a "$oldrun/remote-log"271 while checkremotefile "$i" "$resdir/$ds/remote.run"272 do273 sleep 30274 done275 echo " ---" Collecting results from $i `date` | tee -a "$oldrun/remote-log"276 ( cd "$oldrun"; ssh -o BatchMode=yes $i "cd $rundir; tar -czf - kvm-remote-*.sh.out */console.log */kvm-test-1-run*.sh.out */qemu[_-]pid */qemu-retval */qemu-affinity; rm -rf $T > /dev/null 2>&1" | tar -xzf - )277done278 279( kvm-end-run-stats.sh "$oldrun" "$starttime"; echo $? > $T/exitcode ) | tee -a "$oldrun/remote-log"280exit "`cat $T/exitcode`"281