632 lines · bash
1#!/bin/bash2# SPDX-License-Identifier: GPL-2.0-or-later3# Copyright (c) 2016 Microsemi. All Rights Reserved.4#5# Author: Logan Gunthorpe <logang@deltatee.com>6 7REMOTE_HOST=8LIST_DEVS=FALSE9 10DEBUGFS=${DEBUGFS-/sys/kernel/debug}11 12PERF_RUN_ORDER=3213MAX_MW_SIZE=014RUN_DMA_TESTS=15DONT_CLEANUP=16MW_SIZE=6553617 18function show_help()19{20 echo "Usage: $0 [OPTIONS] LOCAL_DEV REMOTE_DEV"21 echo "Run tests on a pair of NTB endpoints."22 echo23 echo "If the NTB device loops back to the same host then,"24 echo "just specifying the two PCI ids on the command line is"25 echo "sufficient. Otherwise, if the NTB link spans two hosts"26 echo "use the -r option to specify the hostname for the remote"27 echo "device. SSH will then be used to test the remote side."28 echo "An SSH key between the root users of the host would then"29 echo "be highly recommended."30 echo31 echo "Options:"32 echo " -C don't cleanup ntb modules on exit"33 echo " -h show this help message"34 echo " -l list available local and remote PCI ids"35 echo " -r REMOTE_HOST specify the remote's hostname to connect"36 echo " to for the test (using ssh)"37 echo " -m MW_SIZE memory window size for ntb_tool"38 echo " (default: $MW_SIZE)"39 echo " -d run dma tests for ntb_perf"40 echo " -p ORDER total data order for ntb_perf"41 echo " (default: $PERF_RUN_ORDER)"42 echo " -w MAX_MW_SIZE maxmium memory window size for ntb_perf"43 echo44}45 46function parse_args()47{48 OPTIND=049 while getopts "b:Cdhlm:r:p:w:" opt; do50 case "$opt" in51 C) DONT_CLEANUP=1 ;;52 d) RUN_DMA_TESTS=1 ;;53 h) show_help; exit 0 ;;54 l) LIST_DEVS=TRUE ;;55 m) MW_SIZE=${OPTARG} ;;56 r) REMOTE_HOST=${OPTARG} ;;57 p) PERF_RUN_ORDER=${OPTARG} ;;58 w) MAX_MW_SIZE=${OPTARG} ;;59 \?)60 echo "Invalid option: -$OPTARG" >&261 exit 162 ;;63 esac64 done65}66 67parse_args "$@"68shift $((OPTIND-1))69LOCAL_DEV=$170shift71parse_args "$@"72shift $((OPTIND-1))73REMOTE_DEV=$174shift75parse_args "$@"76 77set -e78 79function _modprobe()80{81 modprobe "$@" || return 182 83 if [[ "$REMOTE_HOST" != "" ]]; then84 ssh "$REMOTE_HOST" modprobe "$@" || return 185 fi86}87 88function split_remote()89{90 VPATH=$191 REMOTE=92 93 if [[ "$VPATH" == *":/"* ]]; then94 REMOTE=${VPATH%%:*}95 VPATH=${VPATH#*:}96 fi97}98 99function read_file()100{101 split_remote $1102 if [[ "$REMOTE" != "" ]]; then103 ssh "$REMOTE" cat "$VPATH"104 else105 cat "$VPATH"106 fi107}108 109function write_file()110{111 split_remote $2112 VALUE=$1113 114 if [[ "$REMOTE" != "" ]]; then115 ssh "$REMOTE" "echo \"$VALUE\" > \"$VPATH\""116 else117 echo "$VALUE" > "$VPATH"118 fi119}120 121function check_file()122{123 split_remote $1124 125 if [[ "$REMOTE" != "" ]]; then126 ssh "$REMOTE" "[[ -e ${VPATH} ]]"127 else128 [[ -e ${VPATH} ]]129 fi130}131 132function subdirname()133{134 echo $(basename $(dirname $1)) 2> /dev/null135}136 137function find_pidx()138{139 PORT=$1140 PPATH=$2141 142 for ((i = 0; i < 64; i++)); do143 PEER_DIR="$PPATH/peer$i"144 145 check_file ${PEER_DIR} || break146 147 PEER_PORT=$(read_file "${PEER_DIR}/port")148 if [[ ${PORT} -eq $PEER_PORT ]]; then149 echo $i150 return 0151 fi152 done153 154 return 1155}156 157function port_test()158{159 LOC=$1160 REM=$2161 162 echo "Running port tests on: $(basename $LOC) / $(basename $REM)"163 164 LOCAL_PORT=$(read_file "$LOC/port")165 REMOTE_PORT=$(read_file "$REM/port")166 167 LOCAL_PIDX=$(find_pidx ${REMOTE_PORT} "$LOC")168 REMOTE_PIDX=$(find_pidx ${LOCAL_PORT} "$REM")169 170 echo "Local port ${LOCAL_PORT} with index ${REMOTE_PIDX} on remote host"171 echo "Peer port ${REMOTE_PORT} with index ${LOCAL_PIDX} on local host"172 173 echo " Passed"174}175 176function link_test()177{178 LOC=$1179 REM=$2180 EXP=0181 182 echo "Running link tests on: $(subdirname $LOC) / $(subdirname $REM)"183 184 if ! write_file "N" "$LOC/../link" 2> /dev/null; then185 echo " Unsupported"186 return187 fi188 189 write_file "N" "$LOC/link_event"190 191 if [[ $(read_file "$REM/link") != "N" ]]; then192 echo "Expected link to be down in $REM/link" >&2193 exit -1194 fi195 196 write_file "Y" "$LOC/../link"197 198 echo " Passed"199}200 201function doorbell_test()202{203 LOC=$1204 REM=$2205 EXP=0206 207 echo "Running db tests on: $(basename $LOC) / $(basename $REM)"208 209 DB_VALID_MASK=$(read_file "$LOC/db_valid_mask")210 211 write_file "c $DB_VALID_MASK" "$REM/db"212 213 for ((i = 0; i < 64; i++)); do214 DB=$(read_file "$REM/db")215 if [[ "$DB" -ne "$EXP" ]]; then216 echo "Doorbell doesn't match expected value $EXP " \217 "in $REM/db" >&2218 exit -1219 fi220 221 let "MASK = (1 << $i) & $DB_VALID_MASK" || true222 let "EXP = $EXP | $MASK" || true223 224 write_file "s $MASK" "$LOC/peer_db"225 done226 227 write_file "c $DB_VALID_MASK" "$REM/db_mask"228 write_file $DB_VALID_MASK "$REM/db_event"229 write_file "s $DB_VALID_MASK" "$REM/db_mask"230 231 write_file "c $DB_VALID_MASK" "$REM/db"232 233 echo " Passed"234}235 236function get_files_count()237{238 NAME=$1239 LOC=$2240 241 split_remote $LOC242 243 if [[ "$REMOTE" == "" ]]; then244 echo $(ls -1 "$VPATH"/${NAME}* 2>/dev/null | wc -l)245 else246 echo $(ssh "$REMOTE" "ls -1 \"$VPATH\"/${NAME}* | \247 wc -l" 2> /dev/null)248 fi249}250 251function scratchpad_test()252{253 LOC=$1254 REM=$2255 256 echo "Running spad tests on: $(subdirname $LOC) / $(subdirname $REM)"257 258 CNT=$(get_files_count "spad" "$LOC")259 260 if [[ $CNT -eq 0 ]]; then261 echo " Unsupported"262 return263 fi264 265 for ((i = 0; i < $CNT; i++)); do266 VAL=$RANDOM267 write_file "$VAL" "$LOC/spad$i"268 RVAL=$(read_file "$REM/../spad$i")269 270 if [[ "$VAL" -ne "$RVAL" ]]; then271 echo "Scratchpad $i value $RVAL doesn't match $VAL" >&2272 exit -1273 fi274 done275 276 echo " Passed"277}278 279function message_test()280{281 LOC=$1282 REM=$2283 284 echo "Running msg tests on: $(subdirname $LOC) / $(subdirname $REM)"285 286 CNT=$(get_files_count "msg" "$LOC")287 288 if [[ $CNT -eq 0 ]]; then289 echo " Unsupported"290 return291 fi292 293 MSG_OUTBITS_MASK=$(read_file "$LOC/../msg_inbits")294 MSG_INBITS_MASK=$(read_file "$REM/../msg_inbits")295 296 write_file "c $MSG_OUTBITS_MASK" "$LOC/../msg_sts"297 write_file "c $MSG_INBITS_MASK" "$REM/../msg_sts"298 299 for ((i = 0; i < $CNT; i++)); do300 VAL=$RANDOM301 write_file "$VAL" "$LOC/msg$i"302 RVAL=$(read_file "$REM/../msg$i")303 304 if [[ "$VAL" -ne "${RVAL%%<-*}" ]]; then305 echo "Message $i value $RVAL doesn't match $VAL" >&2306 exit -1307 fi308 done309 310 echo " Passed"311}312 313function get_number()314{315 KEY=$1316 317 sed -n "s/^\(${KEY}\)[ \t]*\(0x[0-9a-fA-F]*\)\(\[p\]\)\?$/\2/p"318}319 320function mw_alloc()321{322 IDX=$1323 LOC=$2324 REM=$3325 326 write_file $MW_SIZE "$LOC/mw_trans$IDX"327 328 INB_MW=$(read_file "$LOC/mw_trans$IDX")329 MW_ALIGNED_SIZE=$(echo "$INB_MW" | get_number "Window Size")330 MW_DMA_ADDR=$(echo "$INB_MW" | get_number "DMA Address")331 332 write_file "$MW_DMA_ADDR:$(($MW_ALIGNED_SIZE))" "$REM/peer_mw_trans$IDX"333 334 if [[ $MW_SIZE -ne $MW_ALIGNED_SIZE ]]; then335 echo "MW $IDX size aligned to $MW_ALIGNED_SIZE"336 fi337}338 339function write_mw()340{341 split_remote $2342 343 if [[ "$REMOTE" != "" ]]; then344 ssh "$REMOTE" \345 dd if=/dev/urandom "of=$VPATH" 2> /dev/null || true346 else347 dd if=/dev/urandom "of=$VPATH" 2> /dev/null || true348 fi349}350 351function mw_check()352{353 IDX=$1354 LOC=$2355 REM=$3356 357 write_mw "$LOC/mw$IDX"358 359 split_remote "$LOC/mw$IDX"360 if [[ "$REMOTE" == "" ]]; then361 A=$VPATH362 else363 A=/tmp/ntb_test.$$.A364 ssh "$REMOTE" cat "$VPATH" > "$A"365 fi366 367 split_remote "$REM/peer_mw$IDX"368 if [[ "$REMOTE" == "" ]]; then369 B=$VPATH370 else371 B=/tmp/ntb_test.$$.B372 ssh "$REMOTE" cat "$VPATH" > "$B"373 fi374 375 cmp -n $MW_ALIGNED_SIZE "$A" "$B"376 if [[ $? != 0 ]]; then377 echo "Memory window $MW did not match!" >&2378 fi379 380 if [[ "$A" == "/tmp/*" ]]; then381 rm "$A"382 fi383 384 if [[ "$B" == "/tmp/*" ]]; then385 rm "$B"386 fi387}388 389function mw_free()390{391 IDX=$1392 LOC=$2393 REM=$3394 395 write_file "$MW_DMA_ADDR:0" "$REM/peer_mw_trans$IDX"396 397 write_file 0 "$LOC/mw_trans$IDX"398}399 400function mw_test()401{402 LOC=$1403 REM=$2404 405 CNT=$(get_files_count "mw_trans" "$LOC")406 407 for ((i = 0; i < $CNT; i++)); do408 echo "Running mw$i tests on: $(subdirname $LOC) / " \409 "$(subdirname $REM)"410 411 mw_alloc $i $LOC $REM412 413 mw_check $i $LOC $REM414 415 mw_free $i $LOC $REM416 417 echo " Passed"418 done419 420}421 422function pingpong_test()423{424 LOC=$1425 REM=$2426 427 echo "Running ping pong tests on: $(basename $LOC) / $(basename $REM)"428 429 LOC_START=$(read_file "$LOC/count")430 REM_START=$(read_file "$REM/count")431 432 sleep 7433 434 LOC_END=$(read_file "$LOC/count")435 REM_END=$(read_file "$REM/count")436 437 if [[ $LOC_START == $LOC_END ]] || [[ $REM_START == $REM_END ]]; then438 echo "Ping pong counter not incrementing!" >&2439 exit 1440 fi441 442 echo " Passed"443}444 445function msi_test()446{447 LOC=$1448 REM=$2449 450 write_file 1 $LOC/ready451 452 echo "Running MSI interrupt tests on: $(subdirname $LOC) / $(subdirname $REM)"453 454 CNT=$(read_file "$LOC/count")455 for ((i = 0; i < $CNT; i++)); do456 START=$(read_file $REM/../irq${i}_occurrences)457 write_file $i $LOC/trigger458 END=$(read_file $REM/../irq${i}_occurrences)459 460 if [[ $(($END - $START)) != 1 ]]; then461 echo "MSI did not trigger the interrupt on the remote side!" >&2462 exit 1463 fi464 done465 466 echo " Passed"467}468 469function perf_test()470{471 USE_DMA=$1472 473 if [[ $USE_DMA == "1" ]]; then474 WITH="with"475 else476 WITH="without"477 fi478 479 _modprobe ntb_perf total_order=$PERF_RUN_ORDER \480 max_mw_size=$MAX_MW_SIZE use_dma=$USE_DMA481 482 echo "Running local perf test $WITH DMA"483 write_file "$LOCAL_PIDX" "$LOCAL_PERF/run"484 echo -n " "485 read_file "$LOCAL_PERF/run"486 echo " Passed"487 488 echo "Running remote perf test $WITH DMA"489 write_file "$REMOTE_PIDX" "$REMOTE_PERF/run"490 echo -n " "491 read_file "$REMOTE_PERF/run"492 echo " Passed"493 494 _modprobe -r ntb_perf495}496 497function ntb_tool_tests()498{499 LOCAL_TOOL="$DEBUGFS/ntb_tool/$LOCAL_DEV"500 REMOTE_TOOL="$REMOTE_HOST:$DEBUGFS/ntb_tool/$REMOTE_DEV"501 502 echo "Starting ntb_tool tests..."503 504 _modprobe ntb_tool505 506 port_test "$LOCAL_TOOL" "$REMOTE_TOOL"507 508 LOCAL_PEER_TOOL="$LOCAL_TOOL/peer$LOCAL_PIDX"509 REMOTE_PEER_TOOL="$REMOTE_TOOL/peer$REMOTE_PIDX"510 511 link_test "$LOCAL_PEER_TOOL" "$REMOTE_PEER_TOOL"512 link_test "$REMOTE_PEER_TOOL" "$LOCAL_PEER_TOOL"513 514 #Ensure the link is up on both sides before continuing515 write_file "Y" "$LOCAL_PEER_TOOL/link_event"516 write_file "Y" "$REMOTE_PEER_TOOL/link_event"517 518 doorbell_test "$LOCAL_TOOL" "$REMOTE_TOOL"519 doorbell_test "$REMOTE_TOOL" "$LOCAL_TOOL"520 521 scratchpad_test "$LOCAL_PEER_TOOL" "$REMOTE_PEER_TOOL"522 scratchpad_test "$REMOTE_PEER_TOOL" "$LOCAL_PEER_TOOL"523 524 message_test "$LOCAL_PEER_TOOL" "$REMOTE_PEER_TOOL"525 message_test "$REMOTE_PEER_TOOL" "$LOCAL_PEER_TOOL"526 527 mw_test "$LOCAL_PEER_TOOL" "$REMOTE_PEER_TOOL"528 mw_test "$REMOTE_PEER_TOOL" "$LOCAL_PEER_TOOL"529 530 _modprobe -r ntb_tool531}532 533function ntb_pingpong_tests()534{535 LOCAL_PP="$DEBUGFS/ntb_pingpong/$LOCAL_DEV"536 REMOTE_PP="$REMOTE_HOST:$DEBUGFS/ntb_pingpong/$REMOTE_DEV"537 538 echo "Starting ntb_pingpong tests..."539 540 _modprobe ntb_pingpong541 542 pingpong_test $LOCAL_PP $REMOTE_PP543 544 _modprobe -r ntb_pingpong545}546 547function ntb_msi_tests()548{549 LOCAL_MSI="$DEBUGFS/ntb_msi_test/$LOCAL_DEV"550 REMOTE_MSI="$REMOTE_HOST:$DEBUGFS/ntb_msi_test/$REMOTE_DEV"551 552 echo "Starting ntb_msi_test tests..."553 554 if ! _modprobe ntb_msi_test 2> /dev/null; then555 echo " Not doing MSI tests seeing the module is not available."556 return557 fi558 559 port_test $LOCAL_MSI $REMOTE_MSI560 561 LOCAL_PEER="$LOCAL_MSI/peer$LOCAL_PIDX"562 REMOTE_PEER="$REMOTE_MSI/peer$REMOTE_PIDX"563 564 msi_test $LOCAL_PEER $REMOTE_PEER565 msi_test $REMOTE_PEER $LOCAL_PEER566 567 _modprobe -r ntb_msi_test568}569 570function ntb_perf_tests()571{572 LOCAL_PERF="$DEBUGFS/ntb_perf/$LOCAL_DEV"573 REMOTE_PERF="$REMOTE_HOST:$DEBUGFS/ntb_perf/$REMOTE_DEV"574 575 echo "Starting ntb_perf tests..."576 577 perf_test 0578 579 if [[ $RUN_DMA_TESTS ]]; then580 perf_test 1581 fi582}583 584function cleanup()585{586 set +e587 _modprobe -r ntb_tool 2> /dev/null588 _modprobe -r ntb_perf 2> /dev/null589 _modprobe -r ntb_pingpong 2> /dev/null590 _modprobe -r ntb_transport 2> /dev/null591 _modprobe -r ntb_msi_test 2> /dev/null592 set -e593}594 595cleanup596 597if ! [[ $$DONT_CLEANUP ]]; then598 trap cleanup EXIT599fi600 601if [ "$(id -u)" != "0" ]; then602 echo "This script must be run as root" 1>&2603 exit 1604fi605 606if [[ "$LIST_DEVS" == TRUE ]]; then607 echo "Local Devices:"608 ls -1 /sys/bus/ntb/devices609 echo610 611 if [[ "$REMOTE_HOST" != "" ]]; then612 echo "Remote Devices:"613 ssh $REMOTE_HOST ls -1 /sys/bus/ntb/devices614 fi615 616 exit 0617fi618 619if [[ "$LOCAL_DEV" == $"" ]] || [[ "$REMOTE_DEV" == $"" ]]; then620 show_help621 exit 1622fi623 624ntb_tool_tests625echo626ntb_pingpong_tests627echo628ntb_msi_tests629echo630ntb_perf_tests631echo632