439 lines · bash
1#!/bin/bash2# SPDX-License-Identifier: GPL-2.03#4# Copyright (c) 2019 Facebook5#6# This program is free software; you can redistribute it and/or7# modify it under the terms of version 2 of the GNU General Public8# License as published by the Free Software Foundation.9 10Usage() {11 echo "Script for testing HBM (Host Bandwidth Manager) framework."12 echo "It creates a cgroup to use for testing and load a BPF program to limit"13 echo "egress or ingress bandwidth. It then uses iperf3 or netperf to create"14 echo "loads. The output is the goodput in Mbps (unless -D was used)."15 echo ""16 echo "USAGE: $name [out] [-b=<prog>|--bpf=<prog>] [-c=<cc>|--cc=<cc>]"17 echo " [-D] [-d=<delay>|--delay=<delay>] [--debug] [-E] [--edt]"18 echo " [-f=<#flows>|--flows=<#flows>] [-h] [-i=<id>|--id=<id >]"19 echo " [-l] [-N] [--no_cn] [-p=<port>|--port=<port>] [-P]"20 echo " [-q=<qdisc>] [-R] [-s=<server>|--server=<server]"21 echo " [-S|--stats] -t=<time>|--time=<time>] [-w] [cubic|dctcp]"22 echo " Where:"23 echo " out egress (default)"24 echo " -b or --bpf BPF program filename to load and attach."25 echo " Default is hbm_out_kern.o for egress,"26 echo " -c or -cc TCP congestion control (cubic or dctcp)"27 echo " --debug print BPF trace buffer"28 echo " -d or --delay add a delay in ms using netem"29 echo " -D In addition to the goodput in Mbps, it also outputs"30 echo " other detailed information. This information is"31 echo " test dependent (i.e. iperf3 or netperf)."32 echo " -E enable ECN (not required for dctcp)"33 echo " --edt use fq's Earliest Departure Time (requires fq)"34 echo " -f or --flows number of concurrent flows (default=1)"35 echo " -i or --id cgroup id (an integer, default is 1)"36 echo " -N use netperf instead of iperf3"37 echo " --no_cn Do not return CN notifications"38 echo " -l do not limit flows using loopback"39 echo " -h Help"40 echo " -p or --port iperf3 port (default is 5201)"41 echo " -P use an iperf3 instance for each flow"42 echo " -q use the specified qdisc"43 echo " -r or --rate rate in Mbps (default 1s 1Gbps)"44 echo " -R Use TCP_RR for netperf. 1st flow has req"45 echo " size of 10KB, rest of 1MB. Reply in all"46 echo " cases is 1 byte."47 echo " More detailed output for each flow can be found"48 echo " in the files netperf.<cg>.<flow>, where <cg> is the"49 echo " cgroup id as specified with the -i flag, and <flow>"50 echo " is the flow id starting at 1 and increasing by 1 for"51 echo " flow (as specified by -f)."52 echo " -s or --server hostname of netperf server. Used to create netperf"53 echo " test traffic between to hosts (default is within host)"54 echo " netserver must be running on the host."55 echo " -S or --stats whether to update hbm stats (default is yes)."56 echo " -t or --time duration of iperf3 in seconds (default=5)"57 echo " -w Work conserving flag. cgroup can increase its"58 echo " bandwidth beyond the rate limit specified"59 echo " while there is available bandwidth. Current"60 echo " implementation assumes there is only one NIC"61 echo " (eth0), but can be extended to support multiple"62 echo " NICs."63 echo " cubic or dctcp specify which TCP CC to use"64 echo " "65 exit66}67 68#set -x69 70debug_flag=071args="$@"72name="$0"73netem=074cc=x75dir="-o"76dir_name="out"77dur=578flows=179id=180prog=""81port=520182rate=100083multi_iperf=084flow_cnt=185use_netperf=086rr=087ecn=088details=089server=""90qdisc=""91flags=""92do_stats=093 94BPFFS=/sys/fs/bpf95function config_bpffs () {96 if mount | grep $BPFFS > /dev/null; then97 echo "bpffs already mounted"98 else99 echo "bpffs not mounted. Mounting..."100 mount -t bpf none $BPFFS101 fi102}103 104function start_hbm () {105 rm -f hbm.out106 echo "./hbm $dir -n $id -r $rate -t $dur $flags $dbg $prog" > hbm.out107 echo " " >> hbm.out108 ./hbm $dir -n $id -r $rate -t $dur $flags $dbg $prog >> hbm.out 2>&1 &109 echo $!110}111 112processArgs () {113 for i in $args ; do114 case $i in115 # Support for upcomming ingress rate limiting116 #in) # support for upcoming ingress rate limiting117 # dir="-i"118 # dir_name="in"119 # ;;120 out)121 dir="-o"122 dir_name="out"123 ;;124 -b=*|--bpf=*)125 prog="${i#*=}"126 ;;127 -c=*|--cc=*)128 cc="${i#*=}"129 ;;130 --no_cn)131 flags="$flags --no_cn"132 ;;133 --debug)134 flags="$flags -d"135 debug_flag=1136 ;;137 -d=*|--delay=*)138 netem="${i#*=}"139 ;;140 -D)141 details=1142 ;;143 -E)144 ecn=1145 ;;146 --edt)147 flags="$flags --edt"148 qdisc="fq"149 ;;150 -f=*|--flows=*)151 flows="${i#*=}"152 ;;153 -i=*|--id=*)154 id="${i#*=}"155 ;;156 -l)157 flags="$flags -l"158 ;;159 -N)160 use_netperf=1161 ;;162 -p=*|--port=*)163 port="${i#*=}"164 ;;165 -P)166 multi_iperf=1167 ;;168 -q=*)169 qdisc="${i#*=}"170 ;;171 -r=*|--rate=*)172 rate="${i#*=}"173 ;;174 -R)175 rr=1176 ;;177 -s=*|--server=*)178 server="${i#*=}"179 ;;180 -S|--stats)181 flags="$flags -s"182 do_stats=1183 ;;184 -t=*|--time=*)185 dur="${i#*=}"186 ;;187 -w)188 flags="$flags -w"189 ;;190 cubic)191 cc=cubic192 ;;193 dctcp)194 cc=dctcp195 ;;196 *)197 echo "Unknown arg:$i"198 Usage199 ;;200 esac201 done202}203 204processArgs205config_bpffs206 207if [ $debug_flag -eq 1 ] ; then208 rm -f hbm_out.log209fi210 211hbm_pid=$(start_hbm)212usleep 100000213 214host=`hostname`215cg_base_dir=/sys/fs/cgroup/unified216cg_dir="$cg_base_dir/cgroup-test-work-dir/hbm$id"217 218echo $$ >> $cg_dir/cgroup.procs219 220ulimit -l unlimited221 222rm -f ss.out223rm -f hbm.[0-9]*.$dir_name224if [ $ecn -ne 0 ] ; then225 sysctl -w -q -n net.ipv4.tcp_ecn=1226fi227 228if [ $use_netperf -eq 0 ] ; then229 cur_cc=`sysctl -n net.ipv4.tcp_congestion_control`230 if [ "$cc" != "x" ] ; then231 sysctl -w -q -n net.ipv4.tcp_congestion_control=$cc232 fi233fi234 235if [ "$netem" -ne "0" ] ; then236 if [ "$qdisc" != "" ] ; then237 echo "WARNING: Ignoring -q options because -d option used"238 fi239 tc qdisc del dev lo root > /dev/null 2>&1240 tc qdisc add dev lo root netem delay $netem\ms > /dev/null 2>&1241elif [ "$qdisc" != "" ] ; then242 tc qdisc del dev eth0 root > /dev/null 2>&1243 tc qdisc add dev eth0 root $qdisc > /dev/null 2>&1244fi245 246n=0247m=$[$dur * 5]248hn="::1"249if [ $use_netperf -ne 0 ] ; then250 if [ "$server" != "" ] ; then251 hn=$server252 fi253fi254 255( ping6 -i 0.2 -c $m $hn > ping.out 2>&1 ) &256 257if [ $use_netperf -ne 0 ] ; then258 begNetserverPid=`ps ax | grep netserver | grep --invert-match "grep" | \259 awk '{ print $1 }'`260 if [ "$begNetserverPid" == "" ] ; then261 if [ "$server" == "" ] ; then262 ( ./netserver > /dev/null 2>&1) &263 usleep 100000264 fi265 fi266 flow_cnt=1267 if [ "$server" == "" ] ; then268 np_server=$host269 else270 np_server=$server271 fi272 if [ "$cc" == "x" ] ; then273 np_cc=""274 else275 np_cc="-K $cc,$cc"276 fi277 replySize=1278 while [ $flow_cnt -le $flows ] ; do279 if [ $rr -ne 0 ] ; then280 reqSize=1M281 if [ $flow_cnt -eq 1 ] ; then282 reqSize=10K283 fi284 if [ "$dir" == "-i" ] ; then285 replySize=$reqSize286 reqSize=1287 fi288 ( ./netperf -H $np_server -l $dur -f m -j -t TCP_RR -- -r $reqSize,$replySize $np_cc -k P50_lATENCY,P90_LATENCY,LOCAL_TRANSPORT_RETRANS,REMOTE_TRANSPORT_RETRANS,LOCAL_SEND_THROUGHPUT,LOCAL_RECV_THROUGHPUT,REQUEST_SIZE,RESPONSE_SIZE > netperf.$id.$flow_cnt ) &289 else290 if [ "$dir" == "-i" ] ; then291 ( ./netperf -H $np_server -l $dur -f m -j -t TCP_RR -- -r 1,10M $np_cc -k P50_LATENCY,P90_LATENCY,LOCAL_TRANSPORT_RETRANS,LOCAL_SEND_THROUGHPUT,REMOTE_TRANSPORT_RETRANS,REMOTE_SEND_THROUGHPUT,REQUEST_SIZE,RESPONSE_SIZE > netperf.$id.$flow_cnt ) &292 else293 ( ./netperf -H $np_server -l $dur -f m -j -t TCP_STREAM -- $np_cc -k P50_lATENCY,P90_LATENCY,LOCAL_TRANSPORT_RETRANS,LOCAL_SEND_THROUGHPUT,REQUEST_SIZE,RESPONSE_SIZE > netperf.$id.$flow_cnt ) &294 fi295 fi296 flow_cnt=$[flow_cnt+1]297 done298 299# sleep for duration of test (plus some buffer)300 n=$[dur+2]301 sleep $n302 303# force graceful termination of netperf304 pids=`pgrep netperf`305 for p in $pids ; do306 kill -SIGALRM $p307 done308 309 flow_cnt=1310 rate=0311 if [ $details -ne 0 ] ; then312 echo ""313 echo "Details for HBM in cgroup $id"314 if [ $do_stats -eq 1 ] ; then315 if [ -e hbm.$id.$dir_name ] ; then316 cat hbm.$id.$dir_name317 fi318 fi319 fi320 while [ $flow_cnt -le $flows ] ; do321 if [ "$dir" == "-i" ] ; then322 r=`cat netperf.$id.$flow_cnt | grep -o "REMOTE_SEND_THROUGHPUT=[0-9]*" | grep -o "[0-9]*"`323 else324 r=`cat netperf.$id.$flow_cnt | grep -o "LOCAL_SEND_THROUGHPUT=[0-9]*" | grep -o "[0-9]*"`325 fi326 echo "rate for flow $flow_cnt: $r"327 rate=$[rate+r]328 if [ $details -ne 0 ] ; then329 echo "-----"330 echo "Details for cgroup $id, flow $flow_cnt"331 cat netperf.$id.$flow_cnt332 fi333 flow_cnt=$[flow_cnt+1]334 done335 if [ $details -ne 0 ] ; then336 echo ""337 delay=`grep "avg" ping.out | grep -o "= [0-9.]*/[0-9.]*" | grep -o "[0-9.]*$"`338 echo "PING AVG DELAY:$delay"339 echo "AGGREGATE_GOODPUT:$rate"340 else341 echo $rate342 fi343elif [ $multi_iperf -eq 0 ] ; then344 (iperf3 -s -p $port -1 > /dev/null 2>&1) &345 usleep 100000346 iperf3 -c $host -p $port -i 0 -P $flows -f m -t $dur > iperf.$id347 rates=`grep receiver iperf.$id | grep -o "[0-9.]* Mbits" | grep -o "^[0-9]*"`348 rate=`echo $rates | grep -o "[0-9]*$"`349 350 if [ $details -ne 0 ] ; then351 echo ""352 echo "Details for HBM in cgroup $id"353 if [ $do_stats -eq 1 ] ; then354 if [ -e hbm.$id.$dir_name ] ; then355 cat hbm.$id.$dir_name356 fi357 fi358 delay=`grep "avg" ping.out | grep -o "= [0-9.]*/[0-9.]*" | grep -o "[0-9.]*$"`359 echo "PING AVG DELAY:$delay"360 echo "AGGREGATE_GOODPUT:$rate"361 else362 echo $rate363 fi364else365 flow_cnt=1366 while [ $flow_cnt -le $flows ] ; do367 (iperf3 -s -p $port -1 > /dev/null 2>&1) &368 ( iperf3 -c $host -p $port -i 0 -P 1 -f m -t $dur | grep receiver | grep -o "[0-9.]* Mbits" | grep -o "^[0-9]*" | grep -o "[0-9]*$" > iperf3.$id.$flow_cnt ) &369 port=$[port+1]370 flow_cnt=$[flow_cnt+1]371 done372 n=$[dur+1]373 sleep $n374 flow_cnt=1375 rate=0376 if [ $details -ne 0 ] ; then377 echo ""378 echo "Details for HBM in cgroup $id"379 if [ $do_stats -eq 1 ] ; then380 if [ -e hbm.$id.$dir_name ] ; then381 cat hbm.$id.$dir_name382 fi383 fi384 fi385 386 while [ $flow_cnt -le $flows ] ; do387 r=`cat iperf3.$id.$flow_cnt`388# echo "rate for flow $flow_cnt: $r"389 if [ $details -ne 0 ] ; then390 echo "Rate for cgroup $id, flow $flow_cnt LOCAL_SEND_THROUGHPUT=$r"391 fi392 rate=$[rate+r]393 flow_cnt=$[flow_cnt+1]394 done395 if [ $details -ne 0 ] ; then396 delay=`grep "avg" ping.out | grep -o "= [0-9.]*/[0-9.]*" | grep -o "[0-9.]*$"`397 echo "PING AVG DELAY:$delay"398 echo "AGGREGATE_GOODPUT:$rate"399 else400 echo $rate401 fi402fi403 404if [ $use_netperf -eq 0 ] ; then405 sysctl -w -q -n net.ipv4.tcp_congestion_control=$cur_cc406fi407if [ $ecn -ne 0 ] ; then408 sysctl -w -q -n net.ipv4.tcp_ecn=0409fi410if [ "$netem" -ne "0" ] ; then411 tc qdisc del dev lo root > /dev/null 2>&1412fi413if [ "$qdisc" != "" ] ; then414 tc qdisc del dev eth0 root > /dev/null 2>&1415fi416sleep 2417 418hbmPid=`ps ax | grep "hbm " | grep --invert-match "grep" | awk '{ print $1 }'`419if [ "$hbmPid" == "$hbm_pid" ] ; then420 kill $hbm_pid421fi422 423sleep 1424 425# Detach any pinned BPF programs that may have lingered426rm -rf $BPFFS/hbm*427 428if [ $use_netperf -ne 0 ] ; then429 if [ "$server" == "" ] ; then430 if [ "$begNetserverPid" == "" ] ; then431 netserverPid=`ps ax | grep netserver | grep --invert-match "grep" | awk '{ print $1 }'`432 if [ "$netserverPid" != "" ] ; then433 kill $netserverPid434 fi435 fi436 fi437fi438exit439