345 lines · bash
1#!/bin/bash2# SPDX-License-Identifier: GPL-2.03#4# A test for switch behavior under MC overload. An issue in Spectrum chips5# causes throughput of UC traffic to drop severely when a switch is under heavy6# MC load. This issue can be overcome by putting the switch to MC-aware mode.7# This test verifies that UC performance stays intact even as the switch is8# under MC flood, and therefore that the MC-aware mode is enabled and correctly9# configured.10#11# Because mlxsw throttles CPU port, the traffic can't actually reach userspace12# at full speed. That makes it impossible to use iperf3 to simply measure the13# throughput, because many packets (that reach $h3) don't get to the kernel at14# all even in UDP mode (the situation is even worse in TCP mode, where one can't15# hope to see more than a couple Mbps).16#17# So instead we send traffic with mausezahn and use RX ethtool counters at $h3.18# Multicast traffic is untagged, unicast traffic is tagged with PCP 1. Therefore19# each gets a different priority and we can use per-prio ethtool counters to20# measure the throughput. In order to avoid prioritizing unicast traffic, prio21# qdisc is installed on $swp3 and maps all priorities to the same band #7 (and22# thus TC 0).23#24# Mausezahn can't actually saturate the links unless it's using large frames.25# Thus we set MTU to 10K on all involved interfaces. Then both unicast and26# multicast traffic uses 8K frames.27#28# +---------------------------+ +----------------------------------+29# | H1 | | H2 |30# | | | unicast --> + $h2.111 |31# | multicast | | traffic | 192.0.2.129/28 |32# | traffic | | | e-qos-map 0:1 |33# | $h1 + <----- | | | |34# | 192.0.2.65/28 | | | + $h2 |35# +---------------|-----------+ +--------------|-------------------+36# | |37# +---------------|---------------------------------------|-------------------+38# | $swp1 + + $swp2 |39# | >1Gbps | | >1Gbps |40# | +-------------|------+ +----------|----------------+ |41# | | $swp1.1 + | | + $swp2.111 | |42# | | BR1 | SW | BR111 | |43# | | $swp3.1 + | | + $swp3.111 | |44# | +-------------|------+ +----------|----------------+ |45# | \_______________________________________/ |46# | | |47# | + $swp3 |48# | | 1Gbps bottleneck |49# | | prio qdisc: {0..7} -> 7 |50# +------------------------------------|--------------------------------------+51# |52# +--|-----------------+53# | + $h3 H3 |54# | | 192.0.2.66/28 |55# | | |56# | + $h3.111 |57# | 192.0.2.130/28 |58# +--------------------+59 60ALL_TESTS="61 ping_ipv462 test_mc_aware63 test_uc_aware64"65 66lib_dir=$(dirname $0)/../../../net/forwarding67 68NUM_NETIFS=669source $lib_dir/lib.sh70source $lib_dir/devlink_lib.sh71source qos_lib.sh72 73h1_create()74{75 simple_if_init $h1 192.0.2.65/2876 mtu_set $h1 1000077}78 79h1_destroy()80{81 mtu_restore $h182 simple_if_fini $h1 192.0.2.65/2883}84 85h2_create()86{87 simple_if_init $h288 mtu_set $h2 1000089 90 vlan_create $h2 111 v$h2 192.0.2.129/2891 ip link set dev $h2.111 type vlan egress-qos-map 0:192}93 94h2_destroy()95{96 vlan_destroy $h2 11197 98 mtu_restore $h299 simple_if_fini $h2100}101 102h3_create()103{104 simple_if_init $h3 192.0.2.66/28105 mtu_set $h3 10000106 107 vlan_create $h3 111 v$h3 192.0.2.130/28108}109 110h3_destroy()111{112 vlan_destroy $h3 111113 114 mtu_restore $h3115 simple_if_fini $h3 192.0.2.66/28116}117 118switch_create()119{120 ip link set dev $swp1 up121 mtu_set $swp1 10000122 123 ip link set dev $swp2 up124 mtu_set $swp2 10000125 126 ip link set dev $swp3 up127 mtu_set $swp3 10000128 129 vlan_create $swp2 111130 vlan_create $swp3 111131 132 tc qdisc replace dev $swp3 root handle 3: tbf rate 1gbit \133 burst 128K limit 1G134 tc qdisc replace dev $swp3 parent 3:3 handle 33: \135 prio bands 8 priomap 7 7 7 7 7 7 7 7136 137 ip link add name br1 type bridge vlan_filtering 0138 ip link set dev br1 addrgenmode none139 ip link set dev br1 up140 ip link set dev $swp1 master br1141 ip link set dev $swp3 master br1142 143 ip link add name br111 type bridge vlan_filtering 0144 ip link set dev br111 addrgenmode none145 ip link set dev br111 up146 ip link set dev $swp2.111 master br111147 ip link set dev $swp3.111 master br111148 149 # Make sure that ingress quotas are smaller than egress so that there is150 # room for both streams of traffic to be admitted to shared buffer.151 devlink_port_pool_th_save $swp1 0152 devlink_port_pool_th_set $swp1 0 5153 devlink_tc_bind_pool_th_save $swp1 0 ingress154 devlink_tc_bind_pool_th_set $swp1 0 ingress 0 5155 156 devlink_port_pool_th_save $swp2 0157 devlink_port_pool_th_set $swp2 0 5158 devlink_tc_bind_pool_th_save $swp2 1 ingress159 devlink_tc_bind_pool_th_set $swp2 1 ingress 0 5160 161 devlink_port_pool_th_save $swp3 4162 devlink_port_pool_th_set $swp3 4 12163}164 165switch_destroy()166{167 devlink_port_pool_th_restore $swp3 4168 169 devlink_tc_bind_pool_th_restore $swp2 1 ingress170 devlink_port_pool_th_restore $swp2 0171 172 devlink_tc_bind_pool_th_restore $swp1 0 ingress173 devlink_port_pool_th_restore $swp1 0174 175 ip link del dev br111176 ip link del dev br1177 178 tc qdisc del dev $swp3 parent 3:3 handle 33:179 tc qdisc del dev $swp3 root handle 3:180 181 vlan_destroy $swp3 111182 vlan_destroy $swp2 111183 184 mtu_restore $swp3185 ip link set dev $swp3 down186 187 mtu_restore $swp2188 ip link set dev $swp2 down189 190 mtu_restore $swp1191 ip link set dev $swp1 down192}193 194setup_prepare()195{196 h1=${NETIFS[p1]}197 swp1=${NETIFS[p2]}198 199 swp2=${NETIFS[p3]}200 h2=${NETIFS[p4]}201 202 swp3=${NETIFS[p5]}203 h3=${NETIFS[p6]}204 205 h3mac=$(mac_get $h3)206 207 vrf_prepare208 209 h1_create210 h2_create211 h3_create212 switch_create213}214 215cleanup()216{217 pre_cleanup218 219 switch_destroy220 h3_destroy221 h2_destroy222 h1_destroy223 224 vrf_cleanup225}226 227ping_ipv4()228{229 ping_test $h2 192.0.2.130230}231 232test_mc_aware()233{234 RET=0235 236 local -a uc_rate237 start_traffic $h2.111 192.0.2.129 192.0.2.130 $h3mac238 uc_rate=($(measure_rate $swp2 $h3 rx_octets_prio_1 "UC-only"))239 check_err $? "Could not get high enough UC-only ingress rate"240 stop_traffic241 local ucth1=${uc_rate[1]}242 243 start_traffic $h1 192.0.2.65 bc bc244 245 local d0=$(date +%s)246 local t0=$(ethtool_stats_get $h3 rx_octets_prio_0)247 local u0=$(ethtool_stats_get $swp1 rx_octets_prio_0)248 249 local -a uc_rate_2250 start_traffic $h2.111 192.0.2.129 192.0.2.130 $h3mac251 uc_rate_2=($(measure_rate $swp2 $h3 rx_octets_prio_1 "UC+MC"))252 check_err $? "Could not get high enough UC+MC ingress rate"253 stop_traffic254 local ucth2=${uc_rate_2[1]}255 256 local d1=$(date +%s)257 local t1=$(ethtool_stats_get $h3 rx_octets_prio_0)258 local u1=$(ethtool_stats_get $swp1 rx_octets_prio_0)259 260 local deg=$(bc <<< "261 scale=2262 ret = 100 * ($ucth1 - $ucth2) / $ucth1263 if (ret > 0) { ret } else { 0 }264 ")265 266 # Minimum shaper of 200Mbps on MC TCs should cause about 20% of267 # degradation on 1Gbps link.268 check_err $(bc <<< "$deg < 15") "Minimum shaper not in effect"269 check_err $(bc <<< "$deg > 25") "MC traffic degrades UC performance too much"270 271 local interval=$((d1 - d0))272 local mc_ir=$(rate $u0 $u1 $interval)273 local mc_er=$(rate $t0 $t1 $interval)274 275 stop_traffic276 277 log_test "UC performance under MC overload"278 279 echo "UC-only throughput $(humanize $ucth1)"280 echo "UC+MC throughput $(humanize $ucth2)"281 echo "Degradation $deg %"282 echo283 echo "Full report:"284 echo " UC only:"285 echo " ingress UC throughput $(humanize ${uc_rate[0]})"286 echo " egress UC throughput $(humanize ${uc_rate[1]})"287 echo " UC+MC:"288 echo " ingress UC throughput $(humanize ${uc_rate_2[0]})"289 echo " egress UC throughput $(humanize ${uc_rate_2[1]})"290 echo " ingress MC throughput $(humanize $mc_ir)"291 echo " egress MC throughput $(humanize $mc_er)"292 echo293}294 295test_uc_aware()296{297 RET=0298 299 start_traffic $h2.111 192.0.2.129 192.0.2.130 $h3mac300 301 local d0=$(date +%s)302 local t0=$(ethtool_stats_get $h3 rx_octets_prio_1)303 local u0=$(ethtool_stats_get $swp2 rx_octets_prio_1)304 sleep 1305 306 local attempts=50307 local passes=0308 local i309 310 for ((i = 0; i < attempts; ++i)); do311 if $ARPING -c 1 -I $h1 -b 192.0.2.66 -q -w 1; then312 ((passes++))313 fi314 315 sleep 0.1316 done317 318 local d1=$(date +%s)319 local t1=$(ethtool_stats_get $h3 rx_octets_prio_1)320 local u1=$(ethtool_stats_get $swp2 rx_octets_prio_1)321 322 local interval=$((d1 - d0))323 local uc_ir=$(rate $u0 $u1 $interval)324 local uc_er=$(rate $t0 $t1 $interval)325 326 ((attempts == passes))327 check_err $?328 329 stop_traffic330 331 log_test "MC performance under UC overload"332 echo " ingress UC throughput $(humanize ${uc_ir})"333 echo " egress UC throughput $(humanize ${uc_er})"334 echo " sent $attempts BC ARPs, got $passes responses"335}336 337trap cleanup EXIT338 339setup_prepare340setup_wait341 342tests_run343 344exit $EXIT_STATUS345