1002 lines · bash
1#!/bin/bash2# SPDX-License-Identifier: GPL-2.03#4# Test for cpuset v2 partition root state (PRS)5#6# The sched verbose flag can be optionally set so that the console log7# can be examined for the correct setting of scheduling domain.8#9 10skip_test() {11 echo "$1"12 echo "Test SKIPPED"13 exit 4 # ksft_skip14}15 16[[ $(id -u) -eq 0 ]] || skip_test "Test must be run as root!"17 18 19# Get wait_inotify location20WAIT_INOTIFY=$(cd $(dirname $0); pwd)/wait_inotify21 22# Find cgroup v2 mount point23CGROUP2=$(mount -t cgroup2 | head -1 | awk -e '{print $3}')24[[ -n "$CGROUP2" ]] || skip_test "Cgroup v2 mount point not found!"25SUBPARTS_CPUS=$CGROUP2/.__DEBUG__.cpuset.cpus.subpartitions26CPULIST=$(cat $CGROUP2/cpuset.cpus.effective)27 28NR_CPUS=$(lscpu | grep "^CPU(s):" | sed -e "s/.*:[[:space:]]*//")29[[ $NR_CPUS -lt 8 ]] && skip_test "Test needs at least 8 cpus available!"30 31# Check to see if /dev/console exists and is writable32if [[ -c /dev/console && -w /dev/console ]]33then34 CONSOLE=/dev/console35else36 CONSOLE=/dev/null37fi38 39# Set verbose flag and delay factor40PROG=$141VERBOSE=042DELAY_FACTOR=143SCHED_DEBUG=44while [[ "$1" = -* ]]45do46 case "$1" in47 -v) ((VERBOSE++))48 # Enable sched/verbose can slow thing down49 [[ $DELAY_FACTOR -eq 1 ]] &&50 DELAY_FACTOR=251 ;;52 -d) DELAY_FACTOR=$253 shift54 ;;55 *) echo "Usage: $PROG [-v] [-d <delay-factor>"56 exit57 ;;58 esac59 shift60done61 62# Set sched verbose flag if available when "-v" option is specified63if [[ $VERBOSE -gt 0 && -d /sys/kernel/debug/sched ]]64then65 # Used to restore the original setting during cleanup66 SCHED_DEBUG=$(cat /sys/kernel/debug/sched/verbose)67 echo Y > /sys/kernel/debug/sched/verbose68fi69 70cd $CGROUP271echo +cpuset > cgroup.subtree_control72 73#74# If cpuset has been set up and used in child cgroups, we may not be able to75# create partition under root cgroup because of the CPU exclusivity rule.76# So we are going to skip the test if this is the case.77#78[[ -d test ]] || mkdir test79echo 0-6 > test/cpuset.cpus80echo root > test/cpuset.cpus.partition81cat test/cpuset.cpus.partition | grep -q invalid82RESULT=$?83echo member > test/cpuset.cpus.partition84echo "" > test/cpuset.cpus85[[ $RESULT -eq 0 ]] && skip_test "Child cgroups are using cpuset!"86 87#88# If isolated CPUs have been reserved at boot time (as shown in89# cpuset.cpus.isolated), these isolated CPUs should be outside of CPUs 0-790# that will be used by this script for testing purpose. If not, some of91# the tests may fail incorrectly. These isolated CPUs will also be removed92# before being compared with the expected results.93#94BOOT_ISOLCPUS=$(cat $CGROUP2/cpuset.cpus.isolated)95if [[ -n "$BOOT_ISOLCPUS" ]]96then97 [[ $(echo $BOOT_ISOLCPUS | sed -e "s/[,-].*//") -le 7 ]] &&98 skip_test "Pre-isolated CPUs ($BOOT_ISOLCPUS) overlap CPUs to be tested"99 echo "Pre-isolated CPUs: $BOOT_ISOLCPUS"100fi101cleanup()102{103 online_cpus104 cd $CGROUP2105 rmdir A1/A2/A3 A1/A2 A1 B1 > /dev/null 2>&1106 rmdir test > /dev/null 2>&1107 [[ -n "$SCHED_DEBUG" ]] &&108 echo "$SCHED_DEBUG" > /sys/kernel/debug/sched/verbose109}110 111# Pause in ms112pause()113{114 DELAY=$1115 LOOP=0116 while [[ $LOOP -lt $DELAY_FACTOR ]]117 do118 sleep $DELAY119 ((LOOP++))120 done121 return 0122}123 124console_msg()125{126 MSG=$1127 echo "$MSG"128 echo "" > $CONSOLE129 echo "$MSG" > $CONSOLE130 pause 0.01131}132 133test_partition()134{135 EXPECTED_VAL=$1136 echo $EXPECTED_VAL > cpuset.cpus.partition137 [[ $? -eq 0 ]] || exit 1138 ACTUAL_VAL=$(cat cpuset.cpus.partition)139 [[ $ACTUAL_VAL != $EXPECTED_VAL ]] && {140 echo "cpuset.cpus.partition: expect $EXPECTED_VAL, found $ACTUAL_VAL"141 echo "Test FAILED"142 exit 1143 }144}145 146test_effective_cpus()147{148 EXPECTED_VAL=$1149 ACTUAL_VAL=$(cat cpuset.cpus.effective)150 [[ "$ACTUAL_VAL" != "$EXPECTED_VAL" ]] && {151 echo "cpuset.cpus.effective: expect '$EXPECTED_VAL', found '$ACTUAL_VAL'"152 echo "Test FAILED"153 exit 1154 }155}156 157# Adding current process to cgroup.procs as a test158test_add_proc()159{160 OUTSTR="$1"161 ERRMSG=$((echo $$ > cgroup.procs) |& cat)162 echo $ERRMSG | grep -q "$OUTSTR"163 [[ $? -ne 0 ]] && {164 echo "cgroup.procs: expect '$OUTSTR', got '$ERRMSG'"165 echo "Test FAILED"166 exit 1167 }168 echo $$ > $CGROUP2/cgroup.procs # Move out the task169}170 171#172# Cpuset controller state transition test matrix.173#174# Cgroup test hierarchy175#176# root -- A1 -- A2 -- A3177# +- B1178#179# P<v> = set cpus.partition (0:member, 1:root, 2:isolated)180# C<l> = add cpu-list to cpuset.cpus181# X<l> = add cpu-list to cpuset.cpus.exclusive182# S<p> = use prefix in subtree_control183# T = put a task into cgroup184# O<c>=<v> = Write <v> to CPU online file of <c>185#186# ECPUs - effective CPUs of cpusets187# Pstate - partition root state188# ISOLCPUS - isolated CPUs (<icpus>[,<icpus2>])189#190# Note that if there are 2 fields in ISOLCPUS, the first one is for191# sched-debug matching which includes offline CPUs and single-CPU partitions192# while the second one is for matching cpuset.cpus.isolated.193#194SETUP_A123_PARTITIONS="C1-3:P1:S+ C2-3:P1:S+ C3:P1"195TEST_MATRIX=(196 # old-A1 old-A2 old-A3 old-B1 new-A1 new-A2 new-A3 new-B1 fail ECPUs Pstate ISOLCPUS197 # ------ ------ ------ ------ ------ ------ ------ ------ ---- ----- ------ --------198 " C0-1 . . C2-3 S+ C4-5 . . 0 A2:0-1"199 " C0-1 . . C2-3 P1 . . . 0 "200 " C0-1 . . C2-3 P1:S+ C0-1:P1 . . 0 "201 " C0-1 . . C2-3 P1:S+ C1:P1 . . 0 "202 " C0-1:S+ . . C2-3 . . . P1 0 "203 " C0-1:P1 . . C2-3 S+ C1 . . 0 "204 " C0-1:P1 . . C2-3 S+ C1:P1 . . 0 "205 " C0-1:P1 . . C2-3 S+ C1:P1 . P1 0 "206 " C0-1:P1 . . C2-3 C4-5 . . . 0 A1:4-5"207 " C0-1:P1 . . C2-3 S+:C4-5 . . . 0 A1:4-5"208 " C0-1 . . C2-3:P1 . . . C2 0 "209 " C0-1 . . C2-3:P1 . . . C4-5 0 B1:4-5"210 "C0-3:P1:S+ C2-3:P1 . . . . . . 0 A1:0-1,A2:2-3"211 "C0-3:P1:S+ C2-3:P1 . . C1-3 . . . 0 A1:1,A2:2-3"212 "C2-3:P1:S+ C3:P1 . . C3 . . . 0 A1:,A2:3 A1:P1,A2:P1"213 "C2-3:P1:S+ C3:P1 . . C3 P0 . . 0 A1:3,A2:3 A1:P1,A2:P0"214 "C2-3:P1:S+ C2:P1 . . C2-4 . . . 0 A1:3-4,A2:2"215 "C2-3:P1:S+ C3:P1 . . C3 . . C0-2 0 A1:,B1:0-2 A1:P1,A2:P1"216 "$SETUP_A123_PARTITIONS . C2-3 . . . 0 A1:,A2:2,A3:3 A1:P1,A2:P1,A3:P1"217 218 # CPU offlining cases:219 " C0-1 . . C2-3 S+ C4-5 . O2=0 0 A1:0-1,B1:3"220 "C0-3:P1:S+ C2-3:P1 . . O2=0 . . . 0 A1:0-1,A2:3"221 "C0-3:P1:S+ C2-3:P1 . . O2=0 O2=1 . . 0 A1:0-1,A2:2-3"222 "C0-3:P1:S+ C2-3:P1 . . O1=0 . . . 0 A1:0,A2:2-3"223 "C0-3:P1:S+ C2-3:P1 . . O1=0 O1=1 . . 0 A1:0-1,A2:2-3"224 "C2-3:P1:S+ C3:P1 . . O3=0 O3=1 . . 0 A1:2,A2:3 A1:P1,A2:P1"225 "C2-3:P1:S+ C3:P2 . . O3=0 O3=1 . . 0 A1:2,A2:3 A1:P1,A2:P2"226 "C2-3:P1:S+ C3:P1 . . O2=0 O2=1 . . 0 A1:2,A2:3 A1:P1,A2:P1"227 "C2-3:P1:S+ C3:P2 . . O2=0 O2=1 . . 0 A1:2,A2:3 A1:P1,A2:P2"228 "C2-3:P1:S+ C3:P1 . . O2=0 . . . 0 A1:,A2:3 A1:P1,A2:P1"229 "C2-3:P1:S+ C3:P1 . . O3=0 . . . 0 A1:2,A2: A1:P1,A2:P1"230 "C2-3:P1:S+ C3:P1 . . T:O2=0 . . . 0 A1:3,A2:3 A1:P1,A2:P-1"231 "C2-3:P1:S+ C3:P1 . . . T:O3=0 . . 0 A1:2,A2:2 A1:P1,A2:P-1"232 "$SETUP_A123_PARTITIONS . O1=0 . . . 0 A1:,A2:2,A3:3 A1:P1,A2:P1,A3:P1"233 "$SETUP_A123_PARTITIONS . O2=0 . . . 0 A1:1,A2:,A3:3 A1:P1,A2:P1,A3:P1"234 "$SETUP_A123_PARTITIONS . O3=0 . . . 0 A1:1,A2:2,A3: A1:P1,A2:P1,A3:P1"235 "$SETUP_A123_PARTITIONS . T:O1=0 . . . 0 A1:2-3,A2:2-3,A3:3 A1:P1,A2:P-1,A3:P-1"236 "$SETUP_A123_PARTITIONS . . T:O2=0 . . 0 A1:1,A2:3,A3:3 A1:P1,A2:P1,A3:P-1"237 "$SETUP_A123_PARTITIONS . . . T:O3=0 . 0 A1:1,A2:2,A3:2 A1:P1,A2:P1,A3:P-1"238 "$SETUP_A123_PARTITIONS . T:O1=0 O1=1 . . 0 A1:1,A2:2,A3:3 A1:P1,A2:P1,A3:P1"239 "$SETUP_A123_PARTITIONS . . T:O2=0 O2=1 . 0 A1:1,A2:2,A3:3 A1:P1,A2:P1,A3:P1"240 "$SETUP_A123_PARTITIONS . . . T:O3=0 O3=1 0 A1:1,A2:2,A3:3 A1:P1,A2:P1,A3:P1"241 "$SETUP_A123_PARTITIONS . T:O1=0 O2=0 O1=1 . 0 A1:1,A2:,A3:3 A1:P1,A2:P1,A3:P1"242 "$SETUP_A123_PARTITIONS . T:O1=0 O2=0 O2=1 . 0 A1:2-3,A2:2-3,A3:3 A1:P1,A2:P-1,A3:P-1"243 244 # old-A1 old-A2 old-A3 old-B1 new-A1 new-A2 new-A3 new-B1 fail ECPUs Pstate ISOLCPUS245 # ------ ------ ------ ------ ------ ------ ------ ------ ---- ----- ------ --------246 #247 # Remote partition and cpuset.cpus.exclusive tests248 #249 " C0-3:S+ C1-3:S+ C2-3 . X2-3 . . . 0 A1:0-3,A2:1-3,A3:2-3,XA1:2-3"250 " C0-3:S+ C1-3:S+ C2-3 . X2-3 X2-3:P2 . . 0 A1:0-1,A2:2-3,A3:2-3 A1:P0,A2:P2 2-3"251 " C0-3:S+ C1-3:S+ C2-3 . X2-3 X3:P2 . . 0 A1:0-2,A2:3,A3:3 A1:P0,A2:P2 3"252 " C0-3:S+ C1-3:S+ C2-3 . X2-3 X2-3 X2-3:P2 . 0 A1:0-1,A2:1,A3:2-3 A1:P0,A3:P2 2-3"253 " C0-3:S+ C1-3:S+ C2-3 . X2-3 X2-3 X2-3:P2:C3 . 0 A1:0-1,A2:1,A3:2-3 A1:P0,A3:P2 2-3"254 " C0-3:S+ C1-3:S+ C2-3 C2-3 . . . P2 0 A1:0-3,A2:1-3,A3:2-3,B1:2-3 A1:P0,A3:P0,B1:P-2"255 " C0-3:S+ C1-3:S+ C2-3 C4-5 . . . P2 0 B1:4-5 B1:P2 4-5"256 " C0-3:S+ C1-3:S+ C2-3 C4 X2-3 X2-3 X2-3:P2 P2 0 A3:2-3,B1:4 A3:P2,B1:P2 2-4"257 " C0-3:S+ C1-3:S+ C2-3 C4 X2-3 X2-3 X2-3:P2:C1-3 P2 0 A3:2-3,B1:4 A3:P2,B1:P2 2-4"258 " C0-3:S+ C1-3:S+ C2-3 C4 X1-3 X1-3:P2 P2 . 0 A2:1,A3:2-3 A2:P2,A3:P2 1-3"259 " C0-3:S+ C1-3:S+ C2-3 C4 X2-3 X2-3 X2-3:P2 P2:C4-5 0 A3:2-3,B1:4-5 A3:P2,B1:P2 2-5"260 " C4:X0-3:S+ X1-3:S+ X2-3 . . P2 . . 0 A1:4,A2:1-3,A3:1-3 A2:P2 1-3"261 " C4:X0-3:S+ X1-3:S+ X2-3 . . . P2 . 0 A1:4,A2:4,A3:2-3 A3:P2 2-3"262 263 # Nested remote/local partition tests264 " C0-3:S+ C1-3:S+ C2-3 C4-5 X2-3 X2-3:P1 P2 P1 0 A1:0-1,A2:,A3:2-3,B1:4-5 \265 A1:P0,A2:P1,A3:P2,B1:P1 2-3"266 " C0-3:S+ C1-3:S+ C2-3 C4 X2-3 X2-3:P1 P2 P1 0 A1:0-1,A2:,A3:2-3,B1:4 \267 A1:P0,A2:P1,A3:P2,B1:P1 2-4,2-3"268 " C0-3:S+ C1-3:S+ C2-3 C4 X2-3 X2-3:P1 . P1 0 A1:0-1,A2:2-3,A3:2-3,B1:4 \269 A1:P0,A2:P1,A3:P0,B1:P1"270 " C0-3:S+ C1-3:S+ C3 C4 X2-3 X2-3:P1 P2 P1 0 A1:0-1,A2:2,A3:3,B1:4 \271 A1:P0,A2:P1,A3:P2,B1:P1 2-4,3"272 " C0-4:S+ C1-4:S+ C2-4 . X2-4 X2-4:P2 X4:P1 . 0 A1:0-1,A2:2-3,A3:4 \273 A1:P0,A2:P2,A3:P1 2-4,2-3"274 " C0-4:S+ C1-4:S+ C2-4 . X2-4 X2-4:P2 X3-4:P1 . 0 A1:0-1,A2:2,A3:3-4 \275 A1:P0,A2:P2,A3:P1 2"276 " C0-4:X2-4:S+ C1-4:X2-4:S+:P2 C2-4:X4:P1 \277 . . X5 . . 0 A1:0-4,A2:1-4,A3:2-4 \278 A1:P0,A2:P-2,A3:P-1"279 " C0-4:X2-4:S+ C1-4:X2-4:S+:P2 C2-4:X4:P1 \280 . . . X1 . 0 A1:0-1,A2:2-4,A3:2-4 \281 A1:P0,A2:P2,A3:P-1 2-4"282 283 # Remote partition offline tests284 " C0-3:S+ C1-3:S+ C2-3 . X2-3 X2-3 X2-3:P2:O2=0 . 0 A1:0-1,A2:1,A3:3 A1:P0,A3:P2 2-3"285 " C0-3:S+ C1-3:S+ C2-3 . X2-3 X2-3 X2-3:P2:O2=0 O2=1 0 A1:0-1,A2:1,A3:2-3 A1:P0,A3:P2 2-3"286 " C0-3:S+ C1-3:S+ C3 . X2-3 X2-3 P2:O3=0 . 0 A1:0-2,A2:1-2,A3: A1:P0,A3:P2 3"287 " C0-3:S+ C1-3:S+ C3 . X2-3 X2-3 T:P2:O3=0 . 0 A1:0-2,A2:1-2,A3:1-2 A1:P0,A3:P-2 3,"288 289 # An invalidated remote partition cannot self-recover from hotplug290 " C0-3:S+ C1-3:S+ C2 . X2-3 X2-3 T:P2:O2=0 O2=1 0 A1:0-3,A2:1-3,A3:2 A1:P0,A3:P-2"291 292 # cpus.exclusive.effective clearing test293 " C0-3:S+ C1-3:S+ C2 . X2-3:X . . . 0 A1:0-3,A2:1-3,A3:2,XA1:"294 295 # Invalid to valid remote partition transition test296 " C0-3:S+ C1-3 . . . X3:P2 . . 0 A1:0-3,A2:1-3,XA2: A2:P-2"297 " C0-3:S+ C1-3:X3:P2298 . . X2-3 P2 . . 0 A1:0-2,A2:3,XA2:3 A2:P2 3"299 300 # Invalid to valid local partition direct transition tests301 " C1-3:S+:P2 X4:P2 . . . . . . 0 A1:1-3,XA1:1-3,A2:1-3:XA2: A1:P2,A2:P-2 1-3"302 " C1-3:S+:P2 X4:P2 . . . X3:P2 . . 0 A1:1-2,XA1:1-3,A2:3:XA2:3 A1:P2,A2:P2 1-3"303 " C0-3:P2 . . C4-6 C0-4 . . . 0 A1:0-4,B1:4-6 A1:P-2,B1:P0"304 " C0-3:P2 . . C4-6 C0-4:C0-3 . . . 0 A1:0-3,B1:4-6 A1:P2,B1:P0 0-3"305 " C0-3:P2 . . C3-5:C4-5 . . . . 0 A1:0-3,B1:4-5 A1:P2,B1:P0 0-3"306 307 # Local partition invalidation tests308 " C0-3:X1-3:S+:P2 C1-3:X2-3:S+:P2 C2-3:X3:P2 \309 . . . . . 0 A1:1,A2:2,A3:3 A1:P2,A2:P2,A3:P2 1-3"310 " C0-3:X1-3:S+:P2 C1-3:X2-3:S+:P2 C2-3:X3:P2 \311 . . X4 . . 0 A1:1-3,A2:1-3,A3:2-3,XA2:,XA3: A1:P2,A2:P-2,A3:P-2 1-3"312 " C0-3:X1-3:S+:P2 C1-3:X2-3:S+:P2 C2-3:X3:P2 \313 . . C4:X . . 0 A1:1-3,A2:1-3,A3:2-3,XA2:,XA3: A1:P2,A2:P-2,A3:P-2 1-3"314 # Local partition CPU change tests315 " C0-5:S+:P2 C4-5:S+:P1 . . . C3-5 . . 0 A1:0-2,A2:3-5 A1:P2,A2:P1 0-2"316 " C0-5:S+:P2 C4-5:S+:P1 . . C1-5 . . . 0 A1:1-3,A2:4-5 A1:P2,A2:P1 1-3"317 318 # cpus_allowed/exclusive_cpus update tests319 " C0-3:X2-3:S+ C1-3:X2-3:S+ C2-3:X2-3 \320 . X:C4 . P2 . 0 A1:4,A2:4,XA2:,XA3:,A3:4 \321 A1:P0,A3:P-2"322 " C0-3:X2-3:S+ C1-3:X2-3:S+ C2-3:X2-3 \323 . X1 . P2 . 0 A1:0-3,A2:1-3,XA1:1,XA2:,XA3:,A3:2-3 \324 A1:P0,A3:P-2"325 " C0-3:X2-3:S+ C1-3:X2-3:S+ C2-3:X2-3 \326 . . X3 P2 . 0 A1:0-2,A2:1-2,XA2:3,XA3:3,A3:3 \327 A1:P0,A3:P2 3"328 " C0-3:X2-3:S+ C1-3:X2-3:S+ C2-3:X2-3:P2 \329 . . X3 . . 0 A1:0-3,A2:1-3,XA2:3,XA3:3,A3:2-3 \330 A1:P0,A3:P-2"331 " C0-3:X2-3:S+ C1-3:X2-3:S+ C2-3:X2-3:P2 \332 . X4 . . . 0 A1:0-3,A2:1-3,A3:2-3,XA1:4,XA2:,XA3 \333 A1:P0,A3:P-2"334 335 # old-A1 old-A2 old-A3 old-B1 new-A1 new-A2 new-A3 new-B1 fail ECPUs Pstate ISOLCPUS336 # ------ ------ ------ ------ ------ ------ ------ ------ ---- ----- ------ --------337 #338 # Incorrect change to cpuset.cpus[.exclusive] invalidates partition root339 #340 # Adding CPUs to partition root that are not in parent's341 # cpuset.cpus is allowed, but those extra CPUs are ignored.342 "C2-3:P1:S+ C3:P1 . . . C2-4 . . 0 A1:,A2:2-3 A1:P1,A2:P1"343 344 # Taking away all CPUs from parent or itself if there are tasks345 # will make the partition invalid.346 "C2-3:P1:S+ C3:P1 . . T C2-3 . . 0 A1:2-3,A2:2-3 A1:P1,A2:P-1"347 " C3:P1:S+ C3 . . T P1 . . 0 A1:3,A2:3 A1:P1,A2:P-1"348 "$SETUP_A123_PARTITIONS . T:C2-3 . . . 0 A1:2-3,A2:2-3,A3:3 A1:P1,A2:P-1,A3:P-1"349 "$SETUP_A123_PARTITIONS . T:C2-3:C1-3 . . . 0 A1:1,A2:2,A3:3 A1:P1,A2:P1,A3:P1"350 351 # Changing a partition root to member makes child partitions invalid352 "C2-3:P1:S+ C3:P1 . . P0 . . . 0 A1:2-3,A2:3 A1:P0,A2:P-1"353 "$SETUP_A123_PARTITIONS . C2-3 P0 . . 0 A1:2-3,A2:2-3,A3:3 A1:P1,A2:P0,A3:P-1"354 355 # cpuset.cpus can contains cpus not in parent's cpuset.cpus as long356 # as they overlap.357 "C2-3:P1:S+ . . . . C3-4:P1 . . 0 A1:2,A2:3 A1:P1,A2:P1"358 359 # Deletion of CPUs distributed to child cgroup is allowed.360 "C0-1:P1:S+ C1 . C2-3 C4-5 . . . 0 A1:4-5,A2:4-5"361 362 # To become a valid partition root, cpuset.cpus must overlap parent's363 # cpuset.cpus.364 " C0-1:P1 . . C2-3 S+ C4-5:P1 . . 0 A1:0-1,A2:0-1 A1:P1,A2:P-1"365 366 # Enabling partition with child cpusets is allowed367 " C0-1:S+ C1 . C2-3 P1 . . . 0 A1:0-1,A2:1 A1:P1"368 369 # A partition root with non-partition root parent is invalid, but it370 # can be made valid if its parent becomes a partition root too.371 " C0-1:S+ C1 . C2-3 . P2 . . 0 A1:0-1,A2:1 A1:P0,A2:P-2"372 " C0-1:S+ C1:P2 . C2-3 P1 . . . 0 A1:0,A2:1 A1:P1,A2:P2"373 374 # A non-exclusive cpuset.cpus change will invalidate partition and its siblings375 " C0-1:P1 . . C2-3 C0-2 . . . 0 A1:0-2,B1:2-3 A1:P-1,B1:P0"376 " C0-1:P1 . . P1:C2-3 C0-2 . . . 0 A1:0-2,B1:2-3 A1:P-1,B1:P-1"377 " C0-1 . . P1:C2-3 C0-2 . . . 0 A1:0-2,B1:2-3 A1:P0,B1:P-1"378 379 # cpuset.cpus can overlap with sibling cpuset.cpus.exclusive but not subsumed by it380 " C0-3 . . C4-5 X5 . . . 0 A1:0-3,B1:4-5"381 382 # Child partition root that try to take all CPUs from parent partition383 # with tasks will remain invalid.384 " C1-4:P1:S+ P1 . . . . . . 0 A1:1-4,A2:1-4 A1:P1,A2:P-1"385 " C1-4:P1:S+ P1 . . . C1-4 . . 0 A1,A2:1-4 A1:P1,A2:P1"386 " C1-4:P1:S+ P1 . . T C1-4 . . 0 A1:1-4,A2:1-4 A1:P1,A2:P-1"387 388 # Clearing of cpuset.cpus with a preset cpuset.cpus.exclusive shouldn't389 # affect cpuset.cpus.exclusive.effective.390 " C1-4:X3:S+ C1:X3 . . . C . . 0 A2:1-4,XA2:3"391 392 # old-A1 old-A2 old-A3 old-B1 new-A1 new-A2 new-A3 new-B1 fail ECPUs Pstate ISOLCPUS393 # ------ ------ ------ ------ ------ ------ ------ ------ ---- ----- ------ --------394 # Failure cases:395 396 # A task cannot be added to a partition with no cpu397 "C2-3:P1:S+ C3:P1 . . O2=0:T . . . 1 A1:,A2:3 A1:P1,A2:P1"398 399 # Changes to cpuset.cpus.exclusive that violate exclusivity rule is rejected400 " C0-3 . . C4-5 X0-3 . . X3-5 1 A1:0-3,B1:4-5"401 402 # cpuset.cpus cannot be a subset of sibling cpuset.cpus.exclusive403 " C0-3 . . C4-5 X3-5 . . . 1 A1:0-3,B1:4-5"404)405 406#407# Write to the cpu online file408# $1 - <c>=<v> where <c> = cpu number, <v> value to be written409#410write_cpu_online()411{412 CPU=${1%=*}413 VAL=${1#*=}414 CPUFILE=//sys/devices/system/cpu/cpu${CPU}/online415 if [[ $VAL -eq 0 ]]416 then417 OFFLINE_CPUS="$OFFLINE_CPUS $CPU"418 else419 [[ -n "$OFFLINE_CPUS" ]] && {420 OFFLINE_CPUS=$(echo $CPU $CPU $OFFLINE_CPUS | fmt -1 |\421 sort | uniq -u)422 }423 fi424 echo $VAL > $CPUFILE425 pause 0.05426}427 428#429# Set controller state430# $1 - cgroup directory431# $2 - state432# $3 - showerr433#434# The presence of ":" in state means transition from one to the next.435#436set_ctrl_state()437{438 TMPMSG=/tmp/.msg_$$439 CGRP=$1440 STATE=$2441 SHOWERR=${3}442 CTRL=${CTRL:=$CONTROLLER}443 HASERR=0444 REDIRECT="2> $TMPMSG"445 [[ -z "$STATE" || "$STATE" = '.' ]] && return 0446 [[ $VERBOSE -gt 0 ]] && SHOWERR=1447 448 rm -f $TMPMSG449 for CMD in $(echo $STATE | sed -e "s/:/ /g")450 do451 TFILE=$CGRP/cgroup.procs452 SFILE=$CGRP/cgroup.subtree_control453 PFILE=$CGRP/cpuset.cpus.partition454 CFILE=$CGRP/cpuset.cpus455 XFILE=$CGRP/cpuset.cpus.exclusive456 S=$(expr substr $CMD 1 1)457 if [[ $S = S ]]458 then459 PREFIX=${CMD#?}460 COMM="echo ${PREFIX}${CTRL} > $SFILE"461 eval $COMM $REDIRECT462 elif [[ $S = X ]]463 then464 CPUS=${CMD#?}465 COMM="echo $CPUS > $XFILE"466 eval $COMM $REDIRECT467 elif [[ $S = C ]]468 then469 CPUS=${CMD#?}470 COMM="echo $CPUS > $CFILE"471 eval $COMM $REDIRECT472 elif [[ $S = P ]]473 then474 VAL=${CMD#?}475 case $VAL in476 0) VAL=member477 ;;478 1) VAL=root479 ;;480 2) VAL=isolated481 ;;482 *)483 echo "Invalid partition state - $VAL"484 exit 1485 ;;486 esac487 COMM="echo $VAL > $PFILE"488 eval $COMM $REDIRECT489 elif [[ $S = O ]]490 then491 VAL=${CMD#?}492 write_cpu_online $VAL493 elif [[ $S = T ]]494 then495 COMM="echo 0 > $TFILE"496 eval $COMM $REDIRECT497 fi498 RET=$?499 [[ $RET -ne 0 ]] && {500 [[ -n "$SHOWERR" ]] && {501 echo "$COMM"502 cat $TMPMSG503 }504 HASERR=1505 }506 pause 0.01507 rm -f $TMPMSG508 done509 return $HASERR510}511 512set_ctrl_state_noerr()513{514 CGRP=$1515 STATE=$2516 [[ -d $CGRP ]] || mkdir $CGRP517 set_ctrl_state $CGRP $STATE 1518 [[ $? -ne 0 ]] && {519 echo "ERROR: Failed to set $2 to cgroup $1!"520 exit 1521 }522}523 524online_cpus()525{526 [[ -n "OFFLINE_CPUS" ]] && {527 for C in $OFFLINE_CPUS528 do529 write_cpu_online ${C}=1530 done531 }532}533 534#535# Return 1 if the list of effective cpus isn't the same as the initial list.536#537reset_cgroup_states()538{539 echo 0 > $CGROUP2/cgroup.procs540 online_cpus541 rmdir A1/A2/A3 A1/A2 A1 B1 > /dev/null 2>&1542 pause 0.02543 set_ctrl_state . R-544 pause 0.01545}546 547dump_states()548{549 for DIR in . A1 A1/A2 A1/A2/A3 B1550 do551 CPUS=$DIR/cpuset.cpus552 ECPUS=$DIR/cpuset.cpus.effective553 XCPUS=$DIR/cpuset.cpus.exclusive554 XECPUS=$DIR/cpuset.cpus.exclusive.effective555 PRS=$DIR/cpuset.cpus.partition556 PCPUS=$DIR/.__DEBUG__.cpuset.cpus.subpartitions557 ISCPUS=$DIR/cpuset.cpus.isolated558 [[ -e $CPUS ]] && echo "$CPUS: $(cat $CPUS)"559 [[ -e $XCPUS ]] && echo "$XCPUS: $(cat $XCPUS)"560 [[ -e $ECPUS ]] && echo "$ECPUS: $(cat $ECPUS)"561 [[ -e $XECPUS ]] && echo "$XECPUS: $(cat $XECPUS)"562 [[ -e $PRS ]] && echo "$PRS: $(cat $PRS)"563 [[ -e $PCPUS ]] && echo "$PCPUS: $(cat $PCPUS)"564 [[ -e $ISCPUS ]] && echo "$ISCPUS: $(cat $ISCPUS)"565 done566}567 568#569# Check effective cpus570# $1 - check string, format: <cgroup>:<cpu-list>[,<cgroup>:<cpu-list>]*571#572check_effective_cpus()573{574 CHK_STR=$1575 for CHK in $(echo $CHK_STR | sed -e "s/,/ /g")576 do577 set -- $(echo $CHK | sed -e "s/:/ /g")578 CGRP=$1579 CPUS=$2580 if [[ $CGRP = X* ]]581 then582 CGRP=${CGRP#X}583 FILE=cpuset.cpus.exclusive.effective584 else585 FILE=cpuset.cpus.effective586 fi587 [[ $CGRP = A2 ]] && CGRP=A1/A2588 [[ $CGRP = A3 ]] && CGRP=A1/A2/A3589 [[ -e $CGRP/$FILE ]] || return 1590 [[ $CPUS = $(cat $CGRP/$FILE) ]] || return 1591 done592}593 594#595# Check cgroup states596# $1 - check string, format: <cgroup>:<state>[,<cgroup>:<state>]*597#598check_cgroup_states()599{600 CHK_STR=$1601 for CHK in $(echo $CHK_STR | sed -e "s/,/ /g")602 do603 set -- $(echo $CHK | sed -e "s/:/ /g")604 CGRP=$1605 CGRP_DIR=$CGRP606 STATE=$2607 FILE=608 EVAL=$(expr substr $STATE 2 2)609 [[ $CGRP = A2 ]] && CGRP_DIR=A1/A2610 [[ $CGRP = A3 ]] && CGRP_DIR=A1/A2/A3611 612 case $STATE in613 P*) FILE=$CGRP_DIR/cpuset.cpus.partition614 ;;615 *) echo "Unknown state: $STATE!"616 exit 1617 ;;618 esac619 VAL=$(cat $FILE)620 621 case "$VAL" in622 member) VAL=0623 ;;624 root) VAL=1625 ;;626 isolated)627 VAL=2628 ;;629 "root invalid"*)630 VAL=-1631 ;;632 "isolated invalid"*)633 VAL=-2634 ;;635 esac636 [[ $EVAL != $VAL ]] && return 1637 638 #639 # For root partition, dump sched-domains info to console if640 # verbose mode set for manual comparison with sched debug info.641 #642 [[ $VAL -eq 1 && $VERBOSE -gt 0 ]] && {643 DOMS=$(cat $CGRP_DIR/cpuset.cpus.effective)644 [[ -n "$DOMS" ]] &&645 echo " [$CGRP] sched-domain: $DOMS" > $CONSOLE646 }647 done648 return 0649}650 651#652# Get isolated (including offline) CPUs by looking at653# /sys/kernel/debug/sched/domains and cpuset.cpus.isolated control file,654# if available, and compare that with the expected value.655#656# Note that isolated CPUs from the sched/domains context include offline657# CPUs as well as CPUs in non-isolated 1-CPU partition. Those CPUs may658# not be included in the cpuset.cpus.isolated control file which contains659# only CPUs in isolated partitions as well as those that are isolated at660# boot time.661#662# $1 - expected isolated cpu list(s) <isolcpus1>{,<isolcpus2>}663# <isolcpus1> - expected sched/domains value664# <isolcpus2> - cpuset.cpus.isolated value = <isolcpus1> if not defined665#666check_isolcpus()667{668 EXPECT_VAL=$1669 ISOLCPUS=670 LASTISOLCPU=671 SCHED_DOMAINS=/sys/kernel/debug/sched/domains672 ISCPUS=${CGROUP2}/cpuset.cpus.isolated673 if [[ $EXPECT_VAL = . ]]674 then675 EXPECT_VAL=676 EXPECT_VAL2=677 elif [[ $(expr $EXPECT_VAL : ".*,.*") > 0 ]]678 then679 set -- $(echo $EXPECT_VAL | sed -e "s/,/ /g")680 EXPECT_VAL=$1681 EXPECT_VAL2=$2682 else683 EXPECT_VAL2=$EXPECT_VAL684 fi685 686 #687 # Check cpuset.cpus.isolated cpumask688 #689 if [[ -z "$BOOT_ISOLCPUS" ]]690 then691 ISOLCPUS=$(cat $ISCPUS)692 else693 ISOLCPUS=$(cat $ISCPUS | sed -e "s/,*$BOOT_ISOLCPUS//")694 fi695 [[ "$EXPECT_VAL2" != "$ISOLCPUS" ]] && {696 # Take a 50ms pause and try again697 pause 0.05698 ISOLCPUS=$(cat $ISCPUS)699 }700 [[ "$EXPECT_VAL2" != "$ISOLCPUS" ]] && return 1701 ISOLCPUS=702 703 #704 # Use the sched domain in debugfs to check isolated CPUs, if available705 #706 [[ -d $SCHED_DOMAINS ]] || return 0707 708 for ((CPU=0; CPU < $NR_CPUS; CPU++))709 do710 [[ -n "$(ls ${SCHED_DOMAINS}/cpu$CPU)" ]] && continue711 712 if [[ -z "$LASTISOLCPU" ]]713 then714 ISOLCPUS=$CPU715 LASTISOLCPU=$CPU716 elif [[ "$LASTISOLCPU" -eq $((CPU - 1)) ]]717 then718 echo $ISOLCPUS | grep -q "\<$LASTISOLCPU\$"719 if [[ $? -eq 0 ]]720 then721 ISOLCPUS=${ISOLCPUS}-722 fi723 LASTISOLCPU=$CPU724 else725 if [[ $ISOLCPUS = *- ]]726 then727 ISOLCPUS=${ISOLCPUS}$LASTISOLCPU728 fi729 ISOLCPUS=${ISOLCPUS},$CPU730 LASTISOLCPU=$CPU731 fi732 done733 [[ "$ISOLCPUS" = *- ]] && ISOLCPUS=${ISOLCPUS}$LASTISOLCPU734 [[ -n "BOOT_ISOLCPUS" ]] &&735 ISOLCPUS=$(echo $ISOLCPUS | sed -e "s/,*$BOOT_ISOLCPUS//")736 737 [[ "$EXPECT_VAL" = "$ISOLCPUS" ]]738}739 740test_fail()741{742 TESTNUM=$1743 TESTTYPE=$2744 ADDINFO=$3745 echo "Test $TEST[$TESTNUM] failed $TESTTYPE check!"746 [[ -n "$ADDINFO" ]] && echo "*** $ADDINFO ***"747 eval echo \${$TEST[$I]}748 echo749 dump_states750 exit 1751}752 753#754# Check to see if there are unexpected isolated CPUs left beyond the boot755# time isolated ones.756#757null_isolcpus_check()758{759 [[ $VERBOSE -gt 0 ]] || return 0760 # Retry a few times before printing error761 RETRY=0762 while [[ $RETRY -lt 8 ]]763 do764 pause 0.02765 check_isolcpus "."766 [[ $? -eq 0 ]] && return 0767 ((RETRY++))768 done769 echo "Unexpected isolated CPUs: $ISOLCPUS"770 dump_states771 exit 1772}773 774#775# Run cpuset state transition test776# $1 - test matrix name777#778# This test is somewhat fragile as delays (sleep x) are added in various779# places to make sure state changes are fully propagated before the next780# action. These delays may need to be adjusted if running in a slower machine.781#782run_state_test()783{784 TEST=$1785 CONTROLLER=cpuset786 I=0787 eval CNT="\${#$TEST[@]}"788 789 reset_cgroup_states790 console_msg "Running state transition test ..."791 792 while [[ $I -lt $CNT ]]793 do794 echo "Running test $I ..." > $CONSOLE795 [[ $VERBOSE -gt 1 ]] && {796 echo ""797 eval echo \${$TEST[$I]}798 }799 eval set -- "\${$TEST[$I]}"800 OLD_A1=$1801 OLD_A2=$2802 OLD_A3=$3803 OLD_B1=$4804 NEW_A1=$5805 NEW_A2=$6806 NEW_A3=$7807 NEW_B1=$8808 RESULT=$9809 ECPUS=${10}810 STATES=${11}811 ICPUS=${12}812 813 set_ctrl_state_noerr B1 $OLD_B1814 set_ctrl_state_noerr A1 $OLD_A1815 set_ctrl_state_noerr A1/A2 $OLD_A2816 set_ctrl_state_noerr A1/A2/A3 $OLD_A3817 RETVAL=0818 set_ctrl_state A1 $NEW_A1; ((RETVAL += $?))819 set_ctrl_state A1/A2 $NEW_A2; ((RETVAL += $?))820 set_ctrl_state A1/A2/A3 $NEW_A3; ((RETVAL += $?))821 set_ctrl_state B1 $NEW_B1; ((RETVAL += $?))822 823 [[ $RETVAL -ne $RESULT ]] && test_fail $I result824 825 [[ -n "$ECPUS" && "$ECPUS" != . ]] && {826 check_effective_cpus $ECPUS827 [[ $? -ne 0 ]] && test_fail $I "effective CPU"828 }829 830 [[ -n "$STATES" && "$STATES" != . ]] && {831 check_cgroup_states $STATES832 [[ $? -ne 0 ]] && test_fail $I states833 }834 835 # Compare the expected isolated CPUs with the actual ones,836 # if available837 [[ -n "$ICPUS" ]] && {838 check_isolcpus $ICPUS839 [[ $? -ne 0 ]] && test_fail $I "isolated CPU" \840 "Expect $ICPUS, get $ISOLCPUS instead"841 }842 reset_cgroup_states843 #844 # Check to see if effective cpu list changes845 #846 NEWLIST=$(cat cpuset.cpus.effective)847 RETRY=0848 while [[ $NEWLIST != $CPULIST && $RETRY -lt 8 ]]849 do850 # Wait a bit longer & recheck a few times851 pause 0.02852 ((RETRY++))853 NEWLIST=$(cat cpuset.cpus.effective)854 done855 [[ $NEWLIST != $CPULIST ]] && {856 echo "Effective cpus changed to $NEWLIST after test $I!"857 exit 1858 }859 null_isolcpus_check860 [[ $VERBOSE -gt 0 ]] && echo "Test $I done."861 ((I++))862 done863 echo "All $I tests of $TEST PASSED."864}865 866#867# Testing the new "isolated" partition root type868#869test_isolated()870{871 cd $CGROUP2/test872 echo 2-3 > cpuset.cpus873 TYPE=$(cat cpuset.cpus.partition)874 [[ $TYPE = member ]] || echo member > cpuset.cpus.partition875 876 console_msg "Change from member to root"877 test_partition root878 879 console_msg "Change from root to isolated"880 test_partition isolated881 882 console_msg "Change from isolated to member"883 test_partition member884 885 console_msg "Change from member to isolated"886 test_partition isolated887 888 console_msg "Change from isolated to root"889 test_partition root890 891 console_msg "Change from root to member"892 test_partition member893 894 #895 # Testing partition root with no cpu896 #897 console_msg "Distribute all cpus to child partition"898 echo +cpuset > cgroup.subtree_control899 test_partition root900 901 mkdir A1902 cd A1903 echo 2-3 > cpuset.cpus904 test_partition root905 test_effective_cpus 2-3906 cd ..907 test_effective_cpus ""908 909 console_msg "Moving task to partition test"910 test_add_proc "No space left"911 cd A1912 test_add_proc ""913 cd ..914 915 console_msg "Shrink and expand child partition"916 cd A1917 echo 2 > cpuset.cpus918 cd ..919 test_effective_cpus 3920 cd A1921 echo 2-3 > cpuset.cpus922 cd ..923 test_effective_cpus ""924 925 # Cleaning up926 console_msg "Cleaning up"927 echo $$ > $CGROUP2/cgroup.procs928 [[ -d A1 ]] && rmdir A1929 null_isolcpus_check930}931 932#933# Wait for inotify event for the given file and read it934# $1: cgroup file to wait for935# $2: file to store the read result936#937wait_inotify()938{939 CGROUP_FILE=$1940 OUTPUT_FILE=$2941 942 $WAIT_INOTIFY $CGROUP_FILE943 cat $CGROUP_FILE > $OUTPUT_FILE944}945 946#947# Test if inotify events are properly generated when going into and out of948# invalid partition state.949#950test_inotify()951{952 ERR=0953 PRS=/tmp/.prs_$$954 cd $CGROUP2/test955 [[ -f $WAIT_INOTIFY ]] || {956 echo "wait_inotify not found, inotify test SKIPPED."957 return958 }959 960 pause 0.01961 echo 1 > cpuset.cpus962 echo 0 > cgroup.procs963 echo root > cpuset.cpus.partition964 pause 0.01965 rm -f $PRS966 wait_inotify $PWD/cpuset.cpus.partition $PRS &967 pause 0.01968 set_ctrl_state . "O1=0"969 pause 0.01970 check_cgroup_states ".:P-1"971 if [[ $? -ne 0 ]]972 then973 echo "FAILED: Inotify test - partition not invalid"974 ERR=1975 elif [[ ! -f $PRS ]]976 then977 echo "FAILED: Inotify test - event not generated"978 ERR=1979 kill %1980 elif [[ $(cat $PRS) != "root invalid"* ]]981 then982 echo "FAILED: Inotify test - incorrect state"983 cat $PRS984 ERR=1985 fi986 online_cpus987 echo member > cpuset.cpus.partition988 echo 0 > ../cgroup.procs989 if [[ $ERR -ne 0 ]]990 then991 exit 1992 else993 echo "Inotify test PASSED"994 fi995}996 997trap cleanup 0 2 3 6998run_state_test TEST_MATRIX999test_isolated1000test_inotify1001echo "All tests PASSED."1002