114 lines · python
1# RUN: %{lit} %{inputs}/discovery | FileCheck --check-prefix=CHECK-BASIC %s2# CHECK-BASIC: Testing: 5 tests3 4 5# Check that we exit with an error if we do not discover any tests, even with --allow-empty-runs.6#7# RUN: not %{lit} %{inputs}/nonexistent 2>&1 | FileCheck --check-prefix=CHECK-BAD-PATH %s8# RUN: not %{lit} %{inputs}/nonexistent --allow-empty-runs 2>&1 | FileCheck --check-prefix=CHECK-BAD-PATH %s9# CHECK-BAD-PATH: error: did not discover any tests for provided path(s)10 11# Check that we exit with an error if we filter out all tests, but allow it with --allow-empty-runs.12# Check that we exit with an error if we skip all tests, but allow it with --allow-empty-runs.13#14# RUN: not %{lit} --filter 'nonexistent' %{inputs}/discovery 2>&1 | FileCheck --check-prefixes=CHECK-BAD-FILTER,CHECK-BAD-FILTER-ERROR %s15# RUN: %{lit} --filter 'nonexistent' --allow-empty-runs %{inputs}/discovery 2>&1 | FileCheck --check-prefixes=CHECK-BAD-FILTER,CHECK-BAD-FILTER-ALLOW %s16# RUN: not %{lit} --filter-out '.*' %{inputs}/discovery 2>&1 | FileCheck --check-prefixes=CHECK-BAD-FILTER,CHECK-BAD-FILTER-ERROR %s17# RUN: %{lit} --filter-out '.*' --allow-empty-runs %{inputs}/discovery 2>&1 | FileCheck --check-prefixes=CHECK-BAD-FILTER,CHECK-BAD-FILTER-ALLOW %s18# CHECK-BAD-FILTER: error: filter did not match any tests (of 5 discovered).19# CHECK-BAD-FILTER-ERROR: Use '--allow-empty-runs' to suppress this error.20# CHECK-BAD-FILTER-ALLOW: Suppressing error because '--allow-empty-runs' was specified.21 22# Check that regex-filtering works, is case-insensitive, and can be configured via env var.23#24# RUN: %{lit} --filter 'o[a-z]e' %{inputs}/discovery | FileCheck --check-prefix=CHECK-FILTER %s25# RUN: %{lit} --filter 'O[A-Z]E' %{inputs}/discovery | FileCheck --check-prefix=CHECK-FILTER %s26# RUN: env LIT_FILTER='o[a-z]e' %{lit} %{inputs}/discovery | FileCheck --check-prefix=CHECK-FILTER %s27# RUN: %{lit} --filter-out 'test-t[a-z]' %{inputs}/discovery | FileCheck --check-prefix=CHECK-FILTER %s28# RUN: %{lit} --filter-out 'test-t[A-Z]' %{inputs}/discovery | FileCheck --check-prefix=CHECK-FILTER %s29# RUN: env LIT_FILTER_OUT='test-t[a-z]' %{lit} %{inputs}/discovery | FileCheck --check-prefix=CHECK-FILTER %s30# CHECK-FILTER: Testing: 2 of 5 tests31# CHECK-FILTER: Excluded: 332 33 34# Check that maximum counts work35#36# RUN: %{lit} --max-tests 3 %{inputs}/discovery | FileCheck --check-prefix=CHECK-MAX %s37# CHECK-MAX: Testing: 3 of 5 tests38# CHECK-MAX: Excluded: 239 40 41# Check that sharding partitions the testsuite in a way that distributes the42# rounding error nicely (i.e. 5/3 => 2 2 1, not 1 1 3 or whatever)43#44# RUN: %{lit} --num-shards 3 --run-shard 1 %{inputs}/discovery >%t.out 2>%t.err45# RUN: FileCheck --check-prefix=CHECK-SHARD0-ERR < %t.err %s46# RUN: FileCheck --check-prefix=CHECK-SHARD0-OUT < %t.out %s47# CHECK-SHARD0-ERR: note: Selecting shard 1/3 = size 2/5 = tests #(3*k)+1 = [1, 4]48# CHECK-SHARD0-OUT: Testing: 2 of 5 tests49# CHECK-SHARD0-OUT: Excluded: 350#51# RUN: %{lit} --num-shards 3 --run-shard 2 %{inputs}/discovery >%t.out 2>%t.err52# RUN: FileCheck --check-prefix=CHECK-SHARD1-ERR < %t.err %s53# RUN: FileCheck --check-prefix=CHECK-SHARD1-OUT < %t.out %s54# CHECK-SHARD1-ERR: note: Selecting shard 2/3 = size 2/5 = tests #(3*k)+2 = [2, 5]55# CHECK-SHARD1-OUT: Testing: 2 of 5 tests56#57# RUN: %{lit} --num-shards 3 --run-shard 3 %{inputs}/discovery >%t.out 2>%t.err58# RUN: FileCheck --check-prefix=CHECK-SHARD2-ERR < %t.err %s59# RUN: FileCheck --check-prefix=CHECK-SHARD2-OUT < %t.out %s60# CHECK-SHARD2-ERR: note: Selecting shard 3/3 = size 1/5 = tests #(3*k)+3 = [3]61# CHECK-SHARD2-OUT: Testing: 1 of 5 tests62 63 64# Check that sharding via env vars works.65#66# RUN: env LIT_NUM_SHARDS=3 LIT_RUN_SHARD=1 %{lit} %{inputs}/discovery >%t.out 2>%t.err67# RUN: FileCheck --check-prefix=CHECK-SHARD0-ENV-ERR < %t.err %s68# RUN: FileCheck --check-prefix=CHECK-SHARD0-ENV-OUT < %t.out %s69# CHECK-SHARD0-ENV-ERR: note: Selecting shard 1/3 = size 2/5 = tests #(3*k)+1 = [1, 4]70# CHECK-SHARD0-ENV-OUT: Testing: 2 of 5 tests71#72# RUN: env LIT_NUM_SHARDS=3 LIT_RUN_SHARD=2 %{lit} %{inputs}/discovery >%t.out 2>%t.err73# RUN: FileCheck --check-prefix=CHECK-SHARD1-ENV-ERR < %t.err %s74# RUN: FileCheck --check-prefix=CHECK-SHARD1-ENV-OUT < %t.out %s75# CHECK-SHARD1-ENV-ERR: note: Selecting shard 2/3 = size 2/5 = tests #(3*k)+2 = [2, 5]76# CHECK-SHARD1-ENV-OUT: Testing: 2 of 5 tests77#78# RUN: env LIT_NUM_SHARDS=3 LIT_RUN_SHARD=3 %{lit} %{inputs}/discovery >%t.out 2>%t.err79# RUN: FileCheck --check-prefix=CHECK-SHARD2-ENV-ERR < %t.err %s80# RUN: FileCheck --check-prefix=CHECK-SHARD2-ENV-OUT < %t.out %s81# CHECK-SHARD2-ENV-ERR: note: Selecting shard 3/3 = size 1/5 = tests #(3*k)+3 = [3]82# CHECK-SHARD2-ENV-OUT: Testing: 1 of 5 tests83 84 85# Check that providing more shards than tests results in 1 test per shard86# until we run out, then 0.87#88# RUN: %{lit} --num-shards 100 --run-shard 2 %{inputs}/discovery >%t.out 2>%t.err89# RUN: FileCheck --check-prefix=CHECK-SHARD-BIG-ERR1 < %t.err %s90# RUN: FileCheck --check-prefix=CHECK-SHARD-BIG-OUT1 < %t.out %s91# CHECK-SHARD-BIG-ERR1: note: Selecting shard 2/100 = size 1/5 = tests #(100*k)+2 = [2]92# CHECK-SHARD-BIG-OUT1: Testing: 1 of 5 tests93#94# RUN: %{lit} --num-shards 100 --run-shard 6 %{inputs}/discovery >%t.out 2>%t.err95# RUN: FileCheck --check-prefix=CHECK-SHARD-BIG-ERR2 < %t.err %s96# CHECK-SHARD-BIG-ERR2: note: Selecting shard 6/100 = size 0/5 = tests #(100*k)+6 = []97# CHECK-SHARD-BIG-ERR2: warning: shard does not contain any tests. Consider decreasing the number of shards.98#99# RUN: %{lit} --num-shards 100 --run-shard 50 %{inputs}/discovery >%t.out 2>%t.err100# RUN: FileCheck --check-prefix=CHECK-SHARD-BIG-ERR3 < %t.err %s101# CHECK-SHARD-BIG-ERR3: note: Selecting shard 50/100 = size 0/5 = tests #(100*k)+50 = []102# CHECK-SHARD-BIG-ERR3: warning: shard does not contain any tests. Consider decreasing the number of shards.103 104 105# Check that range constraints are enforced106#107# RUN: not %{lit} --num-shards 0 --run-shard 2 %{inputs}/discovery >%t.out 2>%t.err108# RUN: FileCheck --check-prefix=CHECK-SHARD-ERR < %t.err %s109# CHECK-SHARD-ERR: error: argument --num-shards: requires positive integer, but found '0'110#111# RUN: not %{lit} --num-shards 3 --run-shard 4 %{inputs}/discovery >%t.out 2>%t.err112# RUN: FileCheck --check-prefix=CHECK-SHARD-ERR2 < %t.err %s113# CHECK-SHARD-ERR2: error: --run-shard must be between 1 and --num-shards (inclusive)114