215 lines · python
1# Check the basic discovery process, including a sub-suite.2#3# RUN: %{lit} %{inputs}/discovery \4# RUN: --debug --show-tests --show-suites \5# RUN: -v > %t.out 2> %t.err6# RUN: FileCheck --check-prefix=CHECK-BASIC-OUT < %t.out %s7# RUN: FileCheck --check-prefix=CHECK-BASIC-ERR < %t.err %s8#9# CHECK-BASIC-ERR: loading suite config '{{.*(/|\\\\)discovery(/|\\\\)lit.cfg}}'10# CHECK-BASIC-ERR-DAG: loading suite config '{{.*(/|\\\\)discovery(/|\\\\)subsuite(/|\\\\)lit.cfg}}'11# CHECK-BASIC-ERR-DAG: loading local config '{{.*(/|\\\\)discovery(/|\\\\)subdir(/|\\\\)lit.local.cfg}}'12#13# CHECK-BASIC-OUT: -- Test Suites --14# CHECK-BASIC-OUT: sub-suite - 2 tests15# CHECK-BASIC-OUT: Source Root: {{.*[/\\]discovery[/\\]subsuite$}}16# CHECK-BASIC-OUT: Exec Root : {{.*[/\\]discovery[/\\]subsuite$}}17# CHECK-BASIC-OUT: top-level-suite - 3 tests18# CHECK-BASIC-OUT: Source Root: {{.*[/\\]discovery$}}19# CHECK-BASIC-OUT: Exec Root : {{.*[/\\]discovery$}}20# CHECK-BASIC-OUT: Available Features: feature1 feature221# CHECK-BASIC-OUT: Available Substitutions: %key1 => value122# CHECK-BASIC-OUT: %key2 => value223#24# CHECK-BASIC-OUT: -- Available Tests --25# CHECK-BASIC-OUT: sub-suite :: test-one26# CHECK-BASIC-OUT: sub-suite :: test-two27# CHECK-BASIC-OUT: top-level-suite :: subdir/test-three28# CHECK-BASIC-OUT: top-level-suite :: test-one29# CHECK-BASIC-OUT: top-level-suite :: test-two30 31# RUN: %{lit} %{inputs}/discovery \32# RUN: -v > %t.out 2> %t.err33# RUN: FileCheck --check-prefix=CHECK-PERCENTAGES-OUT < %/t.out %s34#35# CHECK-PERCENTAGES-OUT: Total Discovered Tests: {{[0-9]*}}36# CHECK-PERCENTAGES-OUT: Passed: {{[0-9]*}} {{\([0-9]*\.[0-9]*%\)}}37 38# Check discovery when providing the special builtin 'config_map'39# RUN: %{python} %{inputs}/config-map-discovery/driver.py \40# RUN: %{inputs}/config-map-discovery/main-config/lit.cfg \41# RUN: %{inputs}/config-map-discovery/lit.alt.cfg \42# RUN: --workers=1 --debug --show-tests --show-suites > %t.out 2> %t.err43# RUN: FileCheck --check-prefix=CHECK-CONFIG-MAP-OUT < %t.out %s44# RUN: FileCheck --check-prefix=CHECK-CONFIG-MAP-ERR < %t.err %s45 46# CHECK-CONFIG-MAP-OUT-NOT: ERROR: lit.cfg invoked47# CHECK-CONFIG-MAP-OUT: -- Test Suites --48# CHECK-CONFIG-MAP-OUT: config-map - 2 tests49# CHECK-CONFIG-MAP-OUT: Source Root: {{.*[/\\]config-map-discovery[/\\]tests}}50# CHECK-CONFIG-MAP-OUT: Exec Root : {{.*[/\\]tests[/\\]Inputs[/\\]config-map-discovery}}51# CHECK-CONFIG-MAP-OUT: -- Available Tests --52# CHECK-CONFIG-MAP-OUT-NOT: invalid-test.txt53# CHECK-CONFIG-MAP-OUT: config-map :: test1.txt54# CHECK-CONFIG-MAP-OUT: config-map :: test2.txt55 56# CHECK-CONFIG-MAP-ERR: loading suite config '{{.*}}lit.alt.cfg'57# CHECK-CONFIG-MAP-ERR: loaded config '{{.*}}lit.alt.cfg'58# CHECK-CONFIG-MAP-ERR: resolved input '{{.*(/|\\\\)config-map-discovery(/|\\\\)main-config}}' to 'config-map'::()59 60 61# Check discovery when tests are named directly.62#63# RUN: %{lit} \64# RUN: %{inputs}/discovery/subdir/test-three.py \65# RUN: %{inputs}/discovery/subsuite/test-one.txt \66# RUN: --show-tests --show-suites -v > %t.out67# RUN: FileCheck --check-prefix=CHECK-DIRECT-TEST < %t.out %s68#69# CHECK-DIRECT-TEST: -- Available Tests --70# CHECK-DIRECT-TEST: sub-suite :: test-one71# CHECK-DIRECT-TEST: top-level-suite :: subdir/test-three72 73# Check discovery when config files end in .py74# RUN: %{lit} %{inputs}/py-config-discovery \75# RUN: --debug --show-tests --show-suites \76# RUN: -v > %t.out 2> %t.err77# RUN: FileCheck --check-prefix=CHECK-PYCONFIG-OUT < %t.out %s78# RUN: FileCheck --check-prefix=CHECK-PYCONFIG-ERR < %t.err %s79#80# CHECK-PYCONFIG-ERR: loading suite config '{{.*(/|\\\\)py-config-discovery(/|\\\\)lit.site.cfg.py}}'81# CHECK-PYCONFIG-ERR: load_config from '{{.*(/|\\\\)discovery(/|\\\\)lit.cfg}}'82# CHECK-PYCONFIG-ERR: loaded config '{{.*(/|\\\\)discovery(/|\\\\)lit.cfg}}'83# CHECK-PYCONFIG-ERR: loaded config '{{.*(/|\\\\)py-config-discovery(/|\\\\)lit.site.cfg.py}}'84# CHECK-PYCONFIG-ERR-DAG: loading suite config '{{.*(/|\\\\)discovery(/|\\\\)subsuite(/|\\\\)lit.cfg}}'85# CHECK-PYCONFIG-ERR-DAG: loading local config '{{.*(/|\\\\)discovery(/|\\\\)subdir(/|\\\\)lit.local.cfg}}'86#87# CHECK-PYCONFIG-OUT: -- Test Suites --88# CHECK-PYCONFIG-OUT: sub-suite - 2 tests89# CHECK-PYCONFIG-OUT: Source Root: {{.*[/\\]discovery[/\\]subsuite$}}90# CHECK-PYCONFIG-OUT: Exec Root : {{.*[/\\]discovery[/\\]subsuite$}}91# CHECK-PYCONFIG-OUT: top-level-suite - 3 tests92# CHECK-PYCONFIG-OUT: Source Root: {{.*[/\\]discovery$}}93# CHECK-PYCONFIG-OUT: Exec Root : {{.*[/\\]py-config-discovery$}}94#95# CHECK-PYCONFIG-OUT: -- Available Tests --96# CHECK-PYCONFIG-OUT: sub-suite :: test-one97# CHECK-PYCONFIG-OUT: sub-suite :: test-two98# CHECK-PYCONFIG-OUT: top-level-suite :: subdir/test-three99# CHECK-PYCONFIG-OUT: top-level-suite :: test-one100# CHECK-PYCONFIG-OUT: top-level-suite :: test-two101 102# Check discovery when using an exec path.103#104# RUN: %{lit} %{inputs}/exec-discovery \105# RUN: --debug --show-tests --show-suites \106# RUN: -v > %t.out 2> %t.err107# RUN: FileCheck --check-prefix=CHECK-ASEXEC-OUT < %t.out %s108# RUN: FileCheck --check-prefix=CHECK-ASEXEC-ERR < %t.err %s109#110# CHECK-ASEXEC-ERR: loading suite config '{{.*(/|\\\\)exec-discovery(/|\\\\)lit.site.cfg}}'111# CHECK-ASEXEC-ERR: load_config from '{{.*(/|\\\\)discovery(/|\\\\)lit.cfg}}'112# CHECK-ASEXEC-ERR: loaded config '{{.*(/|\\\\)discovery(/|\\\\)lit.cfg}}'113# CHECK-ASEXEC-ERR: loaded config '{{.*(/|\\\\)exec-discovery(/|\\\\)lit.site.cfg}}'114# CHECK-ASEXEC-ERR-DAG: loading suite config '{{.*(/|\\\\)discovery(/|\\\\)subsuite(/|\\\\)lit.cfg}}'115# CHECK-ASEXEC-ERR-DAG: loading local config '{{.*(/|\\\\)discovery(/|\\\\)subdir(/|\\\\)lit.local.cfg}}'116#117# CHECK-ASEXEC-OUT: -- Test Suites --118# CHECK-ASEXEC-OUT: sub-suite - 2 tests119# CHECK-ASEXEC-OUT: Source Root: {{.*[/\\]discovery[/\\]subsuite$}}120# CHECK-ASEXEC-OUT: Exec Root : {{.*[/\\]discovery[/\\]subsuite$}}121# CHECK-ASEXEC-OUT: top-level-suite - 3 tests122# CHECK-ASEXEC-OUT: Source Root: {{.*[/\\]discovery$}}123# CHECK-ASEXEC-OUT: Exec Root : {{.*[/\\]exec-discovery$}}124#125# CHECK-ASEXEC-OUT: -- Available Tests --126# CHECK-ASEXEC-OUT: sub-suite :: test-one127# CHECK-ASEXEC-OUT: sub-suite :: test-two128# CHECK-ASEXEC-OUT: top-level-suite :: subdir/test-three129# CHECK-ASEXEC-OUT: top-level-suite :: test-one130# CHECK-ASEXEC-OUT: top-level-suite :: test-two131 132# Check discovery when tests are named directly.133#134# FIXME: Note that using a path into a subsuite doesn't work correctly here.135#136# RUN: %{lit} \137# RUN: %{inputs}/exec-discovery/subdir/test-three.py \138# RUN: --show-tests --show-suites -v > %t.out139# RUN: FileCheck --check-prefix=CHECK-ASEXEC-DIRECT-TEST < %t.out %s140#141# CHECK-ASEXEC-DIRECT-TEST: -- Available Tests --142# CHECK-ASEXEC-DIRECT-TEST: top-level-suite :: subdir/test-three143 144# Check that an error is emitted when the directly named test does not satisfy145# the test config's requirements.146#147# RUN: not %{lit} \148# RUN: %{inputs}/discovery/test.not-txt 2>%t.err149# RUN: FileCheck --check-prefix=CHECK-ERROR-INPUT-CONTAINED-NO-TESTS < %t.err %s150#151# CHECK-ERROR-INPUT-CONTAINED-NO-TESTS: warning: input 'Inputs/discovery/test.not-txt' contained no tests152# CHECK-ERROR-INPUT-CONTAINED-NO-TESTS: error: did not discover any tests for provided path(s)153 154# Check that a standalone test with no suffixes set is run without any errors.155#156# RUN: %{lit} %{inputs}/standalone-tests/true.txt > %t.out157# RUN: FileCheck --check-prefix=CHECK-STANDALONE < %t.out %s158#159# CHECK-STANDALONE: PASS: Standalone tests :: true.txt160 161# Check that an error is produced if suffixes variable is set for a suite with162# standalone tests.163#164# RUN: not %{lit} %{inputs}/standalone-tests-with-suffixes 2> %t.err165# RUN: FileCheck --check-prefixes=CHECK-STANDALONE-SUFFIXES,CHECK-STANDALONE-DISCOVERY < %t.err %s166#167# CHECK-STANDALONE-SUFFIXES: standalone_tests set {{.*}} but suffixes168 169# Check that an error is produced if excludes variable is set for a suite with170# standalone tests.171#172# RUN: not %{lit} %{inputs}/standalone-tests-with-excludes 2> %t.err173# RUN: FileCheck --check-prefixes=CHECK-STANDALONE-EXCLUDES,CHECK-STANDALONE-DISCOVERY < %t.err %s174#175# CHECK-STANDALONE-EXCLUDES: standalone_tests set {{.*}} but {{.*}} excludes176 177# Check that no discovery is done for testsuite with standalone tests.178#179# RUN: not %{lit} %{inputs}/standalone-tests 2>%t.err180# RUN: FileCheck --check-prefix=CHECK-STANDALONE-DISCOVERY < %t.err %s181#182# CHECK-STANDALONE-DISCOVERY: error: did not discover any tests for provided path(s)183 184# Check that a single file path can result in multiple tests being discovered if185# the test format implements those semantics.186#187# RUN: %{lit} %{inputs}/discovery-getTestsForPath/x.test > %t.out188# RUN: FileCheck --check-prefix=CHECK-getTestsForPath < %t.out %s189#190# CHECK-getTestsForPath: PASS: discovery-getTestsForPath-suite :: {{.+}}one.test191# CHECK-getTestsForPath: PASS: discovery-getTestsForPath-suite :: {{.+}}two.test192 193# Check that we don't recurse infinitely when loading an site specific test194# suite located inside the test source root.195#196# RUN: %{lit} \197# RUN: %{inputs}/exec-discovery-in-tree/obj/ \198# RUN: --show-tests --show-suites -v > %t.out199# RUN: FileCheck --check-prefix=CHECK-ASEXEC-INTREE < %t.out %s200#201# Try it again after cd'ing into the test suite using a short relative path.202#203# RUN: cd %{inputs}/exec-discovery-in-tree/obj/204# RUN: %{lit} . \205# RUN: --show-tests --show-suites -v > %t.out206# RUN: FileCheck --check-prefix=CHECK-ASEXEC-INTREE < %t.out %s207#208# CHECK-ASEXEC-INTREE: exec-discovery-in-tree-suite - 1 tests209# CHECK-ASEXEC-INTREE-NEXT: Source Root: {{.*[/\\]exec-discovery-in-tree$}}210# CHECK-ASEXEC-INTREE-NEXT: Exec Root : {{.*[/\\]exec-discovery-in-tree[/\\]obj$}}211# CHECK-ASEXEC-INTREE-NEXT: Available Features:212# CHECK-ASEXEC-INTREE-NEXT: Available Substitutions:213# CHECK-ASEXEC-INTREE-NEXT: -- Available Tests --214# CHECK-ASEXEC-INTREE-NEXT: exec-discovery-in-tree-suite :: test-one215