125 lines · python
1# Check the behavior of the ALLOW_RETRIES keyword.2 3# This test uses a file that's stable across retries of the test to fail and4# only succeed the fourth time it is retried.5#6# RUN: rm -f %t.counter7# RUN: %{lit} %{inputs}/allow-retries/succeeds-within-limit.py -Dcounter=%t.counter -Dpython=%{python} | FileCheck --check-prefix=CHECK-TEST1 %s8# CHECK-TEST1: Passed With Retry: 19 10# Test that a per-file ALLOW_RETRIES overwrites the config-wide test_retry_attempts property, if any.11#12# RUN: rm -f %t.counter13# RUN: %{lit} %{inputs}/allow-retries/succeeds-within-limit.py -Dtest_retry_attempts=2 -Dcounter=%t.counter -Dpython=%{python} | FileCheck --check-prefix=CHECK-TEST2 %s14# CHECK-TEST2: Passed With Retry: 115 16# This test does not succeed within the allowed retry limit17#18# Check that the execution trace isn't corrupt due to reprocessing the script19# multiple times (e.g., '%dbg(...)' processing used to accumulate across20# retries).21#22# RUN: not %{lit} %{inputs}/allow-retries/does-not-succeed-within-limit.py -v |\23# RUN: FileCheck --check-prefix=CHECK-TEST3 -match-full-lines %s24#25# CHECK-TEST3: FAIL: allow-retries :: does-not-succeed-within-limit.py (1 of 1, 4 of 4 attempts)26# CHECK-TEST3-NEXT: {{\**}} TEST 'allow-retries :: does-not-succeed-within-limit.py' FAILED {{\**}}27# CHECK-TEST3-NEXT: Exit Code: 128# CHECK-TEST3-EMPTY:29# CHECK-TEST3-NEXT: Command Output (stdout):30# CHECK-TEST3-NEXT: --31# CHECK-TEST3-NEXT: # {{RUN}}: at line 332# CHECK-TEST3-NEXT: false33# CHECK-TEST3-NEXT: # executed command: false34# CHECK-TEST3-NEXT: # note: command had no output on stdout or stderr35# CHECK-TEST3-NEXT: # error: command failed with exit status: 136# CHECK-TEST3-EMPTY:37# CHECK-TEST3-NEXT: --38# CHECK-TEST3: Failed Tests (1):39# CHECK-TEST3: allow-retries :: does-not-succeed-within-limit.py40 41# This test should be UNRESOLVED since it has more than one ALLOW_RETRIES42# lines, and that is not allowed.43#44# RUN: not %{lit} %{inputs}/allow-retries/more-than-one-allow-retries-lines.py | FileCheck --check-prefix=CHECK-TEST4 %s45# CHECK-TEST4: Unresolved Tests (1):46# CHECK-TEST4: allow-retries :: more-than-one-allow-retries-lines.py47 48# This test does not provide a valid integer to the ALLOW_RETRIES keyword.49# It should be unresolved.50#51# RUN: not %{lit} %{inputs}/allow-retries/not-a-valid-integer.py | FileCheck --check-prefix=CHECK-TEST5 %s52# CHECK-TEST5: Unresolved Tests (1):53# CHECK-TEST5: allow-retries :: not-a-valid-integer.py54 55# This test checks that the config-wide test_retry_attempts property is used56# when no ALLOW_RETRIES keyword is present.57#58# RUN: rm -f %t.counter59# RUN: %{lit} %{inputs}/test_retry_attempts/test.py -Dcounter=%t.counter -Dpython=%{python} | FileCheck --check-prefix=CHECK-TEST6 %s60# CHECK-TEST6: Passed With Retry: 161 62# This test checks that --per-test-coverage doesn't accumulate inserted63# LLVM_PROFILE_FILE= commands across retries.64#65# RUN: rm -f %t.counter66# RUN: %{lit} -a %{inputs}/test_retry_attempts/test.py --per-test-coverage\67# RUN: -Dcounter=%t.counter -Dpython=%{python} | \68# RUN: FileCheck --check-prefix=CHECK-TEST7 %s69# CHECK-TEST7: Command Output (stdout):70# CHECK-TEST7: # executed command: export LLVM_PROFILE_FILE=71# CHECK-TEST7-NOT: # executed command: export LLVM_PROFILE_FILE=72# CHECK-TEST7: Passed With Retry: 173 74# This test only passes on the 4th try. Here we check that a test can be re-run when:75# * The "--max-retries-per-test" is specified high enough (7).76# * No ALLOW_RETRIES keyword is used in the test script.77# * No config.test_retry_attempts is adjusted in the test suite config file.78# RUN: rm -f %t.counter79# RUN: %{lit} %{inputs}/max-retries-per-test/no-allow-retries-no-test_retry_attempts/test.py \80# RUN: --max-retries-per-test=7 \81# RUN: -Dcounter=%t.counter \82# RUN: -Dpython=%{python} \83# RUN: | FileCheck --check-prefix=CHECK-TEST8 %s84# CHECK-TEST8: FLAKYPASS: no-allow-retries-no-test_retry_attempts :: test.py (1 of 1, 4 of 8 attempts)85# CHECK-TEST8: Passed With Retry: 186 87# This test only passes on the 4th try. Here we check that a test can be re-run when:88# * The "--max-retries-per-test" is specified too low (2).89# * ALLOW_RETRIES is specified high enough (8)90# * No config.test_retry_attempts is adjusted in the test suite config file.91# RUN: rm -f %t.counter92# RUN: %{lit} %{inputs}/max-retries-per-test/allow-retries-no-test_retry_attempts/test.py \93# RUN: --max-retries-per-test=2 \94# RUN: -Dcounter=%t.counter \95# RUN: -Dpython=%{python} \96# RUN: | FileCheck --check-prefix=CHECK-TEST9 %s97# CHECK-TEST9: FLAKYPASS: allow-retries-no-test_retry_attempts :: test.py (1 of 1, 4 of 9 attempts)98# CHECK-TEST9: Passed With Retry: 199 100# This test only passes on the 4th try. Here we check that a test can be re-run when:101# * The "--max-retries-per-test" is specified too low (2).102# * No ALLOW_RETRIES keyword is used in the test script.103# * config.test_retry_attempts is set high enough (9).104# RUN: rm -f %t.counter105# RUN: %{lit} %{inputs}/max-retries-per-test/no-allow-retries-test_retry_attempts/test.py \106# RUN: --max-retries-per-test=2 \107# RUN: -Dcounter=%t.counter \108# RUN: -Dpython=%{python} \109# RUN: | FileCheck --check-prefix=CHECK-TEST10 %s110# CHECK-TEST10: FLAKYPASS: no-allow-retries-test_retry_attempts :: test.py (1 of 1, 4 of 10 attempts)111# CHECK-TEST10: Passed With Retry: 1112 113# This test only passes on the 4th try. Here we check that a test can be re-run when:114# * The "--max-retries-per-test" is specified too low (1).115# * ALLOW_RETRIES keyword is set high enough (10)116# * config.test_retry_attempts is set too low (2).117# RUN: rm -f %t.counter118# RUN: %{lit} %{inputs}/max-retries-per-test/allow-retries-test_retry_attempts/test.py \119# RUN: --max-retries-per-test=1 \120# RUN: -Dcounter=%t.counter \121# RUN: -Dpython=%{python} \122# RUN: | FileCheck --check-prefix=CHECK-TEST11 %s123# CHECK-TEST11: FLAKYPASS: allow-retries-test_retry_attempts :: test.py (1 of 1, 4 of 11 attempts)124# CHECK-TEST11: Passed With Retry: 1125