[commit: ghc] wip/misc-testsuite: testsuite: Allow application of a global timeout multiplier (c91c9f2)

[git at git.haskell.org]

Repository : ssh://git@git.haskell.org/ghc

On branch  : wip/misc-testsuite
Link       : http://ghc.haskell.org/trac/ghc/changeset/c91c9f2870cc88c66b7cfd68206cace8d02d18ae/ghc

>---------------------------------------------------------------

commit c91c9f2870cc88c66b7cfd68206cace8d02d18ae
Author: Ben Gamari <ben at smart-cactus.org>
Date:   Wed Jan 9 11:47:37 2019 -0500

    testsuite: Allow application of a global timeout multiplier
    
    As noted in #16156, many tests fail on Windows with timeouts. I believe this is
    because we tend to underestimate the timeout on Windows, which seems to slow
    severely under load. Work around this by introducing a global timeout
    multiplier.
    
    Note that this doesn't yet add support for the multiplier in Hadrian.


>---------------------------------------------------------------

c91c9f2870cc88c66b7cfd68206cace8d02d18ae
 .gitlab-ci.yml               | 4 ++--
 testsuite/driver/runtests.py | 3 +++
 testsuite/mk/test.mk         | 4 ++++
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 67b2830..9742cb4 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -341,7 +341,7 @@ validate-x86_64-windows-hadrian:
     # FIXME: --no-lint due to #15950
     - bash -c "PATH=`pwd`/toolchain/bin:$PATH hadrian/build.cabal.sh -j`mk/detect-cpu-count.sh` --flavour=Quick --no-lint"
     - bash -c "PATH=`pwd`/toolchain/bin:$PATH hadrian/build.cabal.sh binary-dist"
-    - bash -c 'make V=0 test THREADS=`mk/detect-cpu-count.sh` JUNIT_FILE=../../junit.xml'
+    - bash -c 'make V=0 test THREADS=`mk/detect-cpu-count.sh` JUNIT_FILE=../../junit.xml TIMEOUT_MULT=2'
   cache:
     key: x86_64-windows
   tags:
@@ -362,7 +362,7 @@ validate-x86_64-windows:
       bash -c './configure --enable-tarballs-autodownload GHC=`pwd`/toolchain/bin/ghc HAPPY=`pwd`/toolchain/bin/happy ALEX=`pwd`/toolchain/bin/alex'
     - bash -c "echo include mk/flavours/quick.mk > mk/build.mk"
     - bash -c "PATH=`pwd`/toolchain/bin:$PATH make -j`mk/detect-cpu-count.sh`"
-    - bash -c 'make V=0 test THREADS=`mk/detect-cpu-count.sh` JUNIT_FILE=../../junit.xml'
+    - bash -c 'make V=0 test THREADS=`mk/detect-cpu-count.sh` JUNIT_FILE=../../junit.xml TIMEOUT_MULT=2'
   cache:
     key: x86_64-windows
   tags:
diff --git a/testsuite/driver/runtests.py b/testsuite/driver/runtests.py
index 55b13df..32d5460 100644
--- a/testsuite/driver/runtests.py
+++ b/testsuite/driver/runtests.py
@@ -58,6 +58,7 @@ parser.add_argument("--way", action="append", help="just this way")
 parser.add_argument("--skipway", action="append", help="skip this way")
 parser.add_argument("--threads", type=int, help="threads to run simultaneously")
 parser.add_argument("--verbose", type=int, choices=[0,1,2,3,4,5], help="verbose (Values 0 through 5 accepted)")
+parser.add_argument("--timeout-mult", type=float, default=1, help="apply a global multiplier to timeout values")
 parser.add_argument("--junit", type=argparse.FileType('wb'), help="output testsuite summary in JUnit format")
 parser.add_argument("--test-env", default='local', help="Override default chosen test-env.")
 perf_group.add_argument("--skip-perf-tests", action="store_true", help="skip performance tests")
@@ -244,6 +245,8 @@ testopts_local.x = TestOptions()
 if config.timeout == -1:
     config.timeout = int(read_no_crs(config.top + '/timeout/calibrate.out'))
 
+config.timeout *= args.timeout_mult
+
 print('Timeout is ' + str(config.timeout))
 print('Known ways: ' + ', '.join(config.other_ways))
 print('Run ways: ' + ', '.join(config.run_ways))
diff --git a/testsuite/mk/test.mk b/testsuite/mk/test.mk
index a517698..c9076aa 100644
--- a/testsuite/mk/test.mk
+++ b/testsuite/mk/test.mk
@@ -223,6 +223,10 @@ ifneq "$(TEST_ENV)" ""
 RUNTEST_OPTS += --test-env="$(TEST_ENV)"
 endif
 
+ifneq "$(TIMEOUT_MULT)" ""
+RUNTEST_OPTS += --timeout-mult="$(TIMEOUT_MULT)"
+endif
+
 ifeq "$(CLEANUP)" "0"
 RUNTEST_OPTS += -e config.cleanup=False
 else ifeq "$(CLEANUP)" "NO"