[commit: ghc] wip/T13633, wip/non-det-ci: Exit with exit code 1 when tests unexpectedly pass (a990312)

[git at git.haskell.org]

Repository : ssh://git@git.haskell.org/ghc

On branches: wip/T13633,wip/non-det-ci
Link       : http://ghc.haskell.org/trac/ghc/changeset/a990312e3d36e0e1b18bb966ac3297aa0aae142a/ghc

>---------------------------------------------------------------

commit a990312e3d36e0e1b18bb966ac3297aa0aae142a
Author: Matthew Pickering <matthewtpickering at gmail.com>
Date:   Tue Feb 19 20:23:48 2019 +0000

    Exit with exit code 1 when tests unexpectedly pass
    
    This was causing gitlab to not report from builds as failing. It also
    highlighted a problem with the LLVM tests where some of the external
    interpreter tests are failing.


>---------------------------------------------------------------

a990312e3d36e0e1b18bb966ac3297aa0aae142a
 testsuite/driver/runtests.py |  1 +
 testsuite/driver/testlib.py  |  1 +
 testsuite/tests/th/all.T     | 10 ++++++++--
 3 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/testsuite/driver/runtests.py b/testsuite/driver/runtests.py
index 51f682e..b439abd 100644
--- a/testsuite/driver/runtests.py
+++ b/testsuite/driver/runtests.py
@@ -439,6 +439,7 @@ else:
 
 if len(t.unexpected_failures) > 0 or \
    len(t.unexpected_stat_failures) > 0 or \
+   len(t.unexpected_passes) > 0 or \
    len(t.framework_failures) > 0:
     exitcode = 1
 else:
diff --git a/testsuite/driver/testlib.py b/testsuite/driver/testlib.py
index fb7888c..11c828c 100644
--- a/testsuite/driver/testlib.py
+++ b/testsuite/driver/testlib.py
@@ -2118,6 +2118,7 @@ def summary(t, file, short=False, color=False):
     if color:
         if len(t.unexpected_failures) > 0 or \
             len(t.unexpected_stat_failures) > 0 or \
+            len(t.unexpected_passes) > 0 or \
             len(t.framework_failures) > 0:
             colorize = str_fail
         else:
diff --git a/testsuite/tests/th/all.T b/testsuite/tests/th/all.T
index 2aaa48c..7d6340b 100644
--- a/testsuite/tests/th/all.T
+++ b/testsuite/tests/th/all.T
@@ -12,8 +12,14 @@ setTestOpts(req_interp)
 if config.have_ext_interp :
        setTestOpts(extra_ways(['ext-interp']))
        setTestOpts(only_ways(['normal','ghci','ext-interp']))
-       if config.ghc_built_by_llvm:
-           setTestOpts(expect_broken_for(16087, ['ext-interp']))
+
+broken_tests = ["ClosedFam1TH","T10620","T10828","T11721_TH","T11797","T12045TH2","T12478_1","T12646","T13642","T14060","T15502","T15738","T15792","T15845","T16180","T1835","T3920","T4135","T4188","T5037","T5362","T7477","T7910","T8761","T8884","T8953","T9262","T9692","T9738","TH_Lift","TH_RichKinds","TH_RichKinds2","TH_Roles3","TH_TyInstWhere2","TH_implicitParams","TH_recursiveDo","TH_reifyDecl1","TH_reifyExplicitForAllFams","TH_reifyInstances","TH_reifyMkName","TH_repE2","TH_repGuard","TH_repPrim","TH_repPrim2","TH_repUnboxedTuples","TH_spliceE6"]
+# ext-interp, integer-gmp and llvm is broken see #16087
+def broken_ext_interp(name, opts):
+	if name in broken_tests and config.ghc_built_by_llvm:
+		expect_broken_for(16087, ['ext-interp'])(name, opts)
+
+setTestOpts(broken_ext_interp)
 
 test('TH_mkName', normal, compile, ['-v0'])
 test('TH_overloadedlabels', normal, compile, ['-v0'])