[commit: ghc] master: Split off stat (benchmark) test failures into a separate section in the test runner summary. (4667fb5)

[git at git.haskell.org]

Repository : ssh://git@git.haskell.org/ghc

On branch  : master
Link       : http://ghc.haskell.org/trac/ghc/changeset/4667fb5201833e079256b7e4b420e5c87ca19b1a/ghc

>---------------------------------------------------------------

commit 4667fb5201833e079256b7e4b420e5c87ca19b1a
Author: Gintautas Miliauskas <gintautas.miliauskas at gmail.com>
Date:   Fri Oct 31 15:51:16 2014 -0500

    Split off stat (benchmark) test failures into a separate section in the test runner summary.
    
    Stat tests are generally less reliable than other types of tests, so it's nice to have
    them in a separate section rather than interspersed with potential...
    
    Summary: ...correctness issues.
    
    Reviewers: austin
    
    Reviewed By: austin
    
    Subscribers: thomie, carter, simonmar
    
    Differential Revision: https://phabricator.haskell.org/D406


>---------------------------------------------------------------

4667fb5201833e079256b7e4b420e5c87ca19b1a
 testsuite/driver/testglobals.py |  2 ++
 testsuite/driver/testlib.py     | 28 ++++++++++++++++++++--------
 2 files changed, 22 insertions(+), 8 deletions(-)

diff --git a/testsuite/driver/testglobals.py b/testsuite/driver/testglobals.py
index 7b9bd9a..643dad6 100644
--- a/testsuite/driver/testglobals.py
+++ b/testsuite/driver/testglobals.py
@@ -143,6 +143,8 @@ class TestRun:
        self.unexpected_passes = {}
        self.n_unexpected_failures = 0
        self.unexpected_failures = {}
+       self.n_unexpected_stat_failures = 0
+       self.unexpected_stat_failures = {}
        
 global t
 t = TestRun()
diff --git a/testsuite/driver/testlib.py b/testsuite/driver/testlib.py
index 87e37d5..1549381 100644
--- a/testsuite/driver/testlib.py
+++ b/testsuite/driver/testlib.py
@@ -820,10 +820,16 @@ def do_test(name, way, func, args):
         elif passFail == 'fail':
             if getTestOpts().expect == 'pass' \
                and way not in getTestOpts().expect_fail_for:
-                if_verbose(1, '*** unexpected failure for %s' % full_name)
-                t.n_unexpected_failures = t.n_unexpected_failures + 1
                 reason = result['reason']
-                addFailingTestInfo(t.unexpected_failures, getTestOpts().testdir, name, reason, way)
+                tag = result.get('tag')
+                if tag == 'stat':
+                    if_verbose(1, '*** unexpected stat test failure for %s' % full_name)
+                    t.n_unexpected_stat_failures = t.n_unexpected_stat_failures + 1
+                    addFailingTestInfo(t.unexpected_stat_failures, getTestOpts().testdir, name, reason, way)
+                else:
+                    if_verbose(1, '*** unexpected failure for %s' % full_name)
+                    t.n_unexpected_failures = t.n_unexpected_failures + 1
+                    addFailingTestInfo(t.unexpected_failures, getTestOpts().testdir, name, reason, way)
             else:
                 if getTestOpts().expect == 'missing-lib':
                     t.n_missing_libs = t.n_missing_libs + 1
@@ -898,8 +904,8 @@ def badResult(result):
 def passed():
     return {'passFail': 'pass'}
 
-def failBecause(reason):
-    return {'passFail': 'fail', 'reason': reason}
+def failBecause(reason, tag=None):
+    return {'passFail': 'fail', 'reason': reason, 'tag': tag}
 
 # -----------------------------------------------------------------------------
 # Generic command tests
@@ -1138,10 +1144,10 @@ def checkStats(name, way, stats_file, range_fields):
                 print(field, 'value is too low:')
                 print('(If this is because you have improved GHC, please')
                 print('update the test so that GHC doesn\'t regress again)')
-                result = failBecause('stat too good')
+                result = failBecause('stat too good', tag='stat')
             if val > upperBound:
                 print(field, 'value is too high:')
-                result = failBecause('stat not good enough')
+                result = failBecause('stat not good enough', tag='stat')
 
             if val < lowerBound or val > upperBound or config.verbose >= 4:
                 valStr = str(val)
@@ -2146,7 +2152,7 @@ def findTFiles_(path):
 def summary(t, file):
 
     file.write('\n')
-    printUnexpectedTests(file, [t.unexpected_passes, t.unexpected_failures])
+    printUnexpectedTests(file, [t.unexpected_passes, t.unexpected_failures, t.unexpected_stat_failures])
     file.write('OVERALL SUMMARY for test run started at '
                + time.strftime("%c %Z", t.start_time) + '\n'
                + str(datetime.timedelta(seconds=
@@ -2172,6 +2178,8 @@ def summary(t, file):
                + ' unexpected passes\n'
                + repr(t.n_unexpected_failures).rjust(8)
                + ' unexpected failures\n'
+               + repr(t.n_unexpected_stat_failures).rjust(8)
+               + ' unexpected stat failures\n'
                + '\n')
 
     if t.n_unexpected_passes > 0:
@@ -2182,6 +2190,10 @@ def summary(t, file):
         file.write('Unexpected failures:\n')
         printFailingTestInfosSummary(file, t.unexpected_failures)
 
+    if t.n_unexpected_stat_failures > 0:
+        file.write('Unexpected stat failures:\n')
+        printFailingTestInfosSummary(file, t.unexpected_stat_failures)
+
     if config.check_files_written:
         checkForFilesWrittenProblems(file)