[commit: ghc] master: Testsuite: framework failure improvements (#11165) (782cacf)

[git at git.haskell.org]

Repository : ssh://git@git.haskell.org/ghc

On branch  : master
Link       : http://ghc.haskell.org/trac/ghc/changeset/782cacf57300908d8a608bac7d26be59586f2af0/ghc

>---------------------------------------------------------------

commit 782cacf57300908d8a608bac7d26be59586f2af0
Author: Thomas Miedema <thomasmiedema at gmail.com>
Date:   Fri Jun 24 16:18:05 2016 +0200

    Testsuite: framework failure improvements (#11165)
    
    * add framework failures to unexpected results list
    * report errors in .T files as framework failures (show in summary)
    * don't report missing tests when framework failures in .T files


>---------------------------------------------------------------

782cacf57300908d8a608bac7d26be59586f2af0
 testsuite/driver/runtests.py | 20 +++++++++++++-------
 testsuite/driver/testlib.py  |  7 +++++--
 2 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/testsuite/driver/runtests.py b/testsuite/driver/runtests.py
index efd8b09..8a11f44 100644
--- a/testsuite/driver/runtests.py
+++ b/testsuite/driver/runtests.py
@@ -298,20 +298,26 @@ def cleanup_and_exit(exitcode):
     exit(exitcode)
 
 # First collect all the tests to be run
+t_files_ok = True
 for file in t_files:
     if_verbose(2, '====> Scanning %s' % file)
     newTestDir(tempdir, os.path.dirname(file))
     try:
         exec(open(file).read())
-    except Exception:
-        print('*** framework failure: found an error while executing ', file, ':')
-        t.n_framework_failures = t.n_framework_failures + 1
+    except Exception as e:
         traceback.print_exc()
+        framework_fail(file, '', str(e))
+        t_files_ok = False
 
-if config.only:
-    # See Note [Mutating config.only]
-    sys.stderr.write("ERROR: tests not found: {0}\n".format(list(config.only)))
-    cleanup_and_exit(1)
+for name in config.only:
+    if t_files_ok:
+        # See Note [Mutating config.only]
+        framework_fail(name, '', 'test not found')
+    else:
+        # Let user fix .T file errors before reporting on unfound tests.
+        # The reson the test can not be found is likely because of those
+        # .T file errors.
+        pass
 
 if config.list_broken:
     global brokens
diff --git a/testsuite/driver/testlib.py b/testsuite/driver/testlib.py
index e9548c7..6d4d77c 100644
--- a/testsuite/driver/testlib.py
+++ b/testsuite/driver/testlib.py
@@ -1841,7 +1841,9 @@ def findTFiles(roots):
 def summary(t, file, short=False):
 
     file.write('\n')
-    printUnexpectedTests(file, [t.unexpected_passes, t.unexpected_failures, t.unexpected_stat_failures])
+    printUnexpectedTests(file,
+        [t.unexpected_passes, t.unexpected_failures,
+         t.unexpected_stat_failures, t.framework_failures])
 
     if short:
         # Only print the list of unexpected tests above.
@@ -1897,7 +1899,8 @@ def summary(t, file, short=False):
 
 def printUnexpectedTests(file, testInfoss):
     unexpected = {name for testInfos in testInfoss
-                       for (_, name, _, _) in testInfos}
+                       for (_, name, _, _) in testInfos
+                       if not name.endswith('.T')}
     if unexpected:
         file.write('Unexpected results from:\n')
         file.write('TEST="' + ' '.join(unexpected) + '"\n')