[commit: ghc] master: Testsuite: report and error out on unfound tests (032be43)

[git at git.haskell.org]

Repository : ssh://git@git.haskell.org/ghc

On branch  : master
Link       : http://ghc.haskell.org/trac/ghc/changeset/032be43b66db29ca6893bd4219c0d6036201a94e/ghc

>---------------------------------------------------------------

commit 032be43b66db29ca6893bd4219c0d6036201a94e
Author: Thomas Miedema <thomasmiedema at gmail.com>
Date:   Wed Oct 28 12:28:59 2015 +0100

    Testsuite: report and error out on unfound tests
    
    Users are sometimes confused why their test doesn't run. It is usually
    because of a misspelled testname, for example using 'TEST=1234' instead
    of 'TEST=T1234'. After this patch it is hopefully more clear what the
    problem is, showing:
    
        ERROR: tests not found: ['1234']
    
    Instead of:
    
        0 total tests, which gave rise to
        0 test cases, of which
        0 were skipped
    
    Reviewed by: austin, bgamari
    
    Differential Revision: https://phabricator.haskell.org/D1388


>---------------------------------------------------------------

032be43b66db29ca6893bd4219c0d6036201a94e
 testsuite/driver/runtests.py    |  8 +++++++-
 testsuite/driver/testglobals.py |  3 ++-
 testsuite/driver/testlib.py     | 12 ++++++++++--
 3 files changed, 19 insertions(+), 4 deletions(-)

diff --git a/testsuite/driver/runtests.py b/testsuite/driver/runtests.py
index 491335d..0f751f2 100644
--- a/testsuite/driver/runtests.py
+++ b/testsuite/driver/runtests.py
@@ -79,7 +79,8 @@ for opt,arg in opts:
         config.no_print_summary = True
 
     if opt == '--only':
-        config.only.append(arg)
+        config.run_only_some_tests = True
+        config.only.add(arg)
 
     if opt == '--way':
         if (arg not in config.run_ways and arg not in config.compile_ways and arg not in config.other_ways):
@@ -287,6 +288,11 @@ for file in t_files:
         t.n_framework_failures = t.n_framework_failures + 1
         traceback.print_exc()
 
+if config.only:
+    # See Note [Mutating config.only]
+    sys.stderr.write("ERROR: tests not found: {0}\n".format(list(config.only)))
+    sys.exit(1)
+
 if config.list_broken:
     global brokens
     print('')
diff --git a/testsuite/driver/testglobals.py b/testsuite/driver/testglobals.py
index 95168f3..0891624 100644
--- a/testsuite/driver/testglobals.py
+++ b/testsuite/driver/testglobals.py
@@ -23,7 +23,8 @@ class TestConfig:
         self.rootdirs = []
 
         # Run these tests only (run all tests if empty)
-        self.only = []
+        self.run_only_some_tests = False
+        self.only = set()
 
         # Accept new output which differs from the sample?
         self.accept = 0
diff --git a/testsuite/driver/testlib.py b/testsuite/driver/testlib.py
index 3ca5d02..45e8d3e 100644
--- a/testsuite/driver/testlib.py
+++ b/testsuite/driver/testlib.py
@@ -591,8 +591,16 @@ def runTest (opts, name, func, args):
 # name  :: String
 # setup :: TestOpts -> IO ()
 def test (name, setup, func, args):
-    if config.only and name not in config.only:
-        return
+    if config.run_only_some_tests:
+        if name not in config.only:
+            return
+        else:
+            # Note [Mutating config.only]
+            # config.only is initiallly the set of tests requested by
+            # the user (via 'make TEST='). We then remove all tests that
+            # we've already seen (in .T files), so that we can later
+            # report on any tests we couldn't find and error out.
+            config.only.remove(name)
 
     global aloneTests
     global parallelTests