[commit: ghc] master: testsuite: Classify missing expected perf numbers as merely warnings (597ea1c)

[git at git.haskell.org]

Repository : ssh://git@git.haskell.org/ghc

On branch  : master
Link       : http://ghc.haskell.org/trac/ghc/changeset/597ea1cdca4c127a0a9ad936645c416df80a907b/ghc

>---------------------------------------------------------------

commit 597ea1cdca4c127a0a9ad936645c416df80a907b
Author: Ben Gamari <ben at smart-cactus.org>
Date:   Sun Apr 2 17:20:34 2017 -0400

    testsuite: Classify missing expected perf numbers as merely warnings
    
    Previously these were considered to be framework failures, meaning that validate
    would fail. For better or worse, Windows lacks a good number of metrics and I
    don't see this changing any time soon. Let's consider these to be non-fatal.


>---------------------------------------------------------------

597ea1cdca4c127a0a9ad936645c416df80a907b
 testsuite/driver/testglobals.py |  1 +
 testsuite/driver/testlib.py     | 17 +++++++++++++++--
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/testsuite/driver/testglobals.py b/testsuite/driver/testglobals.py
index 9f37e1a..fc050e6 100644
--- a/testsuite/driver/testglobals.py
+++ b/testsuite/driver/testglobals.py
@@ -138,6 +138,7 @@ class TestRun:
 
        self.missing_libs = []
        self.framework_failures = []
+       self.framework_warnings = []
 
        self.unexpected_passes = []
        self.unexpected_failures = []
diff --git a/testsuite/driver/testlib.py b/testsuite/driver/testlib.py
index a5da1e9..457e380 100644
--- a/testsuite/driver/testlib.py
+++ b/testsuite/driver/testlib.py
@@ -324,7 +324,7 @@ def _stats_num_field( name, opts, field, expecteds ):
             if b:
                 opts.stats_range_fields[field] = (expected, dev)
                 return
-        framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
+        framework_warn(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
 
     else:
         (expected, dev) = expecteds
@@ -347,7 +347,7 @@ def _compiler_stats_num_field( name, opts, field, expecteds ):
             opts.compiler_stats_range_fields[field] = (expected, dev)
             return
 
-    framework_fail(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
+    framework_warn(name, 'numfield-no-expected', 'No expected value found for ' + field + ' in num_field check')
 
 # -----
 
@@ -893,6 +893,13 @@ def framework_fail(name, way, reason):
     if_verbose(1, '*** framework failure for %s %s ' % (full_name, reason))
     t.framework_failures.append((directory, name, way, reason))
 
+def framework_warn(name, way, reason):
+    opts = getTestOpts()
+    directory = re.sub('^\\.[/\\\\]', '', opts.testdir)
+    full_name = name + '(' + way + ')'
+    if_verbose(1, '*** framework warning for %s %s ' % (full_name, reason))
+    t.framework_warnings.append((directory, name, way, reason))
+
 def badResult(result):
     try:
         if result['passFail'] == 'pass':
@@ -1990,6 +1997,8 @@ def summary(t, file, short=False):
                + '\n'
                + repr(len(t.framework_failures)).rjust(8)
                + ' caused framework failures\n'
+               + repr(len(t.framework_warnings)).rjust(8)
+               + ' caused framework warnings\n'
                + repr(len(t.unexpected_passes)).rjust(8)
                + ' unexpected passes\n'
                + repr(len(t.unexpected_failures)).rjust(8)
@@ -2014,6 +2023,10 @@ def summary(t, file, short=False):
         file.write('Framework failures:\n')
         printTestInfosSummary(file, t.framework_failures)
 
+    if t.framework_warnings:
+        file.write('Framework warnings:\n')
+        printTestInfosSummary(file, t.framework_warnings)
+
     if stopping():
         file.write('WARNING: Testsuite run was terminated early\n')