[Git][ghc/ghc][wip/junit-fixes] testsuite: Better failure reporting in JUnit output

[Ben Gamari]

Ben Gamari pushed to branch wip/junit-fixes at Glasgow Haskell Compiler / GHC


Commits:
01acfeee by Ben Gamari at 2019-06-20T03:07:16Z
testsuite: Better failure reporting in JUnit output

* Fix a typo in test name field
* Report more stderr output
* Report stdout output

- - - - -


3 changed files:

- testsuite/driver/junit.py
- testsuite/driver/testglobals.py
- testsuite/driver/testlib.py


Changes:

=====================================
testsuite/driver/junit.py
=====================================
@@ -15,15 +15,21 @@ def junit(t):
 
     for res_type, group in [('stat failure', t.unexpected_stat_failures),
                             ('unexpected failure', t.unexpected_failures),
-                            ('unexpected_passes', t.unexpected_passes)]:
+                            ('unexpected pass', t.unexpected_passes)]:
         for tr in group:
             testcase = ET.SubElement(testsuite, 'testcase',
                                      classname = tr.way,
-                                     name = '%s(%sb)' % (tr.testname, tr.way))
-            new_reason = "\n".join([tr.reason, "STDERR:", tr.stderr.decode("utf-8")]) if tr.stderr else tr.reason
+                                     name = '%s(%s)' % (tr.testname, tr.way))
+            message = []
+            if tr.stdout:
+                message += ['', 'stdout:', '==========', tr.stdout]
+            if tr.stderr:
+                message += ['', 'stderr:', '==========', tr.stderr]
+
             result = ET.SubElement(testcase, 'failure',
                                    type = res_type,
-                                   message = new_reason)
+                                   message = tr.reason)
+            result.text = '\n'.join(message)
 
     for tr in t.framework_failures:
         testcase = ET.SubElement(testsuite, 'testcase',


=====================================
testsuite/driver/testglobals.py
=====================================
@@ -165,12 +165,13 @@ class TestResult:
     framework_failures, framework_warnings, unexpected_passes,
     unexpected_failures, unexpected_stat_failures lists of TestRun.
     """
-    __slots__ = 'directory', 'testname', 'reason', 'way', 'stderr'
-    def __init__(self, directory, testname, reason, way, stderr=None):
+    __slots__ = 'directory', 'testname', 'reason', 'way', 'stdout', 'stderr'
+    def __init__(self, directory, testname, reason, way, stdout=None, stderr=None):
         self.directory = directory
         self.testname = testname
         self.reason = reason
         self.way = way
+        self.stdout = stdout
         self.stderr = stderr
 
 class TestRun:


=====================================
testsuite/driver/testlib.py
=====================================
@@ -994,7 +994,9 @@ def do_test(name, way, func, args, files):
                 t.unexpected_stat_failures.append(TestResult(directory, name, reason, way))
             else:
                 if_verbose(1, '*** unexpected failure for %s' % full_name)
-                result = TestResult(directory, name, reason, way, stderr=result.get('stderr'))
+                result = TestResult(directory, name, reason, way,
+                                    stdout=result.get('stdout'),
+                                    stderr=result.get('stderr'))
                 t.unexpected_failures.append(result)
         else:
             if opts.expect == 'missing-lib':
@@ -1444,9 +1446,13 @@ def simple_run(name, way, prog, extra_run_opts):
         return failBecause('bad exit code (%d)' % exit_code)
 
     if not (opts.ignore_stderr or stderr_ok(name, way) or opts.combined_output):
-        return failBecause('bad stderr')
+        return failBecause('bad stderr',
+                           stderr=read_stderr(name),
+                           stdout=read_stdout(name))
     if not (opts.ignore_stdout or stdout_ok(name, way)):
-        return failBecause('bad stdout')
+        return failBecause('bad stdout',
+                           stderr=read_stderr(name),
+                           stdout=read_stdout(name))
 
     check_hp = '-h' in my_rts_flags and opts.check_hp
     check_prof = '-p' in my_rts_flags
@@ -1528,14 +1534,20 @@ def interpreter_run(name, way, extra_hc_opts, top_mod):
         print('Wrong exit code for ' + name + '(' + way + ') (expected', getTestOpts().exit_code, ', actual', exit_code, ')')
         dump_stdout(name)
         dump_stderr(name)
-        return failBecause('bad exit code (%d)' % exit_code)
+        return failBecause('bad exit code (%d)' % exit_code,
+                           stderr=read_stderr(name),
+                           stdout=read_stdout(name))
 
     # ToDo: if the sub-shell was killed by ^C, then exit
 
     if not (opts.ignore_stderr or stderr_ok(name, way)):
-        return failBecause('bad stderr')
+        return failBecause('bad stderr',
+                           stderr=read_stderr(name),
+                           stdout=read_stdout(name))
     elif not (opts.ignore_stdout or stdout_ok(name, way)):
-        return failBecause('bad stdout')
+        return failBecause('bad stdout',
+                           stderr=read_stderr(name),
+                           stdout=read_stdout(name))
     else:
         return passed()
 
@@ -1582,12 +1594,15 @@ def stdout_ok(name, way):
    return compare_outputs(way, 'stdout', extra_norm,
                           expected_stdout_file, actual_stdout_file)
 
-def dump_stdout( name ):
+def read_stdout( name ):
     with open(in_testdir(name, 'run.stdout'), encoding='utf8') as f:
-        str = f.read().strip()
-        if str:
-            print("Stdout (", name, "):")
-            print(str)
+        return f.read()
+
+def dump_stdout( name ):
+    str = read_stdout(name).strip()
+    if str:
+        print("Stdout (", name, "):")
+        print(str)
 
 def stderr_ok(name, way):
    actual_stderr_file = add_suffix(name, 'run.stderr')
@@ -1598,12 +1613,15 @@ def stderr_ok(name, way):
                           expected_stderr_file, actual_stderr_file,
                           whitespace_normaliser=normalise_whitespace)
 
-def dump_stderr( name ):
+def read_stderr( name ):
     with open(in_testdir(name, 'run.stderr'), encoding='utf8') as f:
-        str = f.read().strip()
-        if str:
-            print("Stderr (", name, "):")
-            print(str)
+        return f.read()
+
+def dump_stderr( name ):
+    str = read_stderr(name).strip()
+    if str:
+        print("Stderr (", name, "):")
+        print(str)
 
 def read_no_crs(file):
     str = ''



View it on GitLab: https://gitlab.haskell.org/ghc/ghc/commit/01acfeeee5b20f8085a8cb3cde151a86f9f670fd

-- 
View it on GitLab: https://gitlab.haskell.org/ghc/ghc/commit/01acfeeee5b20f8085a8cb3cde151a86f9f670fd
You're receiving this email because of your account on gitlab.haskell.org.


-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mail.haskell.org/pipermail/ghc-commits/attachments/20190619/55c75dd9/attachment-0001.html>