[commit: testsuite] master: Test driver: Implement different verbosities (36a70ba)
git at git.haskell.org
git at git.haskell.org
Fri Sep 20 15:18:04 CEST 2013
Repository : ssh://git@git.haskell.org/testsuite
On branch : master
Link : http://ghc.haskell.org/trac/ghc/changeset/36a70ba030b86fb5db923b60dde852287291dba2/testsuite
>---------------------------------------------------------------
commit 36a70ba030b86fb5db923b60dde852287291dba2
Author: Joachim Breitner <mail at joachim-breitner.de>
Date: Fri Sep 20 15:15:51 2013 +0200
Test driver: Implement different verbosities
Select verbosity with "make VERBOSE=n". Options so far:
n=0: No per-test output
n=1: Only failing test results
n=2: As above, plus progress information (names of all tests)
n=3: As aobve, plus commands called.
Default currently is n=3, although n=2 might be a nicer default.
>---------------------------------------------------------------
36a70ba030b86fb5db923b60dde852287291dba2
README.md | 3 +++
driver/runtests.py | 10 +++++++++-
driver/testglobals.py | 2 +-
driver/testlib.py | 38 ++++++++++++++++++++------------------
mk/test.mk | 4 ++++
5 files changed, 37 insertions(+), 20 deletions(-)
diff --git a/README.md b/README.md
index 9b52e30..7620809 100644
--- a/README.md
+++ b/README.md
@@ -13,6 +13,9 @@ Commands to run testsuite:
* Run a specific test: `make TEST=tc054`
* Test a specific 'way': `make WAY=optllvm`
* Test a specifc stage of GHC: `make stage=1`
+ * Set verbosity: `make VERBOSE=n`
+ where n=0: No per-test ouput, n=1: Only failures,
+ n=2: Progress output, n=3: Include commands called (default)
* Pass in extra GHC options: `make EXTRA_HC_OPTS=-fvectorize`
You can also change directory to a specific test folder to run that
diff --git a/driver/runtests.py b/driver/runtests.py
index f4a28b9..b0367f3 100644
--- a/driver/runtests.py
+++ b/driver/runtests.py
@@ -44,6 +44,7 @@ long_options = [
"skipway=", # skip this way
"threads=", # threads to run simultaneously
"check-files-written", # check files aren't written by multiple tests
+ "verbose=", # verbose (0,1,2 so far)
]
opts, args = getopt.getopt(sys.argv[1:], "e:", long_options)
@@ -92,6 +93,13 @@ for opt,arg in opts:
if opt == '--check-files-written':
config.check_files_written = True
+ if opt == '--verbose':
+ if arg not in ["0","1","2","3"]:
+ sys.stderr.write("ERROR: requested verbosity %s not supported, use 0,1,2 or 3" % arg)
+ sys.exit(1)
+ config.verbose = int(arg)
+
+
if config.use_threads == 1:
# Trac #1558 says threads don't work in python 2.4.4, but do
# in 2.5.2. Probably >= 2.5 is sufficient, but let's be
@@ -242,7 +250,7 @@ sys.stdout = os.fdopen(sys.__stdout__.fileno(), "w", 0)
# First collect all the tests to be run
for file in t_files:
- print '====> Scanning', file
+ if_verbose(2, '====> Scanning %s' % file)
newTestDir(os.path.dirname(file))
try:
execfile(file)
diff --git a/driver/testglobals.py b/driver/testglobals.py
index 686fd1d..85a5b7d 100644
--- a/driver/testglobals.py
+++ b/driver/testglobals.py
@@ -43,7 +43,7 @@ class TestConfig:
self.wordsize = ''
# Verbosity level
- self.verbose = 1
+ self.verbose = 3
# run the "fast" version of the test suite
self.fast = 0
diff --git a/driver/testlib.py b/driver/testlib.py
index a163f00..68937c6 100644
--- a/driver/testlib.py
+++ b/driver/testlib.py
@@ -712,10 +712,11 @@ def do_test(name, way, func, args):
full_name = name + '(' + way + ')'
try:
- print '=====>', full_name, t.total_tests, 'of', len(allTestNames), \
- str([t.n_unexpected_passes, \
- t.n_unexpected_failures, \
- t.n_framework_failures])
+ if_verbose(2, "=====> %s %d of %d %s " % \
+ (full_name, t.total_tests, len(allTestNames), \
+ [t.n_unexpected_passes, \
+ t.n_unexpected_failures, \
+ t.n_framework_failures]))
if config.use_threads:
t.lock.release()
@@ -754,13 +755,13 @@ def do_test(name, way, func, args):
else:
t.expected_passes[name] = [way]
else:
- print '*** unexpected pass for', full_name
+ if_verbose(1, '*** unexpected pass for %s' % full_name)
t.n_unexpected_passes = t.n_unexpected_passes + 1
addPassingTestInfo(t.unexpected_passes, getTestOpts().testdir, name, way)
elif passFail == 'fail':
if getTestOpts().expect == 'pass' \
and way not in getTestOpts().expect_fail_for:
- print '*** unexpected failure for', full_name
+ if_verbose(1, '*** unexpected failure for %s' % full_name)
t.n_unexpected_failures = t.n_unexpected_failures + 1
reason = result['reason']
addFailingTestInfo(t.unexpected_failures, getTestOpts().testdir, name, reason, way)
@@ -820,7 +821,7 @@ def skiptest (name, way):
def framework_fail( name, way, reason ):
full_name = name + '(' + way + ')'
- print '*** framework failure for', full_name, reason
+ if_verbose(1, '*** framework failure for %s %s ' %s (full_name, reason))
t.n_framework_failures = t.n_framework_failures + 1
if name in t.framework_failures:
t.framework_failures[name].append(way)
@@ -1595,7 +1596,7 @@ def compare_outputs( kind, normaliser, expected_file, actual_file ):
if expected_str == actual_str:
return 1
else:
- print 'Actual ' + kind + ' output differs from expected:'
+ if_verbose(1, 'Actual ' + kind + ' output differs from expected:')
if expected_file_for_diff == '/dev/null':
expected_normalised_file = '/dev/null'
@@ -1614,17 +1615,18 @@ def compare_outputs( kind, normaliser, expected_file, actual_file ):
# (including newlines) so the diff would be hard to read.
# This does mean that the diff might contain changes that
# would be normalised away.
- r = os.system( 'diff -uw ' + expected_file_for_diff + \
- ' ' + actual_file )
+ if (config.verbose >= 1):
+ r = os.system( 'diff -uw ' + expected_file_for_diff + \
+ ' ' + actual_file )
- # If for some reason there were no non-whitespace differences,
- # then do a full diff
- if r == 0:
- r = os.system( 'diff -u ' + expected_file_for_diff + \
- ' ' + actual_file )
+ # If for some reason there were no non-whitespace differences,
+ # then do a full diff
+ if r == 0:
+ r = os.system( 'diff -u ' + expected_file_for_diff + \
+ ' ' + actual_file )
if config.accept:
- print 'Accepting new output.'
+ if_verbose(1, 'Accepting new output.')
write_file(expected_file, actual_raw)
return 1
else:
@@ -1766,7 +1768,7 @@ def rawSystemWithTimeout(cmd_and_args):
# Then, when using the native Python, os.system will invoke the cmd shell
def runCmd( cmd ):
- if_verbose( 1, cmd )
+ if_verbose( 3, cmd )
r = 0
if config.os == 'mingw32':
# On MinGW, we will always have timeout
@@ -1779,7 +1781,7 @@ def runCmd( cmd ):
return r << 8
def runCmdFor( name, cmd, timeout_multiplier=1.0 ):
- if_verbose( 1, cmd )
+ if_verbose( 3, cmd )
r = 0
if config.os == 'mingw32':
# On MinGW, we will always have timeout
diff --git a/mk/test.mk b/mk/test.mk
index 5b1fad7..928e1e4 100644
--- a/mk/test.mk
+++ b/mk/test.mk
@@ -159,6 +159,10 @@ ifneq "$(THREADS)" ""
RUNTEST_OPTS += --threads=$(THREADS)
endif
+ifneq "$(VERBOSE)" ""
+RUNTEST_OPTS += --verbose=$(VERBOSE)
+endif
+
ifneq "$(CLEAN_ONLY)" ""
RUNTEST_OPTS += -e clean_only=True
else
More information about the ghc-commits
mailing list