[commit: ghc] wip/perf-testsuite: Basic metrics collection and command line options working (83a482c)
git at git.haskell.org
git at git.haskell.org
Fri Jul 28 20:33:11 UTC 2017
Repository : ssh://git@git.haskell.org/ghc
On branch : wip/perf-testsuite
Link : http://ghc.haskell.org/trac/ghc/changeset/83a482c1ff87ef1655b2280cf6f67d83086ae266/ghc
>---------------------------------------------------------------
commit 83a482c1ff87ef1655b2280cf6f67d83086ae266
Author: Jared Weakly <jweakly at pdx.edu>
Date: Thu Jul 6 17:16:49 2017 -0700
Basic metrics collection and command line options working
>---------------------------------------------------------------
83a482c1ff87ef1655b2280cf6f67d83086ae266
libraries/array | 2 +-
libraries/hoopl | 1 +
testsuite/driver/runtests.py | 36 ++++++++++++++++++++++++------------
testsuite/driver/testglobals.py | 11 ++++++++++-
testsuite/driver/testlib.py | 6 ++++++
testsuite/driver/testutil.py | 4 ++++
testsuite/mk/test.mk | 12 ++++++++++++
7 files changed, 58 insertions(+), 14 deletions(-)
diff --git a/libraries/array b/libraries/array
index 9a23fea..f7b69e9 160000
--- a/libraries/array
+++ b/libraries/array
@@ -1 +1 @@
-Subproject commit 9a23feac0b78e713c0f7877066fa24dbc2217c20
+Subproject commit f7b69e9cb914cb69bbede5264729523fb8669db1
diff --git a/libraries/hoopl b/libraries/hoopl
new file mode 160000
index 0000000..ac24864
--- /dev/null
+++ b/libraries/hoopl
@@ -0,0 +1 @@
+Subproject commit ac24864c2db7951a6f34674e2b11b69d37ef84ff
diff --git a/testsuite/driver/runtests.py b/testsuite/driver/runtests.py
index 7e4f375..c09b063 100644
--- a/testsuite/driver/runtests.py
+++ b/testsuite/driver/runtests.py
@@ -42,18 +42,21 @@ def signal_handler(signal, frame):
# cmd-line options
long_options = [
- "configfile=", # config file
- "config=", # config field
- "rootdir=", # root of tree containing tests (default: .)
- "summary-file=", # file in which to save the (human-readable) summary
- "no-print-summary=", # should we print the summary?
- "only=", # just this test (can be give multiple --only= flags)
- "way=", # just this way
- "skipway=", # skip this way
- "threads=", # threads to run simultaneously
- "check-files-written", # check files aren't written by multiple tests
- "verbose=", # verbose (0,1,2 so far)
- "skip-perf-tests", # skip performance tests
+ "configfile=", # config file
+ "config=", # config field
+ "rootdir=", # root of tree containing tests (default: .)
+ "summary-file=", # file in which to save the (human-readable) summary
+ "no-print-summary=", # should we print the summary?
+ "only=", # just this test (can be give multiple --only= flags)
+ "way=", # just this way
+ "skipway=", # skip this way
+ "threads=", # threads to run simultaneously
+ "check-files-written", # check files aren't written by multiple tests
+ "verbose=", # verbose (0,1,2 so far)
+ "skip-perf-tests", # skip performance tests
+ "only-perf-tests", # Only do performance tests
+ "use-git-notes", # use git notes to store metrics. NOTE: This is expected to become the default and will eventually be taken out.
+ "TEST_ENV=", # Override default chosen test-env.
]
opts, args = getopt.getopt(sys.argv[1:], "e:", long_options)
@@ -110,12 +113,21 @@ for opt,arg in opts:
if opt == '--skip-perf-tests':
config.skip_perf_tests = True
+ if opt == '--only-perf-tests':
+ config.only_perf_tests = True
+
+ if opt == '--use-git-notes':
+ config.use_git_notes = True
+
if opt == '--verbose':
if arg not in ["0","1","2","3","4","5"]:
sys.stderr.write("ERROR: requested verbosity %s not supported, use 0,1,2,3,4 or 5" % arg)
sys.exit(1)
config.verbose = int(arg)
+ if opt == '--TEST_ENV':
+ config.TEST_ENV = arg
+
config.cygwin = False
config.msys = False
diff --git a/testsuite/driver/testglobals.py b/testsuite/driver/testglobals.py
index fc050e6..bd8eefe 100644
--- a/testsuite/driver/testglobals.py
+++ b/testsuite/driver/testglobals.py
@@ -117,6 +117,16 @@ class TestConfig:
# Should we skip performance tests
self.skip_perf_tests = False
+ # Only do performance tests
+ self.only_perf_tests = False
+
+ # Should we dump statistics to git notes?
+ self.use_git_notes = False
+ # To accumulate the metrics for the git notes
+ self.accumulate_metrics = []
+ # Has the user defined a custom test environment? Local is default.
+ self.TEST_ENV = 'local'
+
global config
config = TestConfig()
@@ -283,4 +293,3 @@ default_testopts = TestOptions()
# (bug, directory, name) of tests marked broken
global brokens
brokens = []
-
diff --git a/testsuite/driver/testlib.py b/testsuite/driver/testlib.py
index 26e3d17..ae82d1f 100644
--- a/testsuite/driver/testlib.py
+++ b/testsuite/driver/testlib.py
@@ -1084,6 +1084,7 @@ def stats( name, way, stats_file ):
# Check -t stats info
def checkStats(name, way, stats_file, range_fields):
+
full_name = name + '(' + way + ')'
result = passed()
@@ -1107,6 +1108,11 @@ def checkStats(name, way, stats_file, range_fields):
deviation = round(((float(val) * 100)/ expected) - 100, 1)
+ # Add val into the git note if option is set.
+ if config.use_git_notes:
+ test_env = config.TEST_ENV
+ config.accumulate_metrics.append(test_env + '\t' + name + '\t' + way + '\t' + field + '\t' + str(val))
+
if val < lowerBound:
print(field, 'value is too low:')
print('(If this is because you have improved GHC, please')
diff --git a/testsuite/driver/testutil.py b/testsuite/driver/testutil.py
index dcba177..c6297ff 100644
--- a/testsuite/driver/testutil.py
+++ b/testsuite/driver/testutil.py
@@ -47,6 +47,10 @@ def lndir(srcdir, dstdir):
os.mkdir(dst)
lndir(src, dst)
+# def git_append(note):
+# def print_metrics():
+# print(config.accumulate_metrics)
+
# On Windows, os.symlink is not defined with Python 2.7, but is in Python 3
# when using msys2, as GHC does. Unfortunately, only Administrative users have
# the privileges necessary to create symbolic links by default. Consequently we
diff --git a/testsuite/mk/test.mk b/testsuite/mk/test.mk
index a44e200..9896883 100644
--- a/testsuite/mk/test.mk
+++ b/testsuite/mk/test.mk
@@ -195,6 +195,18 @@ ifeq "$(SKIP_PERF_TESTS)" "YES"
RUNTEST_OPTS += --skip-perf-tests
endif
+ifeq "$(ONLY_PERF_TESTS)" "YES"
+RUNTEST_OPTS += --only-perf-tests
+endif
+
+ifeq "$(USE_GIT_NOTES)" "YES"
+RUNTEST_OPTS += --use-git-notes
+endif
+
+ifneq "$(TEST_ENV)" ""
+RUNTEST_OPTS += --TEST_ENV="$(TEST_ENV)"
+endif
+
ifeq "$(CLEANUP)" "0"
RUNTEST_OPTS += -e config.cleanup=False
else ifeq "$(CLEANUP)" "NO"
More information about the ghc-commits
mailing list