[commit: ghc] wip/perf-testsuite: Basic metrics collection and command line options working (0ae1afe)

[git at git.haskell.org]

Repository : ssh://git@git.haskell.org/ghc

On branch  : wip/perf-testsuite
Link       : http://ghc.haskell.org/trac/ghc/changeset/0ae1afe4bbb9cc0fe7916abafaf2fe2016607bb3/ghc

>---------------------------------------------------------------

commit 0ae1afe4bbb9cc0fe7916abafaf2fe2016607bb3
Author: Jared Weakly <jweakly at pdx.edu>
Date:   Thu Jul 6 17:16:49 2017 -0700

    Basic metrics collection and command line options working


>---------------------------------------------------------------

0ae1afe4bbb9cc0fe7916abafaf2fe2016607bb3
 testsuite/driver/testglobals.py | 11 ++++++++++-
 testsuite/driver/testlib.py     |  6 ++++++
 testsuite/driver/testutil.py    |  4 ++++
 testsuite/mk/test.mk            | 12 ++++++++++++
 4 files changed, 32 insertions(+), 1 deletion(-)

diff --git a/testsuite/driver/testglobals.py b/testsuite/driver/testglobals.py
index 5e7142d..5ed54bd 100644
--- a/testsuite/driver/testglobals.py
+++ b/testsuite/driver/testglobals.py
@@ -117,6 +117,16 @@ class TestConfig:
         # Should we skip performance tests
         self.skip_perf_tests = False
 
+        # Only do performance tests
+        self.only_perf_tests = False
+
+        # Should we dump statistics to git notes?
+        self.use_git_notes = False
+        # To accumulate the metrics for the git notes
+        self.accumulate_metrics = []
+        # Has the user defined a custom test environment? Local is default.
+        self.TEST_ENV = 'local'
+
 global config
 config = TestConfig()
 
@@ -284,4 +294,3 @@ default_testopts = TestOptions()
 # (bug, directory, name) of tests marked broken
 global brokens
 brokens = []
-
diff --git a/testsuite/driver/testlib.py b/testsuite/driver/testlib.py
index 15c773e..0da86f2 100644
--- a/testsuite/driver/testlib.py
+++ b/testsuite/driver/testlib.py
@@ -1085,6 +1085,7 @@ def stats( name, way, stats_file ):
 # Check -t stats info
 
 def checkStats(name, way, stats_file, range_fields):
+
     full_name = name + '(' + way + ')'
 
     result = passed()
@@ -1108,6 +1109,11 @@ def checkStats(name, way, stats_file, range_fields):
 
             deviation = round(((float(val) * 100)/ expected) - 100, 1)
 
+            # Add val into the git note if option is set.
+            if config.use_git_notes:
+                test_env = config.TEST_ENV
+                config.accumulate_metrics.append(test_env + '\t' + name + '\t' + way + '\t' + field + '\t' + str(val))
+
             if val < lowerBound:
                 print(field, 'value is too low:')
                 print('(If this is because you have improved GHC, please')
diff --git a/testsuite/driver/testutil.py b/testsuite/driver/testutil.py
index dcba177..c6297ff 100644
--- a/testsuite/driver/testutil.py
+++ b/testsuite/driver/testutil.py
@@ -47,6 +47,10 @@ def lndir(srcdir, dstdir):
             os.mkdir(dst)
             lndir(src, dst)
 
+# def git_append(note):
+# def print_metrics():
+#     print(config.accumulate_metrics)
+
 # On Windows, os.symlink is not defined with Python 2.7, but is in Python 3
 # when using msys2, as GHC does. Unfortunately, only Administrative users have
 # the privileges necessary to create symbolic links by default. Consequently we
diff --git a/testsuite/mk/test.mk b/testsuite/mk/test.mk
index a21c4bb..273b37b 100644
--- a/testsuite/mk/test.mk
+++ b/testsuite/mk/test.mk
@@ -195,6 +195,18 @@ ifeq "$(SKIP_PERF_TESTS)" "YES"
 RUNTEST_OPTS += --skip-perf-tests
 endif
 
+ifeq "$(ONLY_PERF_TESTS)" "YES"
+RUNTEST_OPTS += --only-perf-tests
+endif
+
+ifeq "$(USE_GIT_NOTES)" "YES"
+RUNTEST_OPTS += --use-git-notes
+endif
+
+ifneq "$(TEST_ENV)" ""
+RUNTEST_OPTS += --TEST_ENV="$(TEST_ENV)"
+endif
+
 ifeq "$(CLEANUP)" "0"
 RUNTEST_OPTS += -e config.cleanup=False
 else ifeq "$(CLEANUP)" "NO"