[commit: ghc] wip/perf-testsuite: Basic metrics collection and command line options working (6da06d6)

[git at git.haskell.org]

Repository : ssh://git@git.haskell.org/ghc

On branch  : wip/perf-testsuite
Link       : http://ghc.haskell.org/trac/ghc/changeset/6da06d67e6ccb5d6c115ab6db72eaa634838a6e2/ghc

>---------------------------------------------------------------

commit 6da06d67e6ccb5d6c115ab6db72eaa634838a6e2
Author: Jared Weakly <jweakly at pdx.edu>
Date:   Thu Jul 6 17:16:49 2017 -0700

    Basic metrics collection and command line options working


>---------------------------------------------------------------

6da06d67e6ccb5d6c115ab6db72eaa634838a6e2
 libraries/array                 |  2 +-
 libraries/hoopl                 |  1 +
 testsuite/driver/testglobals.py | 11 ++++++++++-
 testsuite/driver/testlib.py     |  6 ++++++
 testsuite/driver/testutil.py    |  4 ++++
 testsuite/mk/test.mk            | 12 ++++++++++++
 6 files changed, 34 insertions(+), 2 deletions(-)

diff --git a/libraries/array b/libraries/array
index 9a23fea..f7b69e9 160000
--- a/libraries/array
+++ b/libraries/array
@@ -1 +1 @@
-Subproject commit 9a23feac0b78e713c0f7877066fa24dbc2217c20
+Subproject commit f7b69e9cb914cb69bbede5264729523fb8669db1
diff --git a/libraries/hoopl b/libraries/hoopl
new file mode 160000
index 0000000..ac24864
--- /dev/null
+++ b/libraries/hoopl
@@ -0,0 +1 @@
+Subproject commit ac24864c2db7951a6f34674e2b11b69d37ef84ff
diff --git a/testsuite/driver/testglobals.py b/testsuite/driver/testglobals.py
index 5e7142d..5ed54bd 100644
--- a/testsuite/driver/testglobals.py
+++ b/testsuite/driver/testglobals.py
@@ -117,6 +117,16 @@ class TestConfig:
         # Should we skip performance tests
         self.skip_perf_tests = False
 
+        # Only do performance tests
+        self.only_perf_tests = False
+
+        # Should we dump statistics to git notes?
+        self.use_git_notes = False
+        # To accumulate the metrics for the git notes
+        self.accumulate_metrics = []
+        # Has the user defined a custom test environment? Local is default.
+        self.TEST_ENV = 'local'
+
 global config
 config = TestConfig()
 
@@ -284,4 +294,3 @@ default_testopts = TestOptions()
 # (bug, directory, name) of tests marked broken
 global brokens
 brokens = []
-
diff --git a/testsuite/driver/testlib.py b/testsuite/driver/testlib.py
index 15c773e..0da86f2 100644
--- a/testsuite/driver/testlib.py
+++ b/testsuite/driver/testlib.py
@@ -1085,6 +1085,7 @@ def stats( name, way, stats_file ):
 # Check -t stats info
 
 def checkStats(name, way, stats_file, range_fields):
+
     full_name = name + '(' + way + ')'
 
     result = passed()
@@ -1108,6 +1109,11 @@ def checkStats(name, way, stats_file, range_fields):
 
             deviation = round(((float(val) * 100)/ expected) - 100, 1)
 
+            # Add val into the git note if option is set.
+            if config.use_git_notes:
+                test_env = config.TEST_ENV
+                config.accumulate_metrics.append(test_env + '\t' + name + '\t' + way + '\t' + field + '\t' + str(val))
+
             if val < lowerBound:
                 print(field, 'value is too low:')
                 print('(If this is because you have improved GHC, please')
diff --git a/testsuite/driver/testutil.py b/testsuite/driver/testutil.py
index dcba177..c6297ff 100644
--- a/testsuite/driver/testutil.py
+++ b/testsuite/driver/testutil.py
@@ -47,6 +47,10 @@ def lndir(srcdir, dstdir):
             os.mkdir(dst)
             lndir(src, dst)
 
+# def git_append(note):
+# def print_metrics():
+#     print(config.accumulate_metrics)
+
 # On Windows, os.symlink is not defined with Python 2.7, but is in Python 3
 # when using msys2, as GHC does. Unfortunately, only Administrative users have
 # the privileges necessary to create symbolic links by default. Consequently we
diff --git a/testsuite/mk/test.mk b/testsuite/mk/test.mk
index a21c4bb..273b37b 100644
--- a/testsuite/mk/test.mk
+++ b/testsuite/mk/test.mk
@@ -195,6 +195,18 @@ ifeq "$(SKIP_PERF_TESTS)" "YES"
 RUNTEST_OPTS += --skip-perf-tests
 endif
 
+ifeq "$(ONLY_PERF_TESTS)" "YES"
+RUNTEST_OPTS += --only-perf-tests
+endif
+
+ifeq "$(USE_GIT_NOTES)" "YES"
+RUNTEST_OPTS += --use-git-notes
+endif
+
+ifneq "$(TEST_ENV)" ""
+RUNTEST_OPTS += --TEST_ENV="$(TEST_ENV)"
+endif
+
 ifeq "$(CLEANUP)" "0"
 RUNTEST_OPTS += -e config.cleanup=False
 else ifeq "$(CLEANUP)" "NO"