[commit: ghc] wip/perf-testsuite: Basic metrics collection and command line options working (83a482c)

git at git.haskell.org git at git.haskell.org
Fri Jul 28 20:33:11 UTC 2017


Repository : ssh://git@git.haskell.org/ghc

On branch  : wip/perf-testsuite
Link       : http://ghc.haskell.org/trac/ghc/changeset/83a482c1ff87ef1655b2280cf6f67d83086ae266/ghc

>---------------------------------------------------------------

commit 83a482c1ff87ef1655b2280cf6f67d83086ae266
Author: Jared Weakly <jweakly at pdx.edu>
Date:   Thu Jul 6 17:16:49 2017 -0700

    Basic metrics collection and command line options working


>---------------------------------------------------------------

83a482c1ff87ef1655b2280cf6f67d83086ae266
 libraries/array                 |  2 +-
 libraries/hoopl                 |  1 +
 testsuite/driver/runtests.py    | 36 ++++++++++++++++++++++++------------
 testsuite/driver/testglobals.py | 11 ++++++++++-
 testsuite/driver/testlib.py     |  6 ++++++
 testsuite/driver/testutil.py    |  4 ++++
 testsuite/mk/test.mk            | 12 ++++++++++++
 7 files changed, 58 insertions(+), 14 deletions(-)

diff --git a/libraries/array b/libraries/array
index 9a23fea..f7b69e9 160000
--- a/libraries/array
+++ b/libraries/array
@@ -1 +1 @@
-Subproject commit 9a23feac0b78e713c0f7877066fa24dbc2217c20
+Subproject commit f7b69e9cb914cb69bbede5264729523fb8669db1
diff --git a/libraries/hoopl b/libraries/hoopl
new file mode 160000
index 0000000..ac24864
--- /dev/null
+++ b/libraries/hoopl
@@ -0,0 +1 @@
+Subproject commit ac24864c2db7951a6f34674e2b11b69d37ef84ff
diff --git a/testsuite/driver/runtests.py b/testsuite/driver/runtests.py
index 7e4f375..c09b063 100644
--- a/testsuite/driver/runtests.py
+++ b/testsuite/driver/runtests.py
@@ -42,18 +42,21 @@ def signal_handler(signal, frame):
 # cmd-line options
 
 long_options = [
-  "configfile=",	# config file
-  "config=",  		# config field
-  "rootdir=", 		# root of tree containing tests (default: .)
-  "summary-file=",      # file in which to save the (human-readable) summary
-  "no-print-summary=",  # should we print the summary?
-  "only=",		# just this test (can be give multiple --only= flags)
-  "way=",		# just this way
-  "skipway=",		# skip this way
-  "threads=",           # threads to run simultaneously
-  "check-files-written", # check files aren't written by multiple tests
-  "verbose=",          # verbose (0,1,2 so far)
-  "skip-perf-tests",       # skip performance tests
+  "configfile=",          # config file
+  "config=",              # config field
+  "rootdir=",             # root of tree containing tests (default: .)
+  "summary-file=",        # file in which to save the (human-readable) summary
+  "no-print-summary=",    # should we print the summary?
+  "only=",                # just this test (can be give multiple --only= flags)
+  "way=",                 # just this way
+  "skipway=",	          # skip this way
+  "threads=",             # threads to run simultaneously
+  "check-files-written",  # check files aren't written by multiple tests
+  "verbose=",             # verbose (0,1,2 so far)
+  "skip-perf-tests",      # skip performance tests
+  "only-perf-tests",       # Only do performance tests
+  "use-git-notes",        # use git notes to store metrics. NOTE: This is expected to become the default and will eventually be taken out.
+  "TEST_ENV=",            # Override default chosen test-env.
   ]
 
 opts, args = getopt.getopt(sys.argv[1:], "e:", long_options)
@@ -110,12 +113,21 @@ for opt,arg in opts:
     if opt == '--skip-perf-tests':
         config.skip_perf_tests = True
 
+    if opt == '--only-perf-tests':
+        config.only_perf_tests = True
+
+    if opt == '--use-git-notes':
+        config.use_git_notes = True
+
     if opt == '--verbose':
         if arg not in ["0","1","2","3","4","5"]:
             sys.stderr.write("ERROR: requested verbosity %s not supported, use 0,1,2,3,4 or 5" % arg)
             sys.exit(1)
         config.verbose = int(arg)
 
+    if opt == '--TEST_ENV':
+        config.TEST_ENV = arg
+
 
 config.cygwin = False
 config.msys = False
diff --git a/testsuite/driver/testglobals.py b/testsuite/driver/testglobals.py
index fc050e6..bd8eefe 100644
--- a/testsuite/driver/testglobals.py
+++ b/testsuite/driver/testglobals.py
@@ -117,6 +117,16 @@ class TestConfig:
         # Should we skip performance tests
         self.skip_perf_tests = False
 
+        # Only do performance tests
+        self.only_perf_tests = False
+
+        # Should we dump statistics to git notes?
+        self.use_git_notes = False
+        # To accumulate the metrics for the git notes
+        self.accumulate_metrics = []
+        # Has the user defined a custom test environment? Local is default.
+        self.TEST_ENV = 'local'
+
 global config
 config = TestConfig()
 
@@ -283,4 +293,3 @@ default_testopts = TestOptions()
 # (bug, directory, name) of tests marked broken
 global brokens
 brokens = []
-
diff --git a/testsuite/driver/testlib.py b/testsuite/driver/testlib.py
index 26e3d17..ae82d1f 100644
--- a/testsuite/driver/testlib.py
+++ b/testsuite/driver/testlib.py
@@ -1084,6 +1084,7 @@ def stats( name, way, stats_file ):
 # Check -t stats info
 
 def checkStats(name, way, stats_file, range_fields):
+
     full_name = name + '(' + way + ')'
 
     result = passed()
@@ -1107,6 +1108,11 @@ def checkStats(name, way, stats_file, range_fields):
 
             deviation = round(((float(val) * 100)/ expected) - 100, 1)
 
+            # Add val into the git note if option is set.
+            if config.use_git_notes:
+                test_env = config.TEST_ENV
+                config.accumulate_metrics.append(test_env + '\t' + name + '\t' + way + '\t' + field + '\t' + str(val))
+
             if val < lowerBound:
                 print(field, 'value is too low:')
                 print('(If this is because you have improved GHC, please')
diff --git a/testsuite/driver/testutil.py b/testsuite/driver/testutil.py
index dcba177..c6297ff 100644
--- a/testsuite/driver/testutil.py
+++ b/testsuite/driver/testutil.py
@@ -47,6 +47,10 @@ def lndir(srcdir, dstdir):
             os.mkdir(dst)
             lndir(src, dst)
 
+# def git_append(note):
+# def print_metrics():
+#     print(config.accumulate_metrics)
+
 # On Windows, os.symlink is not defined with Python 2.7, but is in Python 3
 # when using msys2, as GHC does. Unfortunately, only Administrative users have
 # the privileges necessary to create symbolic links by default. Consequently we
diff --git a/testsuite/mk/test.mk b/testsuite/mk/test.mk
index a44e200..9896883 100644
--- a/testsuite/mk/test.mk
+++ b/testsuite/mk/test.mk
@@ -195,6 +195,18 @@ ifeq "$(SKIP_PERF_TESTS)" "YES"
 RUNTEST_OPTS += --skip-perf-tests
 endif
 
+ifeq "$(ONLY_PERF_TESTS)" "YES"
+RUNTEST_OPTS += --only-perf-tests
+endif
+
+ifeq "$(USE_GIT_NOTES)" "YES"
+RUNTEST_OPTS += --use-git-notes
+endif
+
+ifneq "$(TEST_ENV)" ""
+RUNTEST_OPTS += --TEST_ENV="$(TEST_ENV)"
+endif
+
 ifeq "$(CLEANUP)" "0"
 RUNTEST_OPTS += -e config.cleanup=False
 else ifeq "$(CLEANUP)" "NO"



More information about the ghc-commits mailing list