[commit: ghc] wip/perf-testsuite: Greatly improved printing. Fixed the delta function. Made things simpler (bf5ee28)
git at git.haskell.org
git at git.haskell.org
Fri Jul 28 20:33:39 UTC 2017
Repository : ssh://git@git.haskell.org/ghc
On branch : wip/perf-testsuite
Link : http://ghc.haskell.org/trac/ghc/changeset/bf5ee28b0660e85e972472a48393f12ae6e7884e/ghc
>---------------------------------------------------------------
commit bf5ee28b0660e85e972472a48393f12ae6e7884e
Author: Jared Weakly <jweakly at pdx.edu>
Date: Wed Jul 26 18:30:37 2017 -0700
Greatly improved printing. Fixed the delta function. Made things simpler
Signed-off-by: Jared Weakly <jweakly at pdx.edu>
>---------------------------------------------------------------
bf5ee28b0660e85e972472a48393f12ae6e7884e
testsuite/driver/perf_notes.py | 104 +++++++++++++++++++----------------------
1 file changed, 47 insertions(+), 57 deletions(-)
diff --git a/testsuite/driver/perf_notes.py b/testsuite/driver/perf_notes.py
index 2e9273d..ffa7656 100644
--- a/testsuite/driver/perf_notes.py
+++ b/testsuite/driver/perf_notes.py
@@ -8,8 +8,6 @@
# metrics between measurements taken for given commits in the environment given
# by --test-env.
-from __future__ import print_function
-
# TODO: Actually figure out what imports I need.
import argparse
import re
@@ -26,11 +24,12 @@ parser.add_argument("--test-name",
help="Optional: If given, filters table to include only \
tests matching the given regular expression.")
# This is always going to be the last processing done on the metrics list because of how destructive it is.
-parser.add_argument("--min-delta",
+parser.add_argument("--min-delta",type=float,
help="Optional: Display only tests where the relative \
spread is greater than the given value.")
-parser.add_argument("--add-note", nargs=2,
- help="Development only. Adds N fake metrics to the given commit")
+parser.add_argument("--add-note", nargs=3,
+ help="Development only. Adds N fake metrics to the given commit. \
+ The third argument is a useless flag to add some functionality.")
parser.add_argument("commits", nargs=argparse.REMAINDER,
help="The rest of the arguments will be the commits that will be used.")
@@ -40,83 +39,74 @@ args = parser.parse_args()
env = 'local'
name = re.compile('.*')
# metrics is a dictionary of the form
-# {commit_1 : parse_git_notes, commit_2 : parse_git_notes, ... }
-metrics = {}
+# [ {'test_env': 'local', 'test': 'T100', 'way': 'some_way', 'metric': 'some_field', 'value': '1000', 'commit': 'HEAD'} ]
+metrics = []
if args.commits:
- metrics = dict(zip(args.commits, [parse_git_notes('perf',[c]) for c in args.commits]))
+ for c in args.commits:
+ metrics += parse_git_notes('perf',c)
if args.test_env:
- temp = []
- for commit in metrics:
- temp.append([test for test in metrics.get(commit) if test['TEST_ENV'] == args.test_env])
-
- metrics = dict(zip(metrics.keys(),temp))
+ metrics = [test for test in metrics if test['test_env'] == args.test_env]
if args.test_name:
name = re.compile(args.test_name)
- temp = []
- for commit in metrics:
- temp.append([test for test in metrics.get(commit) if name.search(test.get('TEST',''))])
-
- metrics = dict(zip(metrics.keys(),temp))
+ metrics = [test for test in metrics if name.search(test.get('test',''))]
if args.min_delta:
- delta = 1.0 - float(args.min_delta)
+ delta = args.min_delta
+ print(delta)
+ # Took me way too long to realize I had the math utterly borked for the comparison
def cmp(v1, v2):
- return (v1/v2) > delta
+ if v1 > v2:
+ return (100 * (v1 - v2)/v2) > delta
+ else:
+ return (100 * (v2 - v1)/v1) > delta
# I only want to compare the first commit with everything else.
# So go through every item in the first commit and look up that test in
# the other commits. Keep the falses.
- m = [] # tempy variable because I don't know how to do this better without internet.
- for tst in metrics.get(list(metrics.keys())[0]):
- for k in list(metrics.keys())[1:]: # Because I needed a separate iterator for this somehow
- for t in metrics.get(k):
- if (t['TEST'] == tst['TEST']):
- m.append((tst,t)) # Pairing off matching test names across commits.
+ latest_commit = [t for t in metrics if t['commit'] == args.commits[0]]
+
+ m = [] # tempy variable
+ for t in latest_commit:
+ m += [(t,test) for test in metrics if (t['test'] == test['test']) and (t['commit'] != test['commit'])]
deltas = []
- for k,v in m:
+ for fst,snd in m:
# So... Much... Casting... oh my gawd.
- if (not cmp(float(k['VALUE']),float(v['VALUE']))):
- deltas.append(k)
+ if cmp(float(fst['value']),float(snd['value'])):
+ deltas.append(fst)
- # metrics :: { commit : [ {tests} ] }
- metrics = { list(metrics.keys())[0] : deltas }
+ metrics = deltas
if args.add_note:
- def note_gen(n, commit):
+ def note_gen(n, commit, delta=''):
note = []
# To generate good testing data, I need some similar test names, test metrics, environments in every commit.
# I also need some same tests but different metric values, same test name, different test environment names, etc.
# This will do for now, but it's not really sufficient.
# There's a better test_metrics = {} dictionary I just stuck in a file for now.
- [note.append('\t'.join(['local', 'T'+ str(i*100), 'some_way', 'some_field', str(i*1000)])) for i in range(1,int(n)+1)]
+ if not delta:
+ [note.append('\t'.join(['local', 'T'+ str(i*100), 'some_way', 'some_field', str(i*1000)])) for i in range(1,int(int(n)/2)+1)]
+ [note.append('\t'.join(['non-local', 'W'+ str(i*100), 'other_way', 'other_field', str(i*100)])) for i in range(int(int(n)/2)+1,int(n)+1)]
+ if delta:
+ [note.append('\t'.join(['local', 'T'+ str(i*100), 'some_way', 'some_field', str(i*10)])) for i in range(1,int(int(n)/2)+1)]
+ [note.append('\t'.join(['non-local', 'W'+ str(i*100), 'other_way', 'other_field', str(i*1)])) for i in range(int(int(n)/2)+1,int(n)+1)]
+
git_note = subprocess.check_output(["git","notes","--ref=perf","append",commit,"-m", "\n".join(note)])
- note_gen(args.add_note[0],args.add_note[1])
-
-
-# At this point, since metrics is a { commit : [lst of tests] } variable,
-# it should be pretty workable hopefully.
-print(metrics)
-
-# It'll be best to go through the newest commit (the one we care about)
-# and find all matching TESTS and then make a table of comparisons.
-# If there's a test in the newest commit that doesn't exist in older ones,
-# - should I just ignore it?
-# - should I just print it as a one line table?
-# -
-# TEST: tst | METRIC: mtrk
-# ----------+------------
-# commit1 | value
-# commit2 | value
-# ... | ...
-# commitN | value
-
-print("{:<12} {:<10} {:<10} {:<20} {:<15}".format('TEST_ENV','TEST','WAY','METRIC','VALUE'))
-for key in metrics:
- print("{:<12} {:<10} {:<10} {:<20} {:<15}"
- .format(key['TEST_ENV'],key['TEST'],key['WAY'],key['METRIC'],key['VALUE']))
+ note_gen(args.add_note[0],args.add_note[1],args.add_note[2])
+
+latest_commit = [t for t in metrics if t['commit'] == args.commits[0]]
+rest = [t for t in metrics if t['commit'] != args.commits[0]]
+
+for test in latest_commit:
+ print("{:<13} {:5} {:<13}".format('TEST: ' + test['test'], ' | ', 'METRIC: ' + test['metric']))
+ print("-------------------------------")
+ print("{:<13} {:5} {:<13}".format('commit:' + test['commit'], ' | ', test['value']))
+ for t in rest:
+ if t['test'] == test['test']:
+ print("{:<13} {:5} {:<13}".format('commit:' + t['commit'], ' | ', t['value']))
+ print('\n')
More information about the ghc-commits
mailing list