[Git][ghc/ghc][wip/explicit-perf-baseline] 2 commits: testsuite: Allow baseline commit to be set explicitly
Ben Gamari
gitlab at gitlab.haskell.org
Sun Jul 5 16:07:48 UTC 2020
Ben Gamari pushed to branch wip/explicit-perf-baseline at Glasgow Haskell Compiler / GHC
Commits:
52037aad by Ben Gamari at 2020-07-05T12:07:41-04:00
testsuite: Allow baseline commit to be set explicitly
- - - - -
020fc776 by Ben Gamari at 2020-07-05T12:07:41-04:00
gitlab-ci: Use MR base commit as performance baseline
- - - - -
7 changed files:
- .gitlab-ci.yml
- hadrian/src/Settings/Builders/RunTest.hs
- testsuite/driver/perf_notes.py
- testsuite/driver/runtests.py
- testsuite/driver/testglobals.py
- testsuite/driver/testlib.py
- testsuite/mk/test.mk
Changes:
=====================================
.gitlab-ci.yml
=====================================
@@ -16,6 +16,9 @@ variables:
GIT_SUBMODULE_STRATEGY: "recursive"
+ # Commit used by testsuite driver as performance baseline.
+ PERF_BASELINE_COMMIT: "$CI_MERGE_REQUEST_TARGET_BRANCH_SHA"
+
stages:
- lint # Source linting
- quick-build # A very quick smoke-test to weed out broken commits
@@ -26,12 +29,26 @@ stages:
- testing # head.hackage correctness and compiler performance testing
- deploy # push documentation
+# Note [The CI Story]
+# ~~~~~~~~~~~~~~~~~~~
+#
+# There are roughly three different types of pipelines:
+#
+# - marge-bot merges to `master`. Here we perform an exhaustive validation
+# across all of the platforms which we support. In addition, we push
+# performance metric notes upstream, providing a persistant record of the
+# performance characteristics of the compiler.
+#
+# - merge requests. Here we perform a slightly less exhaustive battery of
+# testing. Namely we omit some configurations (e.g. the unregisterised job).
+
+
workflow:
- # N.B.Don't run on wip/ branches, instead on run on merge requests.
+ # N.B. Don't run on wip/ branches, instead on run on merge requests.
rules:
- if: $CI_MERGE_REQUEST_ID
- if: $CI_COMMIT_TAG
- - if: '$CI_COMMIT_BRANCH == "master"'
+ - if: '$CI_COMMIT_BRANCH == "wip/marge_bot_batch_merge_job"'
- if: '$CI_COMMIT_BRANCH =~ /ghc-[0.9]+\.[0-9]+/'
- if: '$CI_PIPELINE_SOURCE == "web"'
=====================================
hadrian/src/Settings/Builders/RunTest.hs
=====================================
@@ -78,6 +78,7 @@ runTestBuilderArgs = builder RunTest ? do
<*> (maybe False (=="YES") <$> lookupEnv "OS")
(testEnv, testMetricsFile) <- expr . liftIO $
(,) <$> lookupEnv "TEST_ENV" <*> lookupEnv "METRICS_FILE"
+ perfBaseline <- expr . liftIO $ lookupEnv "PERF_BASELINE_COMMIT"
threads <- shakeThreads <$> expr getShakeOptions
os <- getTestSetting TestHostOS
@@ -141,6 +142,7 @@ runTestBuilderArgs = builder RunTest ? do
, arg "--config", arg $ "timeout_prog=" ++ show (top -/- timeoutProg)
, arg "--config", arg $ "stats_files_dir=" ++ statsFilesDir
, arg $ "--threads=" ++ show threads
+ , emitWhenSet perfBaseline $ \commit -> arg ("--perf-baseline=" ++ show commit)
, emitWhenSet testEnv $ \env -> arg ("--test-env=" ++ show env)
, emitWhenSet testMetricsFile $ \file -> arg ("--metrics-file=" ++ file)
, getTestArgs -- User-provided arguments from command line.
=====================================
testsuite/driver/perf_notes.py
=====================================
@@ -76,8 +76,7 @@ PerfStat = NamedTuple('PerfStat', [('test_env', TestEnv),
# A baseline recovered form stored metrics.
Baseline = NamedTuple('Baseline', [('perfStat', PerfStat),
- ('commit', GitHash),
- ('commitDepth', int)])
+ ('commit', GitHash)])
class MetricChange(Enum):
# The metric appears to have no baseline and is presumably a new test.
@@ -402,7 +401,8 @@ def baseline_metric(commit: GitHash,
name: TestName,
test_env: TestEnv,
metric: MetricName,
- way: WayName
+ way: WayName,
+ baseline_commit: Optional[GitRef]
) -> Optional[Baseline]:
# For performance reasons (in order to avoid calling commit_hash), we assert
# commit is already a commit hash.
@@ -418,11 +418,18 @@ def baseline_metric(commit: GitHash,
def find_baseline(namespace: NoteNamespace,
test_env: TestEnv
) -> Optional[Baseline]:
+ if baseline_commit is not None:
+ current_metric = get_commit_metric(namespace, baseline_commit, test_env, name, metric, way)
+ if current_metric is not None:
+ return Baseline(current_metric, baseline_commit)
+ else:
+ return None
+
for depth, current_commit in list(enumerate(commit_hashes))[1:]:
# Check for a metric on this commit.
current_metric = get_commit_metric(namespace, current_commit, test_env, name, metric, way)
if current_metric is not None:
- return Baseline(current_metric, current_commit, depth)
+ return Baseline(current_metric, current_commit)
# Stop if there is an expected change at this commit. In that case
# metrics on ancestor commits will not be a valid baseline.
@@ -552,7 +559,7 @@ def check_stats_change(actual: PerfStat,
result = passed()
if not change_allowed:
error = str(change) + ' from ' + baseline.perfStat.test_env + \
- ' baseline @ HEAD~' + str(baseline.commitDepth)
+ ' baseline @ %s' % baseline.commit
print(actual.metric, error + ':')
result = failBecause('stat ' + error, tag='stat')
=====================================
testsuite/driver/runtests.py
=====================================
@@ -27,7 +27,7 @@ from testutil import getStdout, Watcher, str_warn, str_info
from testglobals import getConfig, ghc_env, getTestRun, TestConfig, \
TestOptions, brokens, PerfMetric
from my_typing import TestName
-from perf_notes import MetricChange, inside_git_repo, is_worktree_dirty, format_perf_stat
+from perf_notes import MetricChange, GitRef, inside_git_repo, is_worktree_dirty, format_perf_stat
from junit import junit
import term_color
from term_color import Color, colored
@@ -70,6 +70,7 @@ parser.add_argument("--verbose", type=int, choices=[0,1,2,3,4,5], help="verbose
parser.add_argument("--junit", type=argparse.FileType('wb'), help="output testsuite summary in JUnit format")
parser.add_argument("--broken-test", action="append", default=[], help="a test name to mark as broken for this run")
parser.add_argument("--test-env", default='local', help="Override default chosen test-env.")
+parser.add_argument("--perf-baseline", type=GitRef, metavar='COMMIT', help="Baseline commit for performance comparsons.")
perf_group.add_argument("--skip-perf-tests", action="store_true", help="skip performance tests")
perf_group.add_argument("--only-perf-tests", action="store_true", help="Only do performance tests")
@@ -101,6 +102,7 @@ config.metrics_file = args.metrics_file
hasMetricsFile = config.metrics_file is not None
config.summary_file = args.summary_file
config.no_print_summary = args.no_print_summary
+config.baseline_commit = args.perf_baseline
if args.only:
config.only = args.only
@@ -422,6 +424,8 @@ else:
# Dump metrics data.
print("\nPerformance Metrics (test environment: {}):\n".format(config.test_env))
+ if config.baseline_commit:
+ print('Performance baseline: %s\n' % config.baseline_commit)
if any(t.metrics):
tabulate_metrics(t.metrics)
else:
=====================================
testsuite/driver/testglobals.py
=====================================
@@ -4,7 +4,7 @@
from my_typing import *
from pathlib import Path
-from perf_notes import MetricChange, PerfStat, Baseline, MetricOracles
+from perf_notes import MetricChange, PerfStat, Baseline, MetricOracles, GitRef
from datetime import datetime
# -----------------------------------------------------------------------------
@@ -160,6 +160,9 @@ class TestConfig:
# run.
self.broken_tests = set() # type: Set[TestName]
+ # Baseline commit for performane metric comparisons.
+ self.baseline_commit = None # type: Optional[GitRef]
+
# Should we skip performance tests
self.skip_perf_tests = False
=====================================
testsuite/driver/testlib.py
=====================================
@@ -477,7 +477,8 @@ def _collect_stats(name: TestName, opts, metrics, deviation, is_compiler_stats_t
metric = '{}/{}'.format(tag, metric_name)
def baselineByWay(way, target_commit, metric=metric):
return Perf.baseline_metric( \
- target_commit, name, config.test_env, metric, way)
+ target_commit, name, config.test_env, metric, way, \
+ config.baseline_commit )
opts.stats_range_fields[metric] = MetricOracles(baseline=baselineByWay,
deviation=deviation)
=====================================
testsuite/mk/test.mk
=====================================
@@ -226,6 +226,10 @@ ifneq "$(VERBOSE)" ""
RUNTEST_OPTS += --verbose=$(VERBOSE)
endif
+ifneq "$(PERF_TEST_BASELINE_COMMIT)" ""
+RUNTEST_OPTS += --perf-baseline=$(PERF_TEST_BASELINE_COMMIT)
+endif
+
ifeq "$(SKIP_PERF_TESTS)" "YES"
RUNTEST_OPTS += --skip-perf-tests
endif
View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/compare/b0a1fa4f278300fdfa1081ef81d6e8e6056eadf9...020fc7769c33b35855ec20a5fad4961895596ed4
--
View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/compare/b0a1fa4f278300fdfa1081ef81d6e8e6056eadf9...020fc7769c33b35855ec20a5fad4961895596ed4
You're receiving this email because of your account on gitlab.haskell.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mail.haskell.org/pipermail/ghc-commits/attachments/20200705/0bfd45e9/attachment-0001.html>
More information about the ghc-commits
mailing list