[Git][ghc/ghc][wip/perf-ci] 3 commits: gitlab-ci: Add nightly job for running the testsuite with perf profiling support
Hannes Siebenhandl (@fendor)
gitlab at gitlab.haskell.org
Tue May 28 15:51:12 UTC 2024
Hannes Siebenhandl pushed to branch wip/perf-ci at Glasgow Haskell Compiler / GHC
Commits:
3e6607d7 by Fendor at 2024-05-28T17:50:09+02:00
gitlab-ci: Add nightly job for running the testsuite with perf profiling support
- - - - -
58917d26 by Fendor at 2024-05-28T17:50:30+02:00
Enable perf profiling for compiler performance tests
- - - - -
9d6176c4 by Fendor at 2024-05-28T17:50:30+02:00
FIXME: bump ci-images commit to modified version
- - - - -
4 changed files:
- .gitlab-ci.yml
- .gitlab/generate-ci/gen_ci.hs
- .gitlab/jobs.yaml
- testsuite/tests/perf/compiler/all.T
Changes:
=====================================
.gitlab-ci.yml
=====================================
@@ -2,7 +2,7 @@ variables:
GIT_SSL_NO_VERIFY: "1"
# Commit of ghc/ci-images repository from which to pull Docker images
- DOCKER_REV: 064e90c26dffe5709bd5b87dbd211b9a8b21fc5b
+ DOCKER_REV: da1db60cf83e2720c2d347b1cc9128ed330836a9
# Sequential version number of all cached things.
# Bump to invalidate GitLab CI cache.
=====================================
.gitlab/generate-ci/gen_ci.hs
=====================================
@@ -153,6 +153,7 @@ data BuildConfig
, threadSanitiser :: Bool
, noSplitSections :: Bool
, validateNonmovingGc :: Bool
+ , testsuiteUsePerf :: Bool
}
-- Extra arguments to pass to ./configure due to the BuildConfig
@@ -211,6 +212,7 @@ vanilla = BuildConfig
, threadSanitiser = False
, noSplitSections = False
, validateNonmovingGc = False
+ , testsuiteUsePerf = False
}
splitSectionsBroken :: BuildConfig -> BuildConfig
@@ -263,6 +265,9 @@ tsan = vanilla { threadSanitiser = True }
noTntc :: BuildConfig
noTntc = vanilla { tablesNextToCode = False }
+usePerfProfilingTestsuite :: BuildConfig -> BuildConfig
+usePerfProfilingTestsuite bc = bc { testsuiteUsePerf = True }
+
-----------------------------------------------------------------------------
-- Platform specific variables
-----------------------------------------------------------------------------
@@ -283,6 +288,9 @@ runnerTag _ _ = error "Invalid arch/opsys"
tags :: Arch -> Opsys -> BuildConfig -> [String]
tags arch opsys _bc = [runnerTag arch opsys] -- Tag for which runners we can use
+runnerPerfTag :: Arch -> Opsys -> String
+runnerPerfTag arch sys = runnerTag arch sys ++ "-perf"
+
-- These names are used to find the docker image so they have to match what is
-- in the docker registry.
distroName :: LinuxDistro -> String
@@ -761,6 +769,7 @@ job arch opsys buildConfig = NamedJob { name = jobName, jobInfo = Job {..} }
| validateNonmovingGc buildConfig
]
in "RUNTEST_ARGS" =: unwords runtestArgs
+ , if testsuiteUsePerf buildConfig then "RUNTEST_ARGS" =: "--config perf_path=perf" else mempty
]
jobArtifacts = Artifacts
@@ -883,6 +892,12 @@ highCompression = addVariable "XZ_OPT" "-9"
useHashUnitIds :: Job -> Job
useHashUnitIds = addVariable "HADRIAN_ARGS" "--hash-unit-ids"
+-- | Change the tag of the job to make sure the job is scheduled on a
+-- runner that has the necessary capabilties to run the job with 'perf'
+-- profiling counters.
+perfProfilingJobTag :: Arch -> Opsys -> Job -> Job
+perfProfilingJobTag arch opsys j = j { jobTags = [ runnerPerfTag arch opsys ] }
+
-- | Mark the validate job to run in fast-ci mode
-- This is default way, to enable all jobs you have to apply the `full-ci` label.
fastCI :: JobGroup Job -> JobGroup Job
@@ -1022,6 +1037,8 @@ job_groups =
make_wasm_jobs wasm_build_config {unregisterised = True}
, onlyRule NonmovingGc (validateBuilds Amd64 (Linux Debian11) vanilla {validateNonmovingGc = True})
, onlyRule IpeData (validateBuilds Amd64 (Linux Debian10) zstdIpe)
+ -- Run the 'perf' profiling nightly job in the release config.
+ , perfProfilingJob Amd64 (Linux Debian12) releaseConfig
]
where
@@ -1034,6 +1051,12 @@ job_groups =
-- (see Note [Object unloading]).
fullyStaticBrokenTests = modifyJobs (addVariable "BROKEN_TESTS" "ghcilink002 linker_unload_native")
+ perfProfilingJob arch sys buildConfig =
+ -- Rename the job to avoid conflicts
+ rename (<> "-perf")
+ $ modifyJobs (perfProfilingJobTag arch sys)
+ $ disableValidate (standardBuildsWithConfig arch sys $ usePerfProfilingTestsuite buildConfig)
+
hackage_doc_job = rename (<> "-hackage") . modifyJobs (addVariable "HADRIAN_ARGS" "--haddock-for-hackage")
tsan_jobs =
=====================================
.gitlab/jobs.yaml
=====================================
@@ -1895,6 +1895,68 @@
"XZ_OPT": "-9"
}
},
+ "nightly-x86_64-linux-deb12-release-perf": {
+ "after_script": [
+ ".gitlab/ci.sh save_cache",
+ ".gitlab/ci.sh save_test_output",
+ ".gitlab/ci.sh clean",
+ "cat ci_timings"
+ ],
+ "allow_failure": false,
+ "artifacts": {
+ "expire_in": "8 weeks",
+ "paths": [
+ "ghc-x86_64-linux-deb12-release.tar.xz",
+ "junit.xml",
+ "unexpected-test-output.tar.gz"
+ ],
+ "reports": {
+ "junit": "junit.xml"
+ },
+ "when": "always"
+ },
+ "cache": {
+ "key": "x86_64-linux-deb12-$CACHE_REV",
+ "paths": [
+ "cabal-cache",
+ "toolchain"
+ ]
+ },
+ "dependencies": [],
+ "image": "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb12:$DOCKER_REV",
+ "needs": [
+ {
+ "artifacts": false,
+ "job": "hadrian-ghc-in-ghci"
+ }
+ ],
+ "rules": [
+ {
+ "if": "(\"true\" == \"true\") && ($RELEASE_JOB != \"yes\") && ($NIGHTLY)",
+ "when": "on_success"
+ }
+ ],
+ "script": [
+ "sudo chown ghc:ghc -R .",
+ ".gitlab/ci.sh setup",
+ ".gitlab/ci.sh configure",
+ ".gitlab/ci.sh build_hadrian",
+ ".gitlab/ci.sh test_hadrian"
+ ],
+ "stage": "full-build",
+ "tags": [
+ "x86_64-linux-perf"
+ ],
+ "variables": {
+ "BIGNUM_BACKEND": "gmp",
+ "BIN_DIST_NAME": "ghc-x86_64-linux-deb12-release",
+ "BUILD_FLAVOUR": "release",
+ "CONFIGURE_ARGS": "--enable-strict-ghc-toolchain-check",
+ "RUNTEST_ARGS": " --config perf_path=perf",
+ "TEST_ENV": "x86_64-linux-deb12-release",
+ "XZ_OPT": "-9"
+ }
+ },
"nightly-x86_64-linux-deb12-validate": {
"after_script": [
".gitlab/ci.sh save_cache",
@@ -3690,6 +3752,70 @@
"XZ_OPT": "-9"
}
},
+ "release-x86_64-linux-deb12-release-perf": {
+ "after_script": [
+ ".gitlab/ci.sh save_cache",
+ ".gitlab/ci.sh save_test_output",
+ ".gitlab/ci.sh clean",
+ "cat ci_timings"
+ ],
+ "allow_failure": false,
+ "artifacts": {
+ "expire_in": "1 year",
+ "paths": [
+ "ghc-x86_64-linux-deb12-release.tar.xz",
+ "junit.xml",
+ "unexpected-test-output.tar.gz"
+ ],
+ "reports": {
+ "junit": "junit.xml"
+ },
+ "when": "always"
+ },
+ "cache": {
+ "key": "x86_64-linux-deb12-$CACHE_REV",
+ "paths": [
+ "cabal-cache",
+ "toolchain"
+ ]
+ },
+ "dependencies": [],
+ "image": "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb12:$DOCKER_REV",
+ "needs": [
+ {
+ "artifacts": false,
+ "job": "hadrian-ghc-in-ghci"
+ }
+ ],
+ "rules": [
+ {
+ "if": "(\"true\" == \"true\") && ($RELEASE_JOB == \"yes\") && ($NIGHTLY == null)",
+ "when": "on_success"
+ }
+ ],
+ "script": [
+ "sudo chown ghc:ghc -R .",
+ ".gitlab/ci.sh setup",
+ ".gitlab/ci.sh configure",
+ ".gitlab/ci.sh build_hadrian",
+ ".gitlab/ci.sh test_hadrian"
+ ],
+ "stage": "full-build",
+ "tags": [
+ "x86_64-linux-perf"
+ ],
+ "variables": {
+ "BIGNUM_BACKEND": "gmp",
+ "BIN_DIST_NAME": "ghc-x86_64-linux-deb12-release",
+ "BUILD_FLAVOUR": "release",
+ "CONFIGURE_ARGS": "--enable-strict-ghc-toolchain-check",
+ "HADRIAN_ARGS": "--hash-unit-ids",
+ "IGNORE_PERF_FAILURES": "all",
+ "RUNTEST_ARGS": " --config perf_path=perf",
+ "TEST_ENV": "x86_64-linux-deb12-release",
+ "XZ_OPT": "-9"
+ }
+ },
"release-x86_64-linux-deb9-release+no_split_sections": {
"after_script": [
".gitlab/ci.sh save_cache",
=====================================
testsuite/tests/perf/compiler/all.T
=====================================
@@ -8,7 +8,7 @@ test('T1969',
extra_run_opts('+RTS -A64k -RTS'),
# The default RESIDENCY_OPTS is 256k and we need higher sampling
# frequency. Incurs a slow-down by about 2.
- collect_compiler_stats('bytes allocated', 1),
+ collect_compiler_runtime(1),
only_ways(['normal']),
extra_hc_opts('-dcore-lint -static'),
@@ -32,14 +32,14 @@ else:
test('T3294',
[collect_compiler_residency(15),
- collect_compiler_stats('bytes allocated', 1),
+ collect_compiler_runtime(1),
conf_3294,
],
compile,
[''])
test('T4801',
- [collect_compiler_stats('bytes allocated',2),
+ [collect_compiler_runtime(2),
only_ways(['normal']),
extra_hc_opts('-static'),
when(arch('wasm32') and unregisterised(), fragile(23290))
@@ -49,7 +49,7 @@ test('T4801',
test('T3064',
[collect_compiler_residency(20),
- collect_compiler_stats('bytes allocated',2),
+ collect_compiler_runtime(2),
only_ways(['normal']),
],
compile,
@@ -59,7 +59,7 @@ test('T3064',
test('T4007', normal, makefile_test, ['T4007'])
test('T5030',
- [collect_compiler_stats('bytes allocated', 2),
+ [collect_compiler_runtime(2),
only_ways(['normal'])
],
@@ -67,14 +67,14 @@ test('T5030',
['-freduction-depth=300'])
test('T5631',
- [collect_compiler_stats('bytes allocated',2),
+ [collect_compiler_runtime(2),
only_ways(['normal'])
],
compile,
[''])
test('parsing001',
- [collect_compiler_stats('bytes allocated',2),
+ [collect_compiler_runtime(2),
only_ways(['normal']),
],
compile_fail, [''])
@@ -82,27 +82,27 @@ test('parsing001',
test('T783',
[ only_ways(['normal']), # no optimisation for this one
- collect_compiler_stats('bytes allocated',2),
+ collect_compiler_runtime(2),
extra_hc_opts('-static')
],
compile,[''])
test('T5321Fun',
[ only_ways(['normal']), # no optimisation for this one
- collect_compiler_stats('bytes allocated',2)
+ collect_compiler_runtime(2),
],
compile,[''])
test('T5321FD',
[ only_ways(['normal']), # no optimisation for this one
- collect_compiler_stats('bytes allocated',2)
+ collect_compiler_runtime(2),
],
compile,[''])
test('T5642',
[ only_ways(['normal']),
normal,
- collect_compiler_stats('bytes allocated',2)
+ collect_compiler_runtime(2),
],
compile,['-O'])
@@ -114,7 +114,7 @@ test('T5837',
test('T6048',
[ only_ways(['optasm']),
- collect_compiler_stats('bytes allocated',2)
+ collect_compiler_runtime(2),
],
compile,[''])
@@ -134,7 +134,7 @@ test('T9675',
test('T9872a',
[ only_ways(['normal']),
- collect_compiler_stats('bytes allocated', 1),
+ collect_compiler_runtime(1),
high_memory_usage
],
compile_fail,
@@ -142,28 +142,28 @@ test('T9872a',
test('T9872b',
[ only_ways(['normal']),
- collect_compiler_stats('bytes allocated', 1),
+ collect_compiler_runtime(1),
high_memory_usage
],
compile_fail,
[''])
test('T9872b_defer',
[ only_ways(['normal']),
- collect_compiler_stats('bytes allocated', 1),
+ collect_compiler_runtime(1),
high_memory_usage
],
compile,
['-fdefer-type-errors'])
test('T9872c',
[ only_ways(['normal']),
- collect_compiler_stats('bytes allocated', 1),
+ collect_compiler_runtime(1),
high_memory_usage
],
compile_fail,
[''])
test('T9872d',
[ only_ways(['normal']),
- collect_compiler_stats('bytes allocated', 1)
+ collect_compiler_runtime(1)
],
compile,
[''])
@@ -227,14 +227,14 @@ test ('LargeRecord',
test('T9961',
[ only_ways(['normal']),
- collect_compiler_stats('bytes allocated', 1)
+ collect_compiler_runtime(1)
],
compile,
['-O'])
test('T9233',
[ only_ways(['normal']),
- collect_compiler_stats('bytes allocated', 1)
+ collect_compiler_runtime(1)
],
multimod_compile,
['T9233', '-v0 -O2 -fno-spec-constr'])
@@ -249,14 +249,14 @@ test('T10370',
test('T11068', normal, makefile_test, ['T11068'])
test('T10547',
- [ collect_compiler_stats('bytes allocated', 4),
+ [ collect_compiler_runtime(4),
],
compile_fail,
['-fprint-expanded-synonyms'])
test('T12227',
[ only_ways(['normal']),
- collect_compiler_stats('bytes allocated', 1)
+ collect_compiler_runtime(1)
],
compile,
# Use `-M1G` to prevent memory thrashing with ghc-8.0.1.
@@ -264,14 +264,14 @@ test('T12227',
test('T12425',
[ only_ways(['optasm']),
- collect_compiler_stats('bytes allocated', 1)
+ collect_compiler_runtime(1)
],
compile,
[''])
test('T12234',
[ only_ways(['optasm']),
- collect_compiler_stats('bytes allocated', 2),
+ collect_compiler_runtime(2),
],
compile,
[''])
@@ -279,14 +279,14 @@ test('T12234',
# See Note [Sensitivity to unique increment] in T12545.hs; spread was 4.8%
test('T12545',
[ only_ways(['normal']),
- collect_compiler_stats('bytes allocated', 10), #
+ collect_compiler_runtime(10), #
],
multimod_compile,
['T12545', '-v0'] )
test('T13035',
[ only_ways(['normal']),
- collect_compiler_stats('bytes allocated', 1),
+ collect_compiler_runtime(1),
],
compile,
[''] )
@@ -299,7 +299,7 @@ test('T13056',
['-O1'])
test('T12707',
- [ collect_compiler_stats('bytes allocated', 1),
+ [ collect_compiler_runtime(1),
],
compile,
[''])
@@ -311,7 +311,7 @@ test('T12707',
# to avoid spurious errors.
test('T12150',
[ only_ways(['optasm']),
- collect_compiler_stats('bytes allocated', 2)
+ collect_compiler_runtime(2)
],
compile,
[''])
@@ -483,7 +483,7 @@ test('MultiLayerModulesNoCode',
['MultiLayerModulesNoCode.script'])
test('MultiComponentModulesRecomp',
- [ collect_compiler_stats('bytes allocated', 2),
+ [ collect_compiler_runtime(2),
pre_cmd('$MAKE -s --no-print-directory MultiComponentModulesRecomp'),
extra_files(['genMultiComp.py']),
compile_timeout_multiplier(5)
@@ -492,7 +492,7 @@ test('MultiComponentModulesRecomp',
[['unitp%d' % n for n in range(20)], '-fno-code -fwrite-interface -v0'])
test('MultiComponentModules',
- [ collect_compiler_stats('bytes allocated', 2),
+ [ collect_compiler_runtime(2),
pre_cmd('$PYTHON ./genMultiComp.py'),
extra_files(['genMultiComp.py']),
compile_timeout_multiplier(5)
@@ -565,7 +565,7 @@ test('T14683',
test ('T9630',
[ collect_compiler_residency(15),
- collect_compiler_stats('bytes allocated', 2),
+ collect_compiler_runtime(2),
],
multimod_compile,
['T9630', '-v0 -O'])
@@ -611,7 +611,7 @@ test ('T16473',
['-O2 -flate-specialise'])
test('T17516',
- [ collect_compiler_stats('bytes allocated', 5),
+ [ collect_compiler_runtime(5),
],
multimod_compile,
['T17516', '-O -v0'])
@@ -635,13 +635,13 @@ test ('T18140',
['-v0 -O'])
test('T10421',
[ only_ways(['normal']),
- collect_compiler_stats('bytes allocated', 1)
+ collect_compiler_runtime(1)
],
multimod_compile,
['T10421', '-v0 -O'])
test('T10421a',
[ only_ways(['normal']),
- collect_compiler_stats('bytes allocated', 10)
+ collect_compiler_runtime(10)
],
multimod_compile,
['T10421a', '-v0 -O'])
@@ -700,13 +700,13 @@ test ('T19695',
['-v0 -O2'])
test('hard_hole_fits', # Testing multiple hole-fits with lots in scope for #16875
- collect_compiler_stats('bytes allocated', 2), # 1 is 300s, 0.010 is 3s. Without hole-fits it takes 1s
+ collect_compiler_runtime(2), # 1 is 300s, 0.010 is 3s. Without hole-fits it takes 1s
compile, ['-fdefer-type-errors -fno-max-valid-hole-fits -package ghc'])
test('T16875', # Testing one hole-fit with a lot in scope for #16875
# This test is very sensitive to environmental differences.. we should fix
# that but for now the failure threshold is 4% (see #21557)
- collect_compiler_stats('bytes allocated', 4),
+ collect_compiler_runtime(4),
compile, ['-fdefer-type-errors -fno-max-valid-hole-fits -package ghc'])
test ('T20261',
@@ -720,7 +720,7 @@ test ('T20261',
# a compile-time and a run-time performance test
test('T21839c',
[ collect_compiler_stats('all', 10),
- collect_compiler_stats('bytes allocated', 1),
+ collect_compiler_runtime(1),
only_ways(['normal'])],
compile,
['-O'])
View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/compare/514173a7f756dde9193ff887651f217bf8192dfb...9d6176c405ef41729589065f2635a3d95da7a1ba
--
View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/compare/514173a7f756dde9193ff887651f217bf8192dfb...9d6176c405ef41729589065f2635a3d95da7a1ba
You're receiving this email because of your account on gitlab.haskell.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mail.haskell.org/pipermail/ghc-commits/attachments/20240528/d7b60984/attachment-0001.html>
More information about the ghc-commits
mailing list