[Git][ghc/ghc][wip/perf-ci] 3 commits: gitlab-ci: Add nightly job for running the testsuite with perf profiling support

Hannes Siebenhandl (@fendor) gitlab at gitlab.haskell.org
Wed May 29 11:08:23 UTC 2024



Hannes Siebenhandl pushed to branch wip/perf-ci at Glasgow Haskell Compiler / GHC


Commits:
9feba70b by Fendor at 2024-05-29T13:08:07+02:00
gitlab-ci: Add nightly job for running the testsuite with perf profiling support

- - - - -
bab3d134 by Fendor at 2024-05-29T13:08:07+02:00
Enable perf profiling for compiler performance tests

- - - - -
69562e32 by Fendor at 2024-05-29T13:08:07+02:00
FIXME: bump ci-images commit to modified version

- - - - -


4 changed files:

- .gitlab-ci.yml
- .gitlab/generate-ci/gen_ci.hs
- .gitlab/jobs.yaml
- testsuite/tests/perf/compiler/all.T


Changes:

=====================================
.gitlab-ci.yml
=====================================
@@ -2,7 +2,7 @@ variables:
   GIT_SSL_NO_VERIFY: "1"
 
   # Commit of ghc/ci-images repository from which to pull Docker images
-  DOCKER_REV: 064e90c26dffe5709bd5b87dbd211b9a8b21fc5b
+  DOCKER_REV: da1db60cf83e2720c2d347b1cc9128ed330836a9
 
   # Sequential version number of all cached things.
   # Bump to invalidate GitLab CI cache.


=====================================
.gitlab/generate-ci/gen_ci.hs
=====================================
@@ -153,6 +153,7 @@ data BuildConfig
                 , threadSanitiser :: Bool
                 , noSplitSections :: Bool
                 , validateNonmovingGc :: Bool
+                , testsuiteUsePerf :: Bool
                 }
 
 -- Extra arguments to pass to ./configure due to the BuildConfig
@@ -211,6 +212,7 @@ vanilla = BuildConfig
   , threadSanitiser = False
   , noSplitSections = False
   , validateNonmovingGc = False
+  , testsuiteUsePerf = False
   }
 
 splitSectionsBroken :: BuildConfig -> BuildConfig
@@ -263,6 +265,9 @@ tsan = vanilla { threadSanitiser = True }
 noTntc :: BuildConfig
 noTntc = vanilla { tablesNextToCode = False }
 
+usePerfProfilingTestsuite :: BuildConfig -> BuildConfig
+usePerfProfilingTestsuite bc = bc { testsuiteUsePerf = True }
+
 -----------------------------------------------------------------------------
 -- Platform specific variables
 -----------------------------------------------------------------------------
@@ -283,6 +288,9 @@ runnerTag _ _ = error "Invalid arch/opsys"
 tags :: Arch -> Opsys -> BuildConfig -> [String]
 tags arch opsys _bc = [runnerTag arch opsys] -- Tag for which runners we can use
 
+runnerPerfTag :: Arch -> Opsys -> String
+runnerPerfTag arch sys = runnerTag arch sys ++ "-perf"
+
 -- These names are used to find the docker image so they have to match what is
 -- in the docker registry.
 distroName :: LinuxDistro -> String
@@ -761,6 +769,7 @@ job arch opsys buildConfig = NamedJob { name = jobName, jobInfo = Job {..} }
                 | validateNonmovingGc buildConfig
                 ]
         in "RUNTEST_ARGS" =: unwords runtestArgs
+      , if testsuiteUsePerf buildConfig then "RUNTEST_ARGS" =: "--config perf_path=perf" else mempty
       ]
 
     jobArtifacts = Artifacts
@@ -883,6 +892,12 @@ highCompression = addVariable "XZ_OPT" "-9"
 useHashUnitIds :: Job -> Job
 useHashUnitIds = addVariable "HADRIAN_ARGS" "--hash-unit-ids"
 
+-- | Change the tag of the job to make sure the job is scheduled on a
+-- runner that has the necessary capabilties to run the job with 'perf'
+-- profiling counters.
+perfProfilingJobTag :: Arch -> Opsys -> Job -> Job
+perfProfilingJobTag arch opsys j = j { jobTags = [ runnerPerfTag arch opsys ] }
+
 -- | Mark the validate job to run in fast-ci mode
 -- This is default way, to enable all jobs you have to apply the `full-ci` label.
 fastCI :: JobGroup Job -> JobGroup Job
@@ -1022,6 +1037,8 @@ job_groups =
          make_wasm_jobs wasm_build_config {unregisterised = True}
      , onlyRule NonmovingGc (validateBuilds Amd64 (Linux Debian11) vanilla {validateNonmovingGc = True})
      , onlyRule IpeData (validateBuilds Amd64 (Linux Debian10) zstdIpe)
+     -- Run the 'perf' profiling nightly job in the release config.
+     , perfProfilingJob Amd64 (Linux Debian12) releaseConfig
      ]
 
   where
@@ -1034,6 +1051,12 @@ job_groups =
     -- (see Note [Object unloading]).
     fullyStaticBrokenTests = modifyJobs (addVariable "BROKEN_TESTS" "ghcilink002 linker_unload_native")
 
+    perfProfilingJob arch sys buildConfig =
+        -- Rename the job to avoid conflicts
+        rename (<> "-perf")
+          $ modifyJobs (perfProfilingJobTag arch sys)
+          $ disableValidate (validateBuilds arch sys $ usePerfProfilingTestsuite buildConfig)
+
     hackage_doc_job = rename (<> "-hackage") . modifyJobs (addVariable "HADRIAN_ARGS" "--haddock-for-hackage")
 
     tsan_jobs =


=====================================
.gitlab/jobs.yaml
=====================================
@@ -1895,6 +1895,68 @@
       "XZ_OPT": "-9"
     }
   },
+  "nightly-x86_64-linux-deb12-release-perf": {
+    "after_script": [
+      ".gitlab/ci.sh save_cache",
+      ".gitlab/ci.sh save_test_output",
+      ".gitlab/ci.sh clean",
+      "cat ci_timings"
+    ],
+    "allow_failure": false,
+    "artifacts": {
+      "expire_in": "8 weeks",
+      "paths": [
+        "ghc-x86_64-linux-deb12-release.tar.xz",
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
+      ],
+      "reports": {
+        "junit": "junit.xml"
+      },
+      "when": "always"
+    },
+    "cache": {
+      "key": "x86_64-linux-deb12-$CACHE_REV",
+      "paths": [
+        "cabal-cache",
+        "toolchain"
+      ]
+    },
+    "dependencies": [],
+    "image": "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb12:$DOCKER_REV",
+    "needs": [
+      {
+        "artifacts": false,
+        "job": "hadrian-ghc-in-ghci"
+      }
+    ],
+    "rules": [
+      {
+        "if": "(\"true\" == \"true\") && ($RELEASE_JOB != \"yes\") && ($NIGHTLY)",
+        "when": "on_success"
+      }
+    ],
+    "script": [
+      "sudo chown ghc:ghc -R .",
+      ".gitlab/ci.sh setup",
+      ".gitlab/ci.sh configure",
+      ".gitlab/ci.sh build_hadrian",
+      ".gitlab/ci.sh test_hadrian"
+    ],
+    "stage": "full-build",
+    "tags": [
+      "x86_64-linux-perf"
+    ],
+    "variables": {
+      "BIGNUM_BACKEND": "gmp",
+      "BIN_DIST_NAME": "ghc-x86_64-linux-deb12-release",
+      "BUILD_FLAVOUR": "release",
+      "CONFIGURE_ARGS": "--enable-strict-ghc-toolchain-check",
+      "RUNTEST_ARGS": " --config perf_path=perf",
+      "TEST_ENV": "x86_64-linux-deb12-release",
+      "XZ_OPT": "-9"
+    }
+  },
   "nightly-x86_64-linux-deb12-validate": {
     "after_script": [
       ".gitlab/ci.sh save_cache",


=====================================
testsuite/tests/perf/compiler/all.T
=====================================
@@ -8,7 +8,7 @@ test('T1969',
       extra_run_opts('+RTS -A64k -RTS'),
           # The default RESIDENCY_OPTS is 256k and we need higher sampling
           # frequency. Incurs a slow-down by about 2.
-      collect_compiler_stats('bytes allocated', 1),
+      collect_compiler_runtime(1),
       only_ways(['normal']),
 
       extra_hc_opts('-dcore-lint -static'),
@@ -32,14 +32,14 @@ else:
 
 test('T3294',
      [collect_compiler_residency(15),
-      collect_compiler_stats('bytes allocated', 1),
+      collect_compiler_runtime(1),
       conf_3294,
       ],
      compile,
      [''])
 
 test('T4801',
-     [collect_compiler_stats('bytes allocated',2),
+     [collect_compiler_runtime(2),
       only_ways(['normal']),
       extra_hc_opts('-static'),
       when(arch('wasm32') and unregisterised(), fragile(23290))
@@ -49,7 +49,7 @@ test('T4801',
 
 test('T3064',
      [collect_compiler_residency(20),
-      collect_compiler_stats('bytes allocated',2),
+      collect_compiler_runtime(2),
       only_ways(['normal']),
       ],
      compile,
@@ -59,7 +59,7 @@ test('T3064',
 test('T4007', normal, makefile_test, ['T4007'])
 
 test('T5030',
-     [collect_compiler_stats('bytes allocated', 2),
+     [collect_compiler_runtime(2),
 
        only_ways(['normal'])
       ],
@@ -67,14 +67,14 @@ test('T5030',
      ['-freduction-depth=300'])
 
 test('T5631',
-     [collect_compiler_stats('bytes allocated',2),
+     [collect_compiler_runtime(2),
       only_ways(['normal'])
       ],
      compile,
      [''])
 
 test('parsing001',
-     [collect_compiler_stats('bytes allocated',2),
+     [collect_compiler_runtime(2),
        only_ways(['normal']),
       ],
      compile_fail, [''])
@@ -82,27 +82,27 @@ test('parsing001',
 
 test('T783',
      [ only_ways(['normal']),  # no optimisation for this one
-      collect_compiler_stats('bytes allocated',2),
+      collect_compiler_runtime(2),
       extra_hc_opts('-static')
       ],
       compile,[''])
 
 test('T5321Fun',
      [ only_ways(['normal']),  # no optimisation for this one
-       collect_compiler_stats('bytes allocated',2)
+       collect_compiler_runtime(2),
       ],
       compile,[''])
 
 test('T5321FD',
      [ only_ways(['normal']),  # no optimisation for this one
-      collect_compiler_stats('bytes allocated',2)
+      collect_compiler_runtime(2),
       ],
       compile,[''])
 
 test('T5642',
      [ only_ways(['normal']),
        normal,
-       collect_compiler_stats('bytes allocated',2)
+       collect_compiler_runtime(2),
       ],
       compile,['-O'])
 
@@ -114,7 +114,7 @@ test('T5837',
 
 test('T6048',
      [ only_ways(['optasm']),
-      collect_compiler_stats('bytes allocated',2)
+      collect_compiler_runtime(2),
       ],
       compile,[''])
 
@@ -134,7 +134,7 @@ test('T9675',
 
 test('T9872a',
      [ only_ways(['normal']),
-       collect_compiler_stats('bytes allocated', 1),
+       collect_compiler_runtime(1),
        high_memory_usage
       ],
      compile_fail,
@@ -142,28 +142,28 @@ test('T9872a',
 
 test('T9872b',
      [ only_ways(['normal']),
-       collect_compiler_stats('bytes allocated', 1),
+       collect_compiler_runtime(1),
        high_memory_usage
       ],
      compile_fail,
      [''])
 test('T9872b_defer',
      [ only_ways(['normal']),
-       collect_compiler_stats('bytes allocated', 1),
+       collect_compiler_runtime(1),
        high_memory_usage
       ],
      compile,
      ['-fdefer-type-errors'])
 test('T9872c',
      [ only_ways(['normal']),
-       collect_compiler_stats('bytes allocated', 1),
+       collect_compiler_runtime(1),
        high_memory_usage
       ],
      compile_fail,
      [''])
 test('T9872d',
      [ only_ways(['normal']),
-       collect_compiler_stats('bytes allocated', 1)
+       collect_compiler_runtime(1)
       ],
      compile,
      [''])
@@ -227,14 +227,14 @@ test ('LargeRecord',
 
 test('T9961',
      [ only_ways(['normal']),
-       collect_compiler_stats('bytes allocated', 1)
+       collect_compiler_runtime(1)
       ],
      compile,
      ['-O'])
 
 test('T9233',
     [ only_ways(['normal']),
-      collect_compiler_stats('bytes allocated', 1)
+      collect_compiler_runtime(1)
     ],
     multimod_compile,
     ['T9233', '-v0 -O2 -fno-spec-constr'])
@@ -249,14 +249,14 @@ test('T10370',
 test('T11068', normal, makefile_test, ['T11068'])
 
 test('T10547',
-     [ collect_compiler_stats('bytes allocated', 4),
+     [ collect_compiler_runtime(4),
      ],
      compile_fail,
      ['-fprint-expanded-synonyms'])
 
 test('T12227',
      [ only_ways(['normal']),
-       collect_compiler_stats('bytes allocated', 1)
+       collect_compiler_runtime(1)
      ],
      compile,
      # Use `-M1G` to prevent memory thrashing with ghc-8.0.1.
@@ -264,14 +264,14 @@ test('T12227',
 
 test('T12425',
      [ only_ways(['optasm']),
-       collect_compiler_stats('bytes allocated', 1)
+       collect_compiler_runtime(1)
      ],
      compile,
      [''])
 
 test('T12234',
      [ only_ways(['optasm']),
-       collect_compiler_stats('bytes allocated', 2),
+       collect_compiler_runtime(2),
      ],
      compile,
      [''])
@@ -279,14 +279,14 @@ test('T12234',
 # See Note [Sensitivity to unique increment] in T12545.hs; spread was 4.8%
 test('T12545',
      [ only_ways(['normal']),
-       collect_compiler_stats('bytes allocated', 10), #
+       collect_compiler_runtime(10), #
      ],
      multimod_compile,
      ['T12545', '-v0'] )
 
 test('T13035',
      [ only_ways(['normal']),
-       collect_compiler_stats('bytes allocated', 1),
+       collect_compiler_runtime(1),
      ],
      compile,
      [''] )
@@ -299,7 +299,7 @@ test('T13056',
      ['-O1'])
 
 test('T12707',
-     [ collect_compiler_stats('bytes allocated', 1),
+     [ collect_compiler_runtime(1),
      ],
      compile,
      [''])
@@ -311,7 +311,7 @@ test('T12707',
 # to avoid spurious errors.
 test('T12150',
      [ only_ways(['optasm']),
-       collect_compiler_stats('bytes allocated', 2)
+       collect_compiler_runtime(2)
      ],
     compile,
      [''])
@@ -483,7 +483,7 @@ test('MultiLayerModulesNoCode',
      ['MultiLayerModulesNoCode.script'])
 
 test('MultiComponentModulesRecomp',
-     [ collect_compiler_stats('bytes allocated', 2),
+     [ collect_compiler_runtime(2),
        pre_cmd('$MAKE -s --no-print-directory MultiComponentModulesRecomp'),
        extra_files(['genMultiComp.py']),
        compile_timeout_multiplier(5)
@@ -492,7 +492,7 @@ test('MultiComponentModulesRecomp',
      [['unitp%d' % n for n in range(20)], '-fno-code -fwrite-interface -v0'])
 
 test('MultiComponentModules',
-     [ collect_compiler_stats('bytes allocated', 2),
+     [ collect_compiler_runtime(2),
        pre_cmd('$PYTHON ./genMultiComp.py'),
        extra_files(['genMultiComp.py']),
        compile_timeout_multiplier(5)
@@ -565,7 +565,7 @@ test('T14683',
 
 test ('T9630',
       [ collect_compiler_residency(15),
-        collect_compiler_stats('bytes allocated', 2),
+        collect_compiler_runtime(2),
       ],
       multimod_compile,
       ['T9630', '-v0 -O'])
@@ -611,7 +611,7 @@ test ('T16473',
       ['-O2 -flate-specialise'])
 
 test('T17516',
-      [ collect_compiler_stats('bytes allocated', 5),
+      [ collect_compiler_runtime(5),
       ],
       multimod_compile,
       ['T17516', '-O -v0'])
@@ -635,13 +635,13 @@ test ('T18140',
       ['-v0 -O'])
 test('T10421',
     [ only_ways(['normal']),
-      collect_compiler_stats('bytes allocated', 1)
+      collect_compiler_runtime(1)
     ],
     multimod_compile,
     ['T10421', '-v0 -O'])
 test('T10421a',
     [ only_ways(['normal']),
-      collect_compiler_stats('bytes allocated', 10)
+      collect_compiler_runtime(10)
     ],
     multimod_compile,
     ['T10421a', '-v0 -O'])
@@ -700,13 +700,13 @@ test ('T19695',
       ['-v0 -O2'])
 
 test('hard_hole_fits', # Testing multiple hole-fits with lots in scope for #16875
-     collect_compiler_stats('bytes allocated', 2), # 1 is 300s, 0.010 is 3s. Without  hole-fits it takes 1s
+     collect_compiler_runtime(2), # 1 is 300s, 0.010 is 3s. Without  hole-fits it takes 1s
      compile, ['-fdefer-type-errors -fno-max-valid-hole-fits -package ghc'])
 
 test('T16875',  # Testing one hole-fit with a lot in scope for #16875
      # This test is very sensitive to environmental differences.. we should fix
      # that but for now the failure threshold is 4% (see #21557)
-     collect_compiler_stats('bytes allocated', 4),
+     collect_compiler_runtime(4),
      compile, ['-fdefer-type-errors -fno-max-valid-hole-fits -package ghc'])
 
 test ('T20261',
@@ -720,7 +720,7 @@ test ('T20261',
 # a compile-time and a run-time performance test
 test('T21839c',
     [   collect_compiler_stats('all', 10),
-        collect_compiler_stats('bytes allocated', 1),
+        collect_compiler_runtime(1),
         only_ways(['normal'])],
     compile,
     ['-O'])



View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/compare/844befec78f40a0b6046855404d9d35d4984e071...69562e324d128f44341a3364ce9b777d27b52f3d

-- 
View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/compare/844befec78f40a0b6046855404d9d35d4984e071...69562e324d128f44341a3364ce9b777d27b52f3d
You're receiving this email because of your account on gitlab.haskell.org.


-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mail.haskell.org/pipermail/ghc-commits/attachments/20240529/11ea5a25/attachment-0001.html>


More information about the ghc-commits mailing list