[Git][ghc/ghc][wip/testsuite-output-patch] 2 commits: testsuite: Allow preservation of unexpected output

Ben Gamari (@bgamari) gitlab at gitlab.haskell.org
Wed May 31 19:09:33 UTC 2023



Ben Gamari pushed to branch wip/testsuite-output-patch at Glasgow Haskell Compiler / GHC


Commits:
e94fff9d by Ben Gamari at 2023-05-31T15:09:19-04:00
testsuite: Allow preservation of unexpected output

Here we introduce a new flag to the testsuite driver,
--unexpected-output-dir=<dir>, which allows the user to ask
the driver to preserve unexpected output from tests. The intent is for
this to be used in CI to allow users to more easily fix unexpected
platform-dependent output.

- - - - -
12dfd808 by Ben Gamari at 2023-05-31T15:09:19-04:00
gitlab-ci: Preserve unexpected output

Here we enable use of the testsuite driver's `--unexpected-output-dir`
flag by CI, preserving the result as an artifact for use by users.

- - - - -


5 changed files:

- .gitlab/gen_ci.hs
- .gitlab/jobs.yaml
- testsuite/driver/runtests.py
- testsuite/driver/testglobals.py
- testsuite/driver/testlib.py


Changes:

=====================================
.gitlab/gen_ci.hs
=====================================
@@ -684,16 +684,20 @@ job arch opsys buildConfig = NamedJob { name = jobName, jobInfo = Job {..} }
           Emulator s       -> "CROSS_EMULATOR" =: s
           NoEmulatorNeeded -> mempty
       , if withNuma buildConfig then "ENABLE_NUMA" =: "1" else mempty
-      , if validateNonmovingGc buildConfig
-           then "RUNTEST_ARGS" =: "--way=nonmoving --way=nonmoving_thr --way=nonmoving_thr_sanity"
-           else mempty
+      , let runtestArgs =
+                [ "--unexpected-output-dir=./unexpected-output" ] ++
+                [ "--way=nonmoving --way=nonmoving_thr --way=nonmoving_thr_sanity"
+                | validateNonmovingGc buildConfig
+                ]
+        in "RUNTEST_ARGS" =: unwords runtestArgs
       ]
 
     jobArtifacts = Artifacts
       { junitReport = "junit.xml"
       , expireIn = "2 weeks"
       , artifactPaths = [binDistName arch opsys buildConfig ++ ".tar.xz"
-                        ,"junit.xml"]
+                        ,"junit.xml"
+                        , "unexpected-output"]
       , artifactsWhen = ArtifactsAlways
       }
 


=====================================
.gitlab/jobs.yaml
=====================================
@@ -11,7 +11,8 @@
       "expire_in": "2 weeks",
       "paths": [
         "ghc-aarch64-darwin-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -59,6 +60,7 @@
       "LANG": "en_US.UTF-8",
       "MACOSX_DEPLOYMENT_TARGET": "11.0",
       "NIX_SYSTEM": "aarch64-darwin",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "aarch64-darwin-validate"
     }
   },
@@ -73,7 +75,8 @@
       "expire_in": "2 weeks",
       "paths": [
         "ghc-aarch64-linux-deb10-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -117,6 +120,7 @@
       "BIN_DIST_NAME": "ghc-aarch64-linux-deb10-validate",
       "BUILD_FLAVOUR": "validate",
       "CONFIGURE_ARGS": "",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "aarch64-linux-deb10-validate"
     }
   },
@@ -131,7 +135,8 @@
       "expire_in": "2 weeks",
       "paths": [
         "ghc-i386-linux-deb9-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -175,6 +180,7 @@
       "BIN_DIST_NAME": "ghc-i386-linux-deb9-validate",
       "BUILD_FLAVOUR": "validate",
       "CONFIGURE_ARGS": "",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "i386-linux-deb9-validate"
     }
   },
@@ -189,7 +195,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-aarch64-darwin-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -237,6 +244,7 @@
       "LANG": "en_US.UTF-8",
       "MACOSX_DEPLOYMENT_TARGET": "11.0",
       "NIX_SYSTEM": "aarch64-darwin",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "aarch64-darwin-validate",
       "XZ_OPT": "-9"
     }
@@ -252,7 +260,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-aarch64-linux-deb10-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -296,6 +305,7 @@
       "BIN_DIST_NAME": "ghc-aarch64-linux-deb10-validate",
       "BUILD_FLAVOUR": "validate",
       "CONFIGURE_ARGS": "",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "aarch64-linux-deb10-validate",
       "XZ_OPT": "-9"
     }
@@ -311,7 +321,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-aarch64-linux-deb10-validate+llvm.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -355,6 +366,7 @@
       "BIN_DIST_NAME": "ghc-aarch64-linux-deb10-validate+llvm",
       "BUILD_FLAVOUR": "validate+llvm",
       "CONFIGURE_ARGS": "",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "aarch64-linux-deb10-validate+llvm",
       "XZ_OPT": "-9"
     }
@@ -370,7 +382,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-i386-linux-deb9-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -414,6 +427,7 @@
       "BIN_DIST_NAME": "ghc-i386-linux-deb9-validate",
       "BUILD_FLAVOUR": "validate",
       "CONFIGURE_ARGS": "",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "i386-linux-deb9-validate",
       "XZ_OPT": "-9"
     }
@@ -429,7 +443,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-x86_64-darwin-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -477,6 +492,7 @@
       "LANG": "en_US.UTF-8",
       "MACOSX_DEPLOYMENT_TARGET": "10.10",
       "NIX_SYSTEM": "x86_64-darwin",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-darwin-validate",
       "XZ_OPT": "-9",
       "ac_cv_func_clock_gettime": "no",
@@ -494,7 +510,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-x86_64-freebsd13-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -540,6 +557,7 @@
       "CONFIGURE_ARGS": "--with-gmp-includes=/usr/local/include --with-gmp-libraries=/usr/local/lib --with-iconv-includes=/usr/local/include --with-iconv-libraries=/usr/local/lib ",
       "GHC_VERSION": "9.4.3",
       "HADRIAN_ARGS": "--docs=no-sphinx",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-freebsd13-validate",
       "XZ_OPT": "-9"
     }
@@ -555,7 +573,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-x86_64-linux-alpine3_12-int_native-validate+fully_static.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -602,6 +621,7 @@
       "CONFIGURE_ARGS": "--disable-ld-override ",
       "HADRIAN_ARGS": "--docs=no-sphinx",
       "INSTALL_CONFIGURE_ARGS": "--disable-ld-override",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-linux-alpine3_12-int_native-validate+fully_static",
       "XZ_OPT": "-9"
     }
@@ -617,7 +637,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-x86_64-linux-alpine3_12-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -664,6 +685,7 @@
       "CONFIGURE_ARGS": "--disable-ld-override ",
       "HADRIAN_ARGS": "--docs=no-sphinx",
       "INSTALL_CONFIGURE_ARGS": "--disable-ld-override",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-linux-alpine3_12-validate",
       "XZ_OPT": "-9"
     }
@@ -679,7 +701,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-x86_64-linux-alpine3_12-validate+fully_static.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -726,6 +749,7 @@
       "CONFIGURE_ARGS": "--disable-ld-override ",
       "HADRIAN_ARGS": "--docs=no-sphinx",
       "INSTALL_CONFIGURE_ARGS": "--disable-ld-override",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-linux-alpine3_12-validate+fully_static",
       "XZ_OPT": "-9"
     }
@@ -741,7 +765,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-x86_64-linux-alpine3_17-wasm-cross_wasm32-wasi-release+fully_static.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -787,6 +812,7 @@
       "CONFIGURE_ARGS": "--with-intree-gmp --with-system-libffi",
       "CROSS_TARGET": "wasm32-wasi",
       "HADRIAN_ARGS": "--docs=none",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-linux-alpine3_17-wasm-cross_wasm32-wasi-release+fully_static",
       "XZ_OPT": "-9"
     }
@@ -802,7 +828,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-x86_64-linux-alpine3_17-wasm-int_native-cross_wasm32-wasi-release+fully_static.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -848,6 +875,7 @@
       "CONFIGURE_ARGS": "--with-intree-gmp --with-system-libffi",
       "CROSS_TARGET": "wasm32-wasi",
       "HADRIAN_ARGS": "--docs=none",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-linux-alpine3_17-wasm-int_native-cross_wasm32-wasi-release+fully_static",
       "XZ_OPT": "-9"
     }
@@ -863,7 +891,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-x86_64-linux-alpine3_17-wasm-unreg-cross_wasm32-wasi-release+fully_static.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -909,6 +938,7 @@
       "CONFIGURE_ARGS": "--enable-unregisterised --with-intree-gmp --with-system-libffi",
       "CROSS_TARGET": "wasm32-wasi",
       "HADRIAN_ARGS": "--docs=none",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-linux-alpine3_17-wasm-unreg-cross_wasm32-wasi-release+fully_static",
       "XZ_OPT": "-9"
     }
@@ -924,7 +954,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-x86_64-linux-centos7-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -969,6 +1000,7 @@
       "BUILD_FLAVOUR": "validate",
       "CONFIGURE_ARGS": "",
       "HADRIAN_ARGS": "--docs=no-sphinx",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-linux-centos7-validate",
       "XZ_OPT": "-9"
     }
@@ -984,7 +1016,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-x86_64-linux-deb10-int_native-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -1028,6 +1061,7 @@
       "BIN_DIST_NAME": "ghc-x86_64-linux-deb10-int_native-validate",
       "BUILD_FLAVOUR": "validate",
       "CONFIGURE_ARGS": "",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-linux-deb10-int_native-validate",
       "XZ_OPT": "-9"
     }
@@ -1043,7 +1077,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-x86_64-linux-deb10-no_tntc-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -1087,6 +1122,7 @@
       "BIN_DIST_NAME": "ghc-x86_64-linux-deb10-no_tntc-validate",
       "BUILD_FLAVOUR": "validate",
       "CONFIGURE_ARGS": "--disable-tables-next-to-code",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-linux-deb10-no_tntc-validate",
       "XZ_OPT": "-9"
     }
@@ -1102,7 +1138,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-x86_64-linux-deb10-numa-slow-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -1147,6 +1184,7 @@
       "BUILD_FLAVOUR": "slow-validate",
       "CONFIGURE_ARGS": "",
       "ENABLE_NUMA": "1",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-linux-deb10-numa-slow-validate",
       "XZ_OPT": "-9"
     }
@@ -1162,7 +1200,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-x86_64-linux-deb10-unreg-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -1206,6 +1245,7 @@
       "BIN_DIST_NAME": "ghc-x86_64-linux-deb10-unreg-validate",
       "BUILD_FLAVOUR": "validate",
       "CONFIGURE_ARGS": "--enable-unregisterised",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-linux-deb10-unreg-validate",
       "XZ_OPT": "-9"
     }
@@ -1221,7 +1261,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-x86_64-linux-deb10-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -1265,6 +1306,7 @@
       "BIN_DIST_NAME": "ghc-x86_64-linux-deb10-validate",
       "BUILD_FLAVOUR": "validate",
       "CONFIGURE_ARGS": "",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-linux-deb10-validate",
       "XZ_OPT": "-9"
     }
@@ -1280,7 +1322,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-x86_64-linux-deb10-validate+debug_info.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -1324,6 +1367,7 @@
       "BIN_DIST_NAME": "ghc-x86_64-linux-deb10-validate+debug_info",
       "BUILD_FLAVOUR": "validate+debug_info",
       "CONFIGURE_ARGS": "",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-linux-deb10-validate+debug_info",
       "XZ_OPT": "-9"
     }
@@ -1339,7 +1383,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-x86_64-linux-deb10-validate+llvm.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -1383,6 +1428,7 @@
       "BIN_DIST_NAME": "ghc-x86_64-linux-deb10-validate+llvm",
       "BUILD_FLAVOUR": "validate+llvm",
       "CONFIGURE_ARGS": "",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-linux-deb10-validate+llvm",
       "XZ_OPT": "-9"
     }
@@ -1398,7 +1444,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-x86_64-linux-deb10-validate+thread_sanitizer.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -1443,6 +1490,7 @@
       "BUILD_FLAVOUR": "validate+thread_sanitizer",
       "CONFIGURE_ARGS": "",
       "HADRIAN_ARGS": "--docs=none",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-linux-deb10-validate+thread_sanitizer",
       "TSAN_OPTIONS": "suppressions=$CI_PROJECT_DIR/rts/.tsan-suppressions",
       "XZ_OPT": "-9"
@@ -1459,7 +1507,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-x86_64-linux-deb11-cross_aarch64-linux-gnu-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -1505,6 +1554,7 @@
       "CONFIGURE_ARGS": "--with-intree-gmp",
       "CROSS_EMULATOR": "qemu-aarch64 -L /usr/aarch64-linux-gnu",
       "CROSS_TARGET": "aarch64-linux-gnu",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-linux-deb11-cross_aarch64-linux-gnu-validate",
       "XZ_OPT": "-9"
     }
@@ -1520,7 +1570,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-x86_64-linux-deb11-int_native-cross_javascript-unknown-ghcjs-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -1567,6 +1618,7 @@
       "CONFIGURE_WRAPPER": "emconfigure",
       "CROSS_EMULATOR": "js-emulator",
       "CROSS_TARGET": "javascript-unknown-ghcjs",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-linux-deb11-int_native-cross_javascript-unknown-ghcjs-validate",
       "XZ_OPT": "-9"
     }
@@ -1582,7 +1634,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-x86_64-linux-deb11-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -1626,6 +1679,7 @@
       "BIN_DIST_NAME": "ghc-x86_64-linux-deb11-validate",
       "BUILD_FLAVOUR": "validate",
       "CONFIGURE_ARGS": "",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-linux-deb11-validate",
       "XZ_OPT": "-9"
     }
@@ -1641,7 +1695,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-x86_64-linux-deb11-validate+boot_nonmoving_gc.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -1685,7 +1740,7 @@
       "BIN_DIST_NAME": "ghc-x86_64-linux-deb11-validate+boot_nonmoving_gc",
       "BUILD_FLAVOUR": "validate+boot_nonmoving_gc",
       "CONFIGURE_ARGS": "",
-      "RUNTEST_ARGS": "--way=nonmoving --way=nonmoving_thr --way=nonmoving_thr_sanity",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output --way=nonmoving --way=nonmoving_thr --way=nonmoving_thr_sanity",
       "TEST_ENV": "x86_64-linux-deb11-validate+boot_nonmoving_gc",
       "XZ_OPT": "-9"
     }
@@ -1701,7 +1756,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-x86_64-linux-deb9-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -1745,6 +1801,7 @@
       "BIN_DIST_NAME": "ghc-x86_64-linux-deb9-validate",
       "BUILD_FLAVOUR": "validate",
       "CONFIGURE_ARGS": "",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-linux-deb9-validate",
       "XZ_OPT": "-9"
     }
@@ -1760,7 +1817,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-x86_64-linux-fedora33-release.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -1806,6 +1864,7 @@
       "CONFIGURE_ARGS": "",
       "LLC": "/bin/false",
       "OPT": "/bin/false",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-linux-fedora33-release",
       "XZ_OPT": "-9"
     }
@@ -1821,7 +1880,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-x86_64-linux-fedora33-release.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -1868,6 +1928,7 @@
       "HADRIAN_ARGS": "--haddock-base-url",
       "LLC": "/bin/false",
       "OPT": "/bin/false",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-linux-fedora33-release",
       "XZ_OPT": "-9"
     }
@@ -1883,7 +1944,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-x86_64-linux-fedora33-validate+debug_info.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -1929,6 +1991,7 @@
       "CONFIGURE_ARGS": "",
       "LLC": "/bin/false",
       "OPT": "/bin/false",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-linux-fedora33-validate+debug_info",
       "XZ_OPT": "-9"
     }
@@ -1944,7 +2007,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-x86_64-linux-rocky8-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -1989,6 +2053,7 @@
       "BUILD_FLAVOUR": "validate",
       "CONFIGURE_ARGS": "",
       "HADRIAN_ARGS": "--docs=no-sphinx",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-linux-rocky8-validate",
       "XZ_OPT": "-9"
     }
@@ -2004,7 +2069,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-x86_64-linux-ubuntu18_04-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -2048,6 +2114,7 @@
       "BIN_DIST_NAME": "ghc-x86_64-linux-ubuntu18_04-validate",
       "BUILD_FLAVOUR": "validate",
       "CONFIGURE_ARGS": "",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-linux-ubuntu18_04-validate",
       "XZ_OPT": "-9"
     }
@@ -2063,7 +2130,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-x86_64-linux-ubuntu20_04-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -2107,6 +2175,7 @@
       "BIN_DIST_NAME": "ghc-x86_64-linux-ubuntu20_04-validate",
       "BUILD_FLAVOUR": "validate",
       "CONFIGURE_ARGS": "",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-linux-ubuntu20_04-validate",
       "XZ_OPT": "-9"
     }
@@ -2121,7 +2190,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-x86_64-windows-int_native-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -2166,6 +2236,7 @@
       "HADRIAN_ARGS": "--docs=no-sphinx",
       "LANG": "en_US.UTF-8",
       "MSYSTEM": "CLANG64",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-windows-int_native-validate",
       "XZ_OPT": "-9"
     }
@@ -2180,7 +2251,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-x86_64-windows-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -2225,6 +2297,7 @@
       "HADRIAN_ARGS": "--docs=no-sphinx",
       "LANG": "en_US.UTF-8",
       "MSYSTEM": "CLANG64",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-windows-validate",
       "XZ_OPT": "-9"
     }
@@ -2240,7 +2313,8 @@
       "expire_in": "1 year",
       "paths": [
         "ghc-aarch64-darwin-release.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -2289,6 +2363,7 @@
       "LANG": "en_US.UTF-8",
       "MACOSX_DEPLOYMENT_TARGET": "11.0",
       "NIX_SYSTEM": "aarch64-darwin",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "aarch64-darwin-release",
       "XZ_OPT": "-9"
     }
@@ -2304,7 +2379,8 @@
       "expire_in": "1 year",
       "paths": [
         "ghc-aarch64-linux-deb10-release+no_split_sections.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -2350,6 +2426,7 @@
       "CONFIGURE_ARGS": "",
       "HADRIAN_ARGS": "--hash-unit-ids",
       "IGNORE_PERF_FAILURES": "all",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "aarch64-linux-deb10-release+no_split_sections",
       "XZ_OPT": "-9"
     }
@@ -2365,7 +2442,8 @@
       "expire_in": "1 year",
       "paths": [
         "ghc-i386-linux-deb9-release+no_split_sections.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -2411,6 +2489,7 @@
       "CONFIGURE_ARGS": "",
       "HADRIAN_ARGS": "--hash-unit-ids",
       "IGNORE_PERF_FAILURES": "all",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "i386-linux-deb9-release+no_split_sections",
       "XZ_OPT": "-9"
     }
@@ -2426,7 +2505,8 @@
       "expire_in": "1 year",
       "paths": [
         "ghc-x86_64-darwin-release.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -2475,6 +2555,7 @@
       "LANG": "en_US.UTF-8",
       "MACOSX_DEPLOYMENT_TARGET": "10.10",
       "NIX_SYSTEM": "x86_64-darwin",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-darwin-release",
       "XZ_OPT": "-9",
       "ac_cv_func_clock_gettime": "no",
@@ -2492,7 +2573,8 @@
       "expire_in": "1 year",
       "paths": [
         "ghc-x86_64-linux-alpine3_12-int_native-release+fully_static.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -2540,6 +2622,7 @@
       "HADRIAN_ARGS": "--hash-unit-ids --docs=no-sphinx",
       "IGNORE_PERF_FAILURES": "all",
       "INSTALL_CONFIGURE_ARGS": "--disable-ld-override",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-linux-alpine3_12-int_native-release+fully_static",
       "XZ_OPT": "-9"
     }
@@ -2555,7 +2638,8 @@
       "expire_in": "1 year",
       "paths": [
         "ghc-x86_64-linux-alpine3_12-release+fully_static+no_split_sections.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -2603,6 +2687,7 @@
       "HADRIAN_ARGS": "--hash-unit-ids --docs=no-sphinx",
       "IGNORE_PERF_FAILURES": "all",
       "INSTALL_CONFIGURE_ARGS": "--disable-ld-override",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-linux-alpine3_12-release+fully_static+no_split_sections",
       "XZ_OPT": "-9"
     }
@@ -2618,7 +2703,8 @@
       "expire_in": "1 year",
       "paths": [
         "ghc-x86_64-linux-alpine3_12-release+no_split_sections.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -2666,6 +2752,7 @@
       "HADRIAN_ARGS": "--hash-unit-ids --docs=no-sphinx",
       "IGNORE_PERF_FAILURES": "all",
       "INSTALL_CONFIGURE_ARGS": "--disable-ld-override",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-linux-alpine3_12-release+no_split_sections",
       "XZ_OPT": "-9"
     }
@@ -2681,7 +2768,8 @@
       "expire_in": "1 year",
       "paths": [
         "ghc-x86_64-linux-centos7-release+no_split_sections.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -2727,6 +2815,7 @@
       "CONFIGURE_ARGS": "",
       "HADRIAN_ARGS": "--hash-unit-ids --docs=no-sphinx",
       "IGNORE_PERF_FAILURES": "all",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-linux-centos7-release+no_split_sections",
       "XZ_OPT": "-9"
     }
@@ -2742,7 +2831,8 @@
       "expire_in": "1 year",
       "paths": [
         "ghc-x86_64-linux-deb10-release.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -2788,6 +2878,7 @@
       "CONFIGURE_ARGS": "",
       "HADRIAN_ARGS": "--hash-unit-ids",
       "IGNORE_PERF_FAILURES": "all",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-linux-deb10-release",
       "XZ_OPT": "-9"
     }
@@ -2803,7 +2894,8 @@
       "expire_in": "1 year",
       "paths": [
         "ghc-x86_64-linux-deb10-release+debug_info.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -2849,6 +2941,7 @@
       "CONFIGURE_ARGS": "",
       "HADRIAN_ARGS": "--hash-unit-ids",
       "IGNORE_PERF_FAILURES": "all",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-linux-deb10-release+debug_info",
       "XZ_OPT": "-9"
     }
@@ -2864,7 +2957,8 @@
       "expire_in": "1 year",
       "paths": [
         "ghc-x86_64-linux-deb11-release.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -2910,6 +3004,7 @@
       "CONFIGURE_ARGS": "",
       "HADRIAN_ARGS": "--hash-unit-ids",
       "IGNORE_PERF_FAILURES": "all",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-linux-deb11-release",
       "XZ_OPT": "-9"
     }
@@ -2925,7 +3020,8 @@
       "expire_in": "1 year",
       "paths": [
         "ghc-x86_64-linux-deb11-release+boot_nonmoving_gc.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -2971,7 +3067,7 @@
       "CONFIGURE_ARGS": "",
       "HADRIAN_ARGS": "--hash-unit-ids",
       "IGNORE_PERF_FAILURES": "all",
-      "RUNTEST_ARGS": "--way=nonmoving --way=nonmoving_thr --way=nonmoving_thr_sanity",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output --way=nonmoving --way=nonmoving_thr --way=nonmoving_thr_sanity",
       "TEST_ENV": "x86_64-linux-deb11-release+boot_nonmoving_gc",
       "XZ_OPT": "-9"
     }
@@ -2987,7 +3083,8 @@
       "expire_in": "1 year",
       "paths": [
         "ghc-x86_64-linux-deb9-release+no_split_sections.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -3033,6 +3130,7 @@
       "CONFIGURE_ARGS": "",
       "HADRIAN_ARGS": "--hash-unit-ids",
       "IGNORE_PERF_FAILURES": "all",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-linux-deb9-release+no_split_sections",
       "XZ_OPT": "-9"
     }
@@ -3048,7 +3146,8 @@
       "expire_in": "1 year",
       "paths": [
         "ghc-x86_64-linux-fedora33-release.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -3096,6 +3195,7 @@
       "IGNORE_PERF_FAILURES": "all",
       "LLC": "/bin/false",
       "OPT": "/bin/false",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-linux-fedora33-release",
       "XZ_OPT": "-9"
     }
@@ -3111,7 +3211,8 @@
       "expire_in": "1 year",
       "paths": [
         "ghc-x86_64-linux-fedora33-release+debug_info.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -3159,6 +3260,7 @@
       "IGNORE_PERF_FAILURES": "all",
       "LLC": "/bin/false",
       "OPT": "/bin/false",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-linux-fedora33-release+debug_info",
       "XZ_OPT": "-9"
     }
@@ -3174,7 +3276,8 @@
       "expire_in": "1 year",
       "paths": [
         "ghc-x86_64-linux-fedora33-release.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -3222,6 +3325,7 @@
       "IGNORE_PERF_FAILURES": "all",
       "LLC": "/bin/false",
       "OPT": "/bin/false",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-linux-fedora33-release",
       "XZ_OPT": "-9"
     }
@@ -3237,7 +3341,8 @@
       "expire_in": "1 year",
       "paths": [
         "ghc-x86_64-linux-rocky8-release.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -3283,6 +3388,7 @@
       "CONFIGURE_ARGS": "",
       "HADRIAN_ARGS": "--hash-unit-ids --docs=no-sphinx",
       "IGNORE_PERF_FAILURES": "all",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-linux-rocky8-release",
       "XZ_OPT": "-9"
     }
@@ -3298,7 +3404,8 @@
       "expire_in": "1 year",
       "paths": [
         "ghc-x86_64-linux-ubuntu18_04-release.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -3344,6 +3451,7 @@
       "CONFIGURE_ARGS": "",
       "HADRIAN_ARGS": "--hash-unit-ids",
       "IGNORE_PERF_FAILURES": "all",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-linux-ubuntu18_04-release",
       "XZ_OPT": "-9"
     }
@@ -3359,7 +3467,8 @@
       "expire_in": "1 year",
       "paths": [
         "ghc-x86_64-linux-ubuntu20_04-release.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -3405,6 +3514,7 @@
       "CONFIGURE_ARGS": "",
       "HADRIAN_ARGS": "--hash-unit-ids",
       "IGNORE_PERF_FAILURES": "all",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-linux-ubuntu20_04-release",
       "XZ_OPT": "-9"
     }
@@ -3419,7 +3529,8 @@
       "expire_in": "1 year",
       "paths": [
         "ghc-x86_64-windows-int_native-release+no_split_sections.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -3465,6 +3576,7 @@
       "IGNORE_PERF_FAILURES": "all",
       "LANG": "en_US.UTF-8",
       "MSYSTEM": "CLANG64",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-windows-int_native-release+no_split_sections",
       "XZ_OPT": "-9"
     }
@@ -3479,7 +3591,8 @@
       "expire_in": "1 year",
       "paths": [
         "ghc-x86_64-windows-release+no_split_sections.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -3525,6 +3638,7 @@
       "IGNORE_PERF_FAILURES": "all",
       "LANG": "en_US.UTF-8",
       "MSYSTEM": "CLANG64",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-windows-release+no_split_sections",
       "XZ_OPT": "-9"
     }
@@ -3540,7 +3654,8 @@
       "expire_in": "2 weeks",
       "paths": [
         "ghc-x86_64-darwin-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -3588,6 +3703,7 @@
       "LANG": "en_US.UTF-8",
       "MACOSX_DEPLOYMENT_TARGET": "10.10",
       "NIX_SYSTEM": "x86_64-darwin",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-darwin-validate",
       "ac_cv_func_clock_gettime": "no",
       "ac_cv_func_utimensat": "no"
@@ -3604,7 +3720,8 @@
       "expire_in": "2 weeks",
       "paths": [
         "ghc-x86_64-freebsd13-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -3650,6 +3767,7 @@
       "CONFIGURE_ARGS": "--with-gmp-includes=/usr/local/include --with-gmp-libraries=/usr/local/lib --with-iconv-includes=/usr/local/include --with-iconv-libraries=/usr/local/lib ",
       "GHC_VERSION": "9.4.3",
       "HADRIAN_ARGS": "--docs=no-sphinx",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-freebsd13-validate"
     }
   },
@@ -3664,7 +3782,8 @@
       "expire_in": "2 weeks",
       "paths": [
         "ghc-x86_64-linux-alpine3_12-validate+fully_static.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -3711,6 +3830,7 @@
       "CONFIGURE_ARGS": "--disable-ld-override ",
       "HADRIAN_ARGS": "--docs=no-sphinx",
       "INSTALL_CONFIGURE_ARGS": "--disable-ld-override",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-linux-alpine3_12-validate+fully_static"
     }
   },
@@ -3725,7 +3845,8 @@
       "expire_in": "2 weeks",
       "paths": [
         "ghc-x86_64-linux-alpine3_17-wasm-cross_wasm32-wasi-release+fully_static.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -3771,6 +3892,7 @@
       "CONFIGURE_ARGS": "--with-intree-gmp --with-system-libffi",
       "CROSS_TARGET": "wasm32-wasi",
       "HADRIAN_ARGS": "--docs=none",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-linux-alpine3_17-wasm-cross_wasm32-wasi-release+fully_static"
     }
   },
@@ -3785,7 +3907,8 @@
       "expire_in": "2 weeks",
       "paths": [
         "ghc-x86_64-linux-alpine3_17-wasm-int_native-cross_wasm32-wasi-release+fully_static.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -3832,6 +3955,7 @@
       "CONFIGURE_ARGS": "--with-intree-gmp --with-system-libffi",
       "CROSS_TARGET": "wasm32-wasi",
       "HADRIAN_ARGS": "--docs=none",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-linux-alpine3_17-wasm-int_native-cross_wasm32-wasi-release+fully_static"
     }
   },
@@ -3846,7 +3970,8 @@
       "expire_in": "2 weeks",
       "paths": [
         "ghc-x86_64-linux-alpine3_17-wasm-unreg-cross_wasm32-wasi-release+fully_static.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -3893,6 +4018,7 @@
       "CONFIGURE_ARGS": "--enable-unregisterised --with-intree-gmp --with-system-libffi",
       "CROSS_TARGET": "wasm32-wasi",
       "HADRIAN_ARGS": "--docs=none",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-linux-alpine3_17-wasm-unreg-cross_wasm32-wasi-release+fully_static"
     }
   },
@@ -3907,7 +4033,8 @@
       "expire_in": "2 weeks",
       "paths": [
         "ghc-x86_64-linux-deb10-int_native-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -3951,6 +4078,7 @@
       "BIN_DIST_NAME": "ghc-x86_64-linux-deb10-int_native-validate",
       "BUILD_FLAVOUR": "validate",
       "CONFIGURE_ARGS": "",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-linux-deb10-int_native-validate"
     }
   },
@@ -3965,7 +4093,8 @@
       "expire_in": "2 weeks",
       "paths": [
         "ghc-x86_64-linux-deb10-no_tntc-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -4010,6 +4139,7 @@
       "BIN_DIST_NAME": "ghc-x86_64-linux-deb10-no_tntc-validate",
       "BUILD_FLAVOUR": "validate",
       "CONFIGURE_ARGS": "--disable-tables-next-to-code",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-linux-deb10-no_tntc-validate"
     }
   },
@@ -4024,7 +4154,8 @@
       "expire_in": "2 weeks",
       "paths": [
         "ghc-x86_64-linux-deb10-numa-slow-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -4069,6 +4200,7 @@
       "BUILD_FLAVOUR": "slow-validate",
       "CONFIGURE_ARGS": "",
       "ENABLE_NUMA": "1",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-linux-deb10-numa-slow-validate"
     }
   },
@@ -4083,7 +4215,8 @@
       "expire_in": "2 weeks",
       "paths": [
         "ghc-x86_64-linux-deb10-unreg-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -4127,6 +4260,7 @@
       "BIN_DIST_NAME": "ghc-x86_64-linux-deb10-unreg-validate",
       "BUILD_FLAVOUR": "validate",
       "CONFIGURE_ARGS": "--enable-unregisterised",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-linux-deb10-unreg-validate"
     }
   },
@@ -4141,7 +4275,8 @@
       "expire_in": "2 weeks",
       "paths": [
         "ghc-x86_64-linux-deb10-validate+debug_info.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -4185,6 +4320,7 @@
       "BIN_DIST_NAME": "ghc-x86_64-linux-deb10-validate+debug_info",
       "BUILD_FLAVOUR": "validate+debug_info",
       "CONFIGURE_ARGS": "",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-linux-deb10-validate+debug_info"
     }
   },
@@ -4199,7 +4335,8 @@
       "expire_in": "2 weeks",
       "paths": [
         "ghc-x86_64-linux-deb10-validate+llvm.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -4243,6 +4380,7 @@
       "BIN_DIST_NAME": "ghc-x86_64-linux-deb10-validate+llvm",
       "BUILD_FLAVOUR": "validate+llvm",
       "CONFIGURE_ARGS": "",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-linux-deb10-validate+llvm"
     }
   },
@@ -4257,7 +4395,8 @@
       "expire_in": "2 weeks",
       "paths": [
         "ghc-x86_64-linux-deb10-validate+thread_sanitizer.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -4303,6 +4442,7 @@
       "BUILD_FLAVOUR": "validate+thread_sanitizer",
       "CONFIGURE_ARGS": "",
       "HADRIAN_ARGS": "--docs=none",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-linux-deb10-validate+thread_sanitizer",
       "TSAN_OPTIONS": "suppressions=$CI_PROJECT_DIR/rts/.tsan-suppressions"
     }
@@ -4318,7 +4458,8 @@
       "expire_in": "2 weeks",
       "paths": [
         "ghc-x86_64-linux-deb11-cross_aarch64-linux-gnu-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -4364,6 +4505,7 @@
       "CONFIGURE_ARGS": "--with-intree-gmp",
       "CROSS_EMULATOR": "qemu-aarch64 -L /usr/aarch64-linux-gnu",
       "CROSS_TARGET": "aarch64-linux-gnu",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-linux-deb11-cross_aarch64-linux-gnu-validate"
     }
   },
@@ -4378,7 +4520,8 @@
       "expire_in": "2 weeks",
       "paths": [
         "ghc-x86_64-linux-deb11-int_native-cross_javascript-unknown-ghcjs-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -4425,6 +4568,7 @@
       "CONFIGURE_WRAPPER": "emconfigure",
       "CROSS_EMULATOR": "js-emulator",
       "CROSS_TARGET": "javascript-unknown-ghcjs",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-linux-deb11-int_native-cross_javascript-unknown-ghcjs-validate"
     }
   },
@@ -4439,7 +4583,8 @@
       "expire_in": "2 weeks",
       "paths": [
         "ghc-x86_64-linux-deb11-validate+boot_nonmoving_gc.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -4483,7 +4628,7 @@
       "BIN_DIST_NAME": "ghc-x86_64-linux-deb11-validate+boot_nonmoving_gc",
       "BUILD_FLAVOUR": "validate+boot_nonmoving_gc",
       "CONFIGURE_ARGS": "",
-      "RUNTEST_ARGS": "--way=nonmoving --way=nonmoving_thr --way=nonmoving_thr_sanity",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output --way=nonmoving --way=nonmoving_thr --way=nonmoving_thr_sanity",
       "TEST_ENV": "x86_64-linux-deb11-validate+boot_nonmoving_gc"
     }
   },
@@ -4498,7 +4643,8 @@
       "expire_in": "2 weeks",
       "paths": [
         "ghc-x86_64-linux-fedora33-release.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -4544,6 +4690,7 @@
       "CONFIGURE_ARGS": "",
       "LLC": "/bin/false",
       "OPT": "/bin/false",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-linux-fedora33-release"
     }
   },
@@ -4557,7 +4704,8 @@
       "expire_in": "2 weeks",
       "paths": [
         "ghc-x86_64-windows-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-output"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -4602,6 +4750,7 @@
       "HADRIAN_ARGS": "--docs=no-sphinx",
       "LANG": "en_US.UTF-8",
       "MSYSTEM": "CLANG64",
+      "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
       "TEST_ENV": "x86_64-windows-validate"
     }
   }


=====================================
testsuite/driver/runtests.py
=====================================
@@ -71,6 +71,7 @@ parser.add_argument("--config", action='append', help="config field")
 parser.add_argument("--rootdir", action='append', help="root of tree containing tests (default: .)")
 parser.add_argument("--metrics-file", help="file in which to save (append) the performance test metrics. If omitted, git notes will be used.")
 parser.add_argument("--summary-file", help="file in which to save the (human-readable) summary")
+parser.add_argument("--unexpected-output-dir", help="directory in which to place unexpected output")
 parser.add_argument("--target-wrapper", help="wrapper executable to use when executing binaries compiled for the target")
 parser.add_argument("--no-print-summary", action="store_true", help="should we print the summary?")
 parser.add_argument("--only", action="append", help="just this test (can be give multiple --only= flags)")
@@ -128,6 +129,9 @@ if args.top:
 if args.test_package_db:
     config.test_package_db = args.test_package_db
 
+if args.unexpected_output_dir:
+    config.unexpected_output_dir = Path(args.unexpected_output_dir)
+
 if args.only:
     config.only = args.only
     config.run_only_some_tests = True


=====================================
testsuite/driver/testglobals.py
=====================================
@@ -203,7 +203,7 @@ class TestConfig:
         self.baseline_commit = None # type: Optional[GitRef]
 
         # Additional package dbs to inspect for test dependencies.
-        self.test_package_db = [] # type: [PathToPackageDb]
+        self.test_package_db = [] # type: List[PathToPackageDb]
 
         # Should we skip performance tests
         self.skip_perf_tests = False
@@ -225,6 +225,10 @@ class TestConfig:
         # See Note [Haddock runtime stats files] at the bottom of this file.
         self.stats_files_dir = Path('/please_set_stats_files_dir')
 
+        # Where to place observed output files on when unexpected output
+        # is observed.
+        self.unexpected_output_dir = None # type: Optional[Path]
+
         # Should we cleanup after test runs?
         self.cleanup = True
 


=====================================
testsuite/driver/testlib.py
=====================================
@@ -2196,11 +2196,15 @@ async def check_prof_ok(name: TestName, way: WayName) -> bool:
 # new output. Returns true if output matched or was accepted, false
 # otherwise. See Note [Output comparison] for the meaning of the
 # normaliser and whitespace_normaliser parameters.
-async def compare_outputs(way: WayName,
-                    kind: str,
-                    normaliser: OutputNormalizer,
-                    expected_file, actual_file, diff_file=None,
-                    whitespace_normaliser: OutputNormalizer=lambda x:x) -> bool:
+async def compare_outputs(
+        way: WayName,
+        kind: str,
+        normaliser: OutputNormalizer,
+        expected_file: Path,
+        actual_file: Path,
+        diff_file: Optional[Path]=None,
+        whitespace_normaliser: OutputNormalizer=lambda x:x) -> bool:
+
     # Respect ignore_stdout and ignore_stderr options
     if kind == 'stderr' and getTestOpts().ignore_stderr:
         return True
@@ -2277,6 +2281,12 @@ async def compare_outputs(way: WayName,
             expected_path.unlink()
             return True
         else:
+            if config.unexpected_output_dir is not None:
+                ghc_root = expected_path.relative_to(config.top.parent)
+                out = config.unexpected_output_dir / ghc_root
+                out.parent.mkdir(exist_ok=True, parents=True)
+                write_file(out, expected_str)
+
             return False
 
 # Checks that each line from pattern_file is present in actual_file as
@@ -2331,6 +2341,15 @@ def grep_output(normaliser: OutputNormalizer, pattern_file, actual_file, is_subs
 #    squash all whitespace, making the diff unreadable. Instead we rely
 #    on the `diff` program to ignore whitespace changes as much as
 #    possible (#10152).
+#
+# In addition, to aid CI users we will optionally collect all
+# of the unexpected output that we encountered in the
+# directory at config.unexpected_output_dir. The intent here is for this
+# directory to be preserved as a CI artifact which can then
+# be downloaded by the user and committed to their branch
+# to address CI failures on platforms which they cannot
+# test locally.
+
 
 # Note [Null device handling]
 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~



View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/compare/971e82f43ab79bb6631b3129afa78ed3ff586d21...12dfd80847136b182fad201a461480c81a2f514c

-- 
View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/compare/971e82f43ab79bb6631b3129afa78ed3ff586d21...12dfd80847136b182fad201a461480c81a2f514c
You're receiving this email because of your account on gitlab.haskell.org.


-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mail.haskell.org/pipermail/ghc-commits/attachments/20230531/2095ced4/attachment-0001.html>


More information about the ghc-commits mailing list