[Git][ghc/ghc][wip/testsuite-output-patch] 2 commits: testsuite: Allow preservation of unexpected output
Ben Gamari (@bgamari)
gitlab at gitlab.haskell.org
Thu May 25 22:37:24 UTC 2023
Ben Gamari pushed to branch wip/testsuite-output-patch at Glasgow Haskell Compiler / GHC
Commits:
e2e039e1 by Ben Gamari at 2023-05-25T18:37:08-04:00
testsuite: Allow preservation of unexpected output
Here we introduce a new flag to the testsuite driver,
--unexpected-output-dir=<dir>, which allows the user to ask
the driver to preserve unexpected output from tests. The intent is for
this to be used in CI to allow users to more easily fix unexpected
platform-dependent output.
- - - - -
46f435d2 by Ben Gamari at 2023-05-25T18:37:08-04:00
gitlab-ci: Preserve unexpected output
Here we enable use of the testsuite driver's `--unexpected-output-dir`
flag by CI, preserving the result as an artifact for use by users.
- - - - -
5 changed files:
- .gitlab/gen_ci.hs
- .gitlab/jobs.yaml
- testsuite/driver/runtests.py
- testsuite/driver/testglobals.py
- testsuite/driver/testlib.py
Changes:
=====================================
.gitlab/gen_ci.hs
=====================================
@@ -684,16 +684,20 @@ job arch opsys buildConfig = NamedJob { name = jobName, jobInfo = Job {..} }
Emulator s -> "CROSS_EMULATOR" =: s
NoEmulatorNeeded -> mempty
, if withNuma buildConfig then "ENABLE_NUMA" =: "1" else mempty
- , if validateNonmovingGc buildConfig
- then "RUNTEST_ARGS" =: "--way=nonmoving --way=nonmoving_thr --way=nonmoving_thr_sanity"
- else mempty
+ , let runtestArgs =
+ [ "--unexpected-output-dir=./unexpected-output" ] ++
+ [ "--way=nonmoving --way=nonmoving_thr --way=nonmoving_thr_sanity"
+ | validateNonmovingGc buildConfig
+ ]
+ in "RUNTEST_ARGS" =: unwords runtestArgs
]
jobArtifacts = Artifacts
{ junitReport = "junit.xml"
, expireIn = "2 weeks"
, artifactPaths = [binDistName arch opsys buildConfig ++ ".tar.xz"
- ,"junit.xml"]
+ ,"junit.xml"
+ , "unexpected-output"]
, artifactsWhen = ArtifactsAlways
}
=====================================
.gitlab/jobs.yaml
=====================================
@@ -11,7 +11,8 @@
"expire_in": "2 weeks",
"paths": [
"ghc-aarch64-darwin-validate.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -59,6 +60,7 @@
"LANG": "en_US.UTF-8",
"MACOSX_DEPLOYMENT_TARGET": "11.0",
"NIX_SYSTEM": "aarch64-darwin",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "aarch64-darwin-validate"
}
},
@@ -73,7 +75,8 @@
"expire_in": "2 weeks",
"paths": [
"ghc-aarch64-linux-deb10-validate.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -117,6 +120,7 @@
"BIN_DIST_NAME": "ghc-aarch64-linux-deb10-validate",
"BUILD_FLAVOUR": "validate",
"CONFIGURE_ARGS": "",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "aarch64-linux-deb10-validate"
}
},
@@ -131,7 +135,8 @@
"expire_in": "2 weeks",
"paths": [
"ghc-i386-linux-deb9-validate.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -175,6 +180,7 @@
"BIN_DIST_NAME": "ghc-i386-linux-deb9-validate",
"BUILD_FLAVOUR": "validate",
"CONFIGURE_ARGS": "",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "i386-linux-deb9-validate"
}
},
@@ -189,7 +195,8 @@
"expire_in": "8 weeks",
"paths": [
"ghc-aarch64-darwin-validate.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -237,6 +244,7 @@
"LANG": "en_US.UTF-8",
"MACOSX_DEPLOYMENT_TARGET": "11.0",
"NIX_SYSTEM": "aarch64-darwin",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "aarch64-darwin-validate",
"XZ_OPT": "-9"
}
@@ -252,7 +260,8 @@
"expire_in": "8 weeks",
"paths": [
"ghc-aarch64-linux-deb10-validate.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -296,6 +305,7 @@
"BIN_DIST_NAME": "ghc-aarch64-linux-deb10-validate",
"BUILD_FLAVOUR": "validate",
"CONFIGURE_ARGS": "",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "aarch64-linux-deb10-validate",
"XZ_OPT": "-9"
}
@@ -311,7 +321,8 @@
"expire_in": "8 weeks",
"paths": [
"ghc-aarch64-linux-deb10-validate+llvm.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -355,6 +366,7 @@
"BIN_DIST_NAME": "ghc-aarch64-linux-deb10-validate+llvm",
"BUILD_FLAVOUR": "validate+llvm",
"CONFIGURE_ARGS": "",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "aarch64-linux-deb10-validate+llvm",
"XZ_OPT": "-9"
}
@@ -370,7 +382,8 @@
"expire_in": "8 weeks",
"paths": [
"ghc-i386-linux-deb9-validate.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -414,6 +427,7 @@
"BIN_DIST_NAME": "ghc-i386-linux-deb9-validate",
"BUILD_FLAVOUR": "validate",
"CONFIGURE_ARGS": "",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "i386-linux-deb9-validate",
"XZ_OPT": "-9"
}
@@ -429,7 +443,8 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-darwin-validate.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -477,6 +492,7 @@
"LANG": "en_US.UTF-8",
"MACOSX_DEPLOYMENT_TARGET": "10.10",
"NIX_SYSTEM": "x86_64-darwin",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-darwin-validate",
"XZ_OPT": "-9",
"ac_cv_func_clock_gettime": "no",
@@ -494,7 +510,8 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-freebsd13-validate.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -540,6 +557,7 @@
"CONFIGURE_ARGS": "--with-gmp-includes=/usr/local/include --with-gmp-libraries=/usr/local/lib --with-iconv-includes=/usr/local/include --with-iconv-libraries=/usr/local/lib ",
"GHC_VERSION": "9.4.3",
"HADRIAN_ARGS": "--docs=no-sphinx",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-freebsd13-validate",
"XZ_OPT": "-9"
}
@@ -555,7 +573,8 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-alpine3_12-int_native-validate+fully_static.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -602,6 +621,7 @@
"CONFIGURE_ARGS": "--disable-ld-override ",
"HADRIAN_ARGS": "--docs=no-sphinx",
"INSTALL_CONFIGURE_ARGS": "--disable-ld-override",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-linux-alpine3_12-int_native-validate+fully_static",
"XZ_OPT": "-9"
}
@@ -617,7 +637,8 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-alpine3_12-validate.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -664,6 +685,7 @@
"CONFIGURE_ARGS": "--disable-ld-override ",
"HADRIAN_ARGS": "--docs=no-sphinx",
"INSTALL_CONFIGURE_ARGS": "--disable-ld-override",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-linux-alpine3_12-validate",
"XZ_OPT": "-9"
}
@@ -679,7 +701,8 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-alpine3_12-validate+fully_static.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -726,6 +749,7 @@
"CONFIGURE_ARGS": "--disable-ld-override ",
"HADRIAN_ARGS": "--docs=no-sphinx",
"INSTALL_CONFIGURE_ARGS": "--disable-ld-override",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-linux-alpine3_12-validate+fully_static",
"XZ_OPT": "-9"
}
@@ -741,7 +765,8 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-alpine3_17-wasm-cross_wasm32-wasi-release+fully_static.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -787,6 +812,7 @@
"CONFIGURE_ARGS": "--with-intree-gmp --with-system-libffi",
"CROSS_TARGET": "wasm32-wasi",
"HADRIAN_ARGS": "--docs=none",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-linux-alpine3_17-wasm-cross_wasm32-wasi-release+fully_static",
"XZ_OPT": "-9"
}
@@ -802,7 +828,8 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-alpine3_17-wasm-int_native-cross_wasm32-wasi-release+fully_static.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -848,6 +875,7 @@
"CONFIGURE_ARGS": "--with-intree-gmp --with-system-libffi",
"CROSS_TARGET": "wasm32-wasi",
"HADRIAN_ARGS": "--docs=none",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-linux-alpine3_17-wasm-int_native-cross_wasm32-wasi-release+fully_static",
"XZ_OPT": "-9"
}
@@ -863,7 +891,8 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-alpine3_17-wasm-unreg-cross_wasm32-wasi-release+fully_static.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -909,6 +938,7 @@
"CONFIGURE_ARGS": "--enable-unregisterised --with-intree-gmp --with-system-libffi",
"CROSS_TARGET": "wasm32-wasi",
"HADRIAN_ARGS": "--docs=none",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-linux-alpine3_17-wasm-unreg-cross_wasm32-wasi-release+fully_static",
"XZ_OPT": "-9"
}
@@ -924,7 +954,8 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-centos7-validate.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -969,6 +1000,7 @@
"BUILD_FLAVOUR": "validate",
"CONFIGURE_ARGS": "",
"HADRIAN_ARGS": "--docs=no-sphinx",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-linux-centos7-validate",
"XZ_OPT": "-9"
}
@@ -984,7 +1016,8 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-deb10-int_native-validate.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -1028,6 +1061,7 @@
"BIN_DIST_NAME": "ghc-x86_64-linux-deb10-int_native-validate",
"BUILD_FLAVOUR": "validate",
"CONFIGURE_ARGS": "",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-linux-deb10-int_native-validate",
"XZ_OPT": "-9"
}
@@ -1043,7 +1077,8 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-deb10-no_tntc-validate.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -1087,6 +1122,7 @@
"BIN_DIST_NAME": "ghc-x86_64-linux-deb10-no_tntc-validate",
"BUILD_FLAVOUR": "validate",
"CONFIGURE_ARGS": "--disable-tables-next-to-code",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-linux-deb10-no_tntc-validate",
"XZ_OPT": "-9"
}
@@ -1102,7 +1138,8 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-deb10-numa-slow-validate.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -1147,6 +1184,7 @@
"BUILD_FLAVOUR": "slow-validate",
"CONFIGURE_ARGS": "",
"ENABLE_NUMA": "1",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-linux-deb10-numa-slow-validate",
"XZ_OPT": "-9"
}
@@ -1162,7 +1200,8 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-deb10-unreg-validate.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -1206,6 +1245,7 @@
"BIN_DIST_NAME": "ghc-x86_64-linux-deb10-unreg-validate",
"BUILD_FLAVOUR": "validate",
"CONFIGURE_ARGS": "--enable-unregisterised",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-linux-deb10-unreg-validate",
"XZ_OPT": "-9"
}
@@ -1221,7 +1261,8 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-deb10-validate.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -1265,6 +1306,7 @@
"BIN_DIST_NAME": "ghc-x86_64-linux-deb10-validate",
"BUILD_FLAVOUR": "validate",
"CONFIGURE_ARGS": "",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-linux-deb10-validate",
"XZ_OPT": "-9"
}
@@ -1280,7 +1322,8 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-deb10-validate+debug_info.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -1324,6 +1367,7 @@
"BIN_DIST_NAME": "ghc-x86_64-linux-deb10-validate+debug_info",
"BUILD_FLAVOUR": "validate+debug_info",
"CONFIGURE_ARGS": "",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-linux-deb10-validate+debug_info",
"XZ_OPT": "-9"
}
@@ -1339,7 +1383,8 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-deb10-validate+llvm.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -1383,6 +1428,7 @@
"BIN_DIST_NAME": "ghc-x86_64-linux-deb10-validate+llvm",
"BUILD_FLAVOUR": "validate+llvm",
"CONFIGURE_ARGS": "",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-linux-deb10-validate+llvm",
"XZ_OPT": "-9"
}
@@ -1398,7 +1444,8 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-deb10-validate+thread_sanitizer.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -1443,6 +1490,7 @@
"BUILD_FLAVOUR": "validate+thread_sanitizer",
"CONFIGURE_ARGS": "",
"HADRIAN_ARGS": "--docs=none",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-linux-deb10-validate+thread_sanitizer",
"TSAN_OPTIONS": "suppressions=$CI_PROJECT_DIR/rts/.tsan-suppressions",
"XZ_OPT": "-9"
@@ -1459,7 +1507,8 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-deb11-cross_aarch64-linux-gnu-validate.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -1505,6 +1554,7 @@
"CONFIGURE_ARGS": "--with-intree-gmp",
"CROSS_EMULATOR": "qemu-aarch64 -L /usr/aarch64-linux-gnu",
"CROSS_TARGET": "aarch64-linux-gnu",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-linux-deb11-cross_aarch64-linux-gnu-validate",
"XZ_OPT": "-9"
}
@@ -1520,7 +1570,8 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-deb11-int_native-cross_javascript-unknown-ghcjs-validate.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -1567,6 +1618,7 @@
"CONFIGURE_WRAPPER": "emconfigure",
"CROSS_EMULATOR": "js-emulator",
"CROSS_TARGET": "javascript-unknown-ghcjs",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-linux-deb11-int_native-cross_javascript-unknown-ghcjs-validate",
"XZ_OPT": "-9"
}
@@ -1582,7 +1634,8 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-deb11-validate.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -1626,6 +1679,7 @@
"BIN_DIST_NAME": "ghc-x86_64-linux-deb11-validate",
"BUILD_FLAVOUR": "validate",
"CONFIGURE_ARGS": "",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-linux-deb11-validate",
"XZ_OPT": "-9"
}
@@ -1641,7 +1695,8 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-deb11-validate+boot_nonmoving_gc.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -1685,7 +1740,7 @@
"BIN_DIST_NAME": "ghc-x86_64-linux-deb11-validate+boot_nonmoving_gc",
"BUILD_FLAVOUR": "validate+boot_nonmoving_gc",
"CONFIGURE_ARGS": "",
- "RUNTEST_ARGS": "--way=nonmoving --way=nonmoving_thr --way=nonmoving_thr_sanity",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output --way=nonmoving --way=nonmoving_thr --way=nonmoving_thr_sanity",
"TEST_ENV": "x86_64-linux-deb11-validate+boot_nonmoving_gc",
"XZ_OPT": "-9"
}
@@ -1701,7 +1756,8 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-deb9-validate.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -1745,6 +1801,7 @@
"BIN_DIST_NAME": "ghc-x86_64-linux-deb9-validate",
"BUILD_FLAVOUR": "validate",
"CONFIGURE_ARGS": "",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-linux-deb9-validate",
"XZ_OPT": "-9"
}
@@ -1760,7 +1817,8 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-fedora33-release.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -1806,6 +1864,7 @@
"CONFIGURE_ARGS": "",
"LLC": "/bin/false",
"OPT": "/bin/false",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-linux-fedora33-release",
"XZ_OPT": "-9"
}
@@ -1821,7 +1880,8 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-fedora33-release.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -1868,6 +1928,7 @@
"HADRIAN_ARGS": "--haddock-base-url",
"LLC": "/bin/false",
"OPT": "/bin/false",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-linux-fedora33-release",
"XZ_OPT": "-9"
}
@@ -1883,7 +1944,8 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-fedora33-validate+debug_info.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -1929,6 +1991,7 @@
"CONFIGURE_ARGS": "",
"LLC": "/bin/false",
"OPT": "/bin/false",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-linux-fedora33-validate+debug_info",
"XZ_OPT": "-9"
}
@@ -1944,7 +2007,8 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-rocky8-validate.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -1989,6 +2053,7 @@
"BUILD_FLAVOUR": "validate",
"CONFIGURE_ARGS": "",
"HADRIAN_ARGS": "--docs=no-sphinx",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-linux-rocky8-validate",
"XZ_OPT": "-9"
}
@@ -2004,7 +2069,8 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-ubuntu18_04-validate.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -2048,6 +2114,7 @@
"BIN_DIST_NAME": "ghc-x86_64-linux-ubuntu18_04-validate",
"BUILD_FLAVOUR": "validate",
"CONFIGURE_ARGS": "",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-linux-ubuntu18_04-validate",
"XZ_OPT": "-9"
}
@@ -2063,7 +2130,8 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-ubuntu20_04-validate.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -2107,6 +2175,7 @@
"BIN_DIST_NAME": "ghc-x86_64-linux-ubuntu20_04-validate",
"BUILD_FLAVOUR": "validate",
"CONFIGURE_ARGS": "",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-linux-ubuntu20_04-validate",
"XZ_OPT": "-9"
}
@@ -2121,7 +2190,8 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-windows-int_native-validate.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -2166,6 +2236,7 @@
"HADRIAN_ARGS": "--docs=no-sphinx",
"LANG": "en_US.UTF-8",
"MSYSTEM": "CLANG64",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-windows-int_native-validate",
"XZ_OPT": "-9"
}
@@ -2180,7 +2251,8 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-windows-validate.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -2225,6 +2297,7 @@
"HADRIAN_ARGS": "--docs=no-sphinx",
"LANG": "en_US.UTF-8",
"MSYSTEM": "CLANG64",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-windows-validate",
"XZ_OPT": "-9"
}
@@ -2240,7 +2313,8 @@
"expire_in": "1 year",
"paths": [
"ghc-aarch64-darwin-release.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -2289,6 +2363,7 @@
"LANG": "en_US.UTF-8",
"MACOSX_DEPLOYMENT_TARGET": "11.0",
"NIX_SYSTEM": "aarch64-darwin",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "aarch64-darwin-release",
"XZ_OPT": "-9"
}
@@ -2304,7 +2379,8 @@
"expire_in": "1 year",
"paths": [
"ghc-aarch64-linux-deb10-release+no_split_sections.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -2350,6 +2426,7 @@
"CONFIGURE_ARGS": "",
"HADRIAN_ARGS": "--hash-unit-ids",
"IGNORE_PERF_FAILURES": "all",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "aarch64-linux-deb10-release+no_split_sections",
"XZ_OPT": "-9"
}
@@ -2365,7 +2442,8 @@
"expire_in": "1 year",
"paths": [
"ghc-i386-linux-deb9-release+no_split_sections.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -2411,6 +2489,7 @@
"CONFIGURE_ARGS": "",
"HADRIAN_ARGS": "--hash-unit-ids",
"IGNORE_PERF_FAILURES": "all",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "i386-linux-deb9-release+no_split_sections",
"XZ_OPT": "-9"
}
@@ -2426,7 +2505,8 @@
"expire_in": "1 year",
"paths": [
"ghc-x86_64-darwin-release.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -2475,6 +2555,7 @@
"LANG": "en_US.UTF-8",
"MACOSX_DEPLOYMENT_TARGET": "10.10",
"NIX_SYSTEM": "x86_64-darwin",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-darwin-release",
"XZ_OPT": "-9",
"ac_cv_func_clock_gettime": "no",
@@ -2492,7 +2573,8 @@
"expire_in": "1 year",
"paths": [
"ghc-x86_64-linux-alpine3_12-int_native-release+fully_static.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -2540,6 +2622,7 @@
"HADRIAN_ARGS": "--hash-unit-ids --docs=no-sphinx",
"IGNORE_PERF_FAILURES": "all",
"INSTALL_CONFIGURE_ARGS": "--disable-ld-override",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-linux-alpine3_12-int_native-release+fully_static",
"XZ_OPT": "-9"
}
@@ -2555,7 +2638,8 @@
"expire_in": "1 year",
"paths": [
"ghc-x86_64-linux-alpine3_12-release+fully_static+no_split_sections.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -2603,6 +2687,7 @@
"HADRIAN_ARGS": "--hash-unit-ids --docs=no-sphinx",
"IGNORE_PERF_FAILURES": "all",
"INSTALL_CONFIGURE_ARGS": "--disable-ld-override",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-linux-alpine3_12-release+fully_static+no_split_sections",
"XZ_OPT": "-9"
}
@@ -2618,7 +2703,8 @@
"expire_in": "1 year",
"paths": [
"ghc-x86_64-linux-alpine3_12-release+no_split_sections.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -2666,6 +2752,7 @@
"HADRIAN_ARGS": "--hash-unit-ids --docs=no-sphinx",
"IGNORE_PERF_FAILURES": "all",
"INSTALL_CONFIGURE_ARGS": "--disable-ld-override",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-linux-alpine3_12-release+no_split_sections",
"XZ_OPT": "-9"
}
@@ -2681,7 +2768,8 @@
"expire_in": "1 year",
"paths": [
"ghc-x86_64-linux-centos7-release+no_split_sections.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -2727,6 +2815,7 @@
"CONFIGURE_ARGS": "",
"HADRIAN_ARGS": "--hash-unit-ids --docs=no-sphinx",
"IGNORE_PERF_FAILURES": "all",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-linux-centos7-release+no_split_sections",
"XZ_OPT": "-9"
}
@@ -2742,7 +2831,8 @@
"expire_in": "1 year",
"paths": [
"ghc-x86_64-linux-deb10-release.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -2788,6 +2878,7 @@
"CONFIGURE_ARGS": "",
"HADRIAN_ARGS": "--hash-unit-ids",
"IGNORE_PERF_FAILURES": "all",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-linux-deb10-release",
"XZ_OPT": "-9"
}
@@ -2803,7 +2894,8 @@
"expire_in": "1 year",
"paths": [
"ghc-x86_64-linux-deb10-release+debug_info.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -2849,6 +2941,7 @@
"CONFIGURE_ARGS": "",
"HADRIAN_ARGS": "--hash-unit-ids",
"IGNORE_PERF_FAILURES": "all",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-linux-deb10-release+debug_info",
"XZ_OPT": "-9"
}
@@ -2864,7 +2957,8 @@
"expire_in": "1 year",
"paths": [
"ghc-x86_64-linux-deb11-release.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -2910,6 +3004,7 @@
"CONFIGURE_ARGS": "",
"HADRIAN_ARGS": "--hash-unit-ids",
"IGNORE_PERF_FAILURES": "all",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-linux-deb11-release",
"XZ_OPT": "-9"
}
@@ -2925,7 +3020,8 @@
"expire_in": "1 year",
"paths": [
"ghc-x86_64-linux-deb11-release+boot_nonmoving_gc.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -2971,7 +3067,7 @@
"CONFIGURE_ARGS": "",
"HADRIAN_ARGS": "--hash-unit-ids",
"IGNORE_PERF_FAILURES": "all",
- "RUNTEST_ARGS": "--way=nonmoving --way=nonmoving_thr --way=nonmoving_thr_sanity",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output --way=nonmoving --way=nonmoving_thr --way=nonmoving_thr_sanity",
"TEST_ENV": "x86_64-linux-deb11-release+boot_nonmoving_gc",
"XZ_OPT": "-9"
}
@@ -2987,7 +3083,8 @@
"expire_in": "1 year",
"paths": [
"ghc-x86_64-linux-deb9-release+no_split_sections.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -3033,6 +3130,7 @@
"CONFIGURE_ARGS": "",
"HADRIAN_ARGS": "--hash-unit-ids",
"IGNORE_PERF_FAILURES": "all",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-linux-deb9-release+no_split_sections",
"XZ_OPT": "-9"
}
@@ -3048,7 +3146,8 @@
"expire_in": "1 year",
"paths": [
"ghc-x86_64-linux-fedora33-release.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -3096,6 +3195,7 @@
"IGNORE_PERF_FAILURES": "all",
"LLC": "/bin/false",
"OPT": "/bin/false",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-linux-fedora33-release",
"XZ_OPT": "-9"
}
@@ -3111,7 +3211,8 @@
"expire_in": "1 year",
"paths": [
"ghc-x86_64-linux-fedora33-release+debug_info.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -3159,6 +3260,7 @@
"IGNORE_PERF_FAILURES": "all",
"LLC": "/bin/false",
"OPT": "/bin/false",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-linux-fedora33-release+debug_info",
"XZ_OPT": "-9"
}
@@ -3174,7 +3276,8 @@
"expire_in": "1 year",
"paths": [
"ghc-x86_64-linux-fedora33-release.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -3222,6 +3325,7 @@
"IGNORE_PERF_FAILURES": "all",
"LLC": "/bin/false",
"OPT": "/bin/false",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-linux-fedora33-release",
"XZ_OPT": "-9"
}
@@ -3237,7 +3341,8 @@
"expire_in": "1 year",
"paths": [
"ghc-x86_64-linux-rocky8-release.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -3283,6 +3388,7 @@
"CONFIGURE_ARGS": "",
"HADRIAN_ARGS": "--hash-unit-ids --docs=no-sphinx",
"IGNORE_PERF_FAILURES": "all",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-linux-rocky8-release",
"XZ_OPT": "-9"
}
@@ -3298,7 +3404,8 @@
"expire_in": "1 year",
"paths": [
"ghc-x86_64-linux-ubuntu18_04-release.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -3344,6 +3451,7 @@
"CONFIGURE_ARGS": "",
"HADRIAN_ARGS": "--hash-unit-ids",
"IGNORE_PERF_FAILURES": "all",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-linux-ubuntu18_04-release",
"XZ_OPT": "-9"
}
@@ -3359,7 +3467,8 @@
"expire_in": "1 year",
"paths": [
"ghc-x86_64-linux-ubuntu20_04-release.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -3405,6 +3514,7 @@
"CONFIGURE_ARGS": "",
"HADRIAN_ARGS": "--hash-unit-ids",
"IGNORE_PERF_FAILURES": "all",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-linux-ubuntu20_04-release",
"XZ_OPT": "-9"
}
@@ -3419,7 +3529,8 @@
"expire_in": "1 year",
"paths": [
"ghc-x86_64-windows-int_native-release+no_split_sections.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -3465,6 +3576,7 @@
"IGNORE_PERF_FAILURES": "all",
"LANG": "en_US.UTF-8",
"MSYSTEM": "CLANG64",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-windows-int_native-release+no_split_sections",
"XZ_OPT": "-9"
}
@@ -3479,7 +3591,8 @@
"expire_in": "1 year",
"paths": [
"ghc-x86_64-windows-release+no_split_sections.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -3525,6 +3638,7 @@
"IGNORE_PERF_FAILURES": "all",
"LANG": "en_US.UTF-8",
"MSYSTEM": "CLANG64",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-windows-release+no_split_sections",
"XZ_OPT": "-9"
}
@@ -3540,7 +3654,8 @@
"expire_in": "2 weeks",
"paths": [
"ghc-x86_64-darwin-validate.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -3588,6 +3703,7 @@
"LANG": "en_US.UTF-8",
"MACOSX_DEPLOYMENT_TARGET": "10.10",
"NIX_SYSTEM": "x86_64-darwin",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-darwin-validate",
"ac_cv_func_clock_gettime": "no",
"ac_cv_func_utimensat": "no"
@@ -3604,7 +3720,8 @@
"expire_in": "2 weeks",
"paths": [
"ghc-x86_64-freebsd13-validate.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -3650,6 +3767,7 @@
"CONFIGURE_ARGS": "--with-gmp-includes=/usr/local/include --with-gmp-libraries=/usr/local/lib --with-iconv-includes=/usr/local/include --with-iconv-libraries=/usr/local/lib ",
"GHC_VERSION": "9.4.3",
"HADRIAN_ARGS": "--docs=no-sphinx",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-freebsd13-validate"
}
},
@@ -3664,7 +3782,8 @@
"expire_in": "2 weeks",
"paths": [
"ghc-x86_64-linux-alpine3_12-validate+fully_static.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -3711,6 +3830,7 @@
"CONFIGURE_ARGS": "--disable-ld-override ",
"HADRIAN_ARGS": "--docs=no-sphinx",
"INSTALL_CONFIGURE_ARGS": "--disable-ld-override",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-linux-alpine3_12-validate+fully_static"
}
},
@@ -3725,7 +3845,8 @@
"expire_in": "2 weeks",
"paths": [
"ghc-x86_64-linux-alpine3_17-wasm-cross_wasm32-wasi-release+fully_static.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -3771,6 +3892,7 @@
"CONFIGURE_ARGS": "--with-intree-gmp --with-system-libffi",
"CROSS_TARGET": "wasm32-wasi",
"HADRIAN_ARGS": "--docs=none",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-linux-alpine3_17-wasm-cross_wasm32-wasi-release+fully_static"
}
},
@@ -3785,7 +3907,8 @@
"expire_in": "2 weeks",
"paths": [
"ghc-x86_64-linux-alpine3_17-wasm-int_native-cross_wasm32-wasi-release+fully_static.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -3832,6 +3955,7 @@
"CONFIGURE_ARGS": "--with-intree-gmp --with-system-libffi",
"CROSS_TARGET": "wasm32-wasi",
"HADRIAN_ARGS": "--docs=none",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-linux-alpine3_17-wasm-int_native-cross_wasm32-wasi-release+fully_static"
}
},
@@ -3846,7 +3970,8 @@
"expire_in": "2 weeks",
"paths": [
"ghc-x86_64-linux-alpine3_17-wasm-unreg-cross_wasm32-wasi-release+fully_static.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -3893,6 +4018,7 @@
"CONFIGURE_ARGS": "--enable-unregisterised --with-intree-gmp --with-system-libffi",
"CROSS_TARGET": "wasm32-wasi",
"HADRIAN_ARGS": "--docs=none",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-linux-alpine3_17-wasm-unreg-cross_wasm32-wasi-release+fully_static"
}
},
@@ -3907,7 +4033,8 @@
"expire_in": "2 weeks",
"paths": [
"ghc-x86_64-linux-deb10-int_native-validate.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -3951,6 +4078,7 @@
"BIN_DIST_NAME": "ghc-x86_64-linux-deb10-int_native-validate",
"BUILD_FLAVOUR": "validate",
"CONFIGURE_ARGS": "",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-linux-deb10-int_native-validate"
}
},
@@ -3965,7 +4093,8 @@
"expire_in": "2 weeks",
"paths": [
"ghc-x86_64-linux-deb10-no_tntc-validate.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -4010,6 +4139,7 @@
"BIN_DIST_NAME": "ghc-x86_64-linux-deb10-no_tntc-validate",
"BUILD_FLAVOUR": "validate",
"CONFIGURE_ARGS": "--disable-tables-next-to-code",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-linux-deb10-no_tntc-validate"
}
},
@@ -4024,7 +4154,8 @@
"expire_in": "2 weeks",
"paths": [
"ghc-x86_64-linux-deb10-numa-slow-validate.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -4069,6 +4200,7 @@
"BUILD_FLAVOUR": "slow-validate",
"CONFIGURE_ARGS": "",
"ENABLE_NUMA": "1",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-linux-deb10-numa-slow-validate"
}
},
@@ -4083,7 +4215,8 @@
"expire_in": "2 weeks",
"paths": [
"ghc-x86_64-linux-deb10-unreg-validate.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -4127,6 +4260,7 @@
"BIN_DIST_NAME": "ghc-x86_64-linux-deb10-unreg-validate",
"BUILD_FLAVOUR": "validate",
"CONFIGURE_ARGS": "--enable-unregisterised",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-linux-deb10-unreg-validate"
}
},
@@ -4141,7 +4275,8 @@
"expire_in": "2 weeks",
"paths": [
"ghc-x86_64-linux-deb10-validate+debug_info.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -4185,6 +4320,7 @@
"BIN_DIST_NAME": "ghc-x86_64-linux-deb10-validate+debug_info",
"BUILD_FLAVOUR": "validate+debug_info",
"CONFIGURE_ARGS": "",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-linux-deb10-validate+debug_info"
}
},
@@ -4199,7 +4335,8 @@
"expire_in": "2 weeks",
"paths": [
"ghc-x86_64-linux-deb10-validate+llvm.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -4243,6 +4380,7 @@
"BIN_DIST_NAME": "ghc-x86_64-linux-deb10-validate+llvm",
"BUILD_FLAVOUR": "validate+llvm",
"CONFIGURE_ARGS": "",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-linux-deb10-validate+llvm"
}
},
@@ -4257,7 +4395,8 @@
"expire_in": "2 weeks",
"paths": [
"ghc-x86_64-linux-deb10-validate+thread_sanitizer.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -4303,6 +4442,7 @@
"BUILD_FLAVOUR": "validate+thread_sanitizer",
"CONFIGURE_ARGS": "",
"HADRIAN_ARGS": "--docs=none",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-linux-deb10-validate+thread_sanitizer",
"TSAN_OPTIONS": "suppressions=$CI_PROJECT_DIR/rts/.tsan-suppressions"
}
@@ -4318,7 +4458,8 @@
"expire_in": "2 weeks",
"paths": [
"ghc-x86_64-linux-deb11-cross_aarch64-linux-gnu-validate.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -4364,6 +4505,7 @@
"CONFIGURE_ARGS": "--with-intree-gmp",
"CROSS_EMULATOR": "qemu-aarch64 -L /usr/aarch64-linux-gnu",
"CROSS_TARGET": "aarch64-linux-gnu",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-linux-deb11-cross_aarch64-linux-gnu-validate"
}
},
@@ -4378,7 +4520,8 @@
"expire_in": "2 weeks",
"paths": [
"ghc-x86_64-linux-deb11-int_native-cross_javascript-unknown-ghcjs-validate.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -4425,6 +4568,7 @@
"CONFIGURE_WRAPPER": "emconfigure",
"CROSS_EMULATOR": "js-emulator",
"CROSS_TARGET": "javascript-unknown-ghcjs",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-linux-deb11-int_native-cross_javascript-unknown-ghcjs-validate"
}
},
@@ -4439,7 +4583,8 @@
"expire_in": "2 weeks",
"paths": [
"ghc-x86_64-linux-deb11-validate+boot_nonmoving_gc.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -4483,7 +4628,7 @@
"BIN_DIST_NAME": "ghc-x86_64-linux-deb11-validate+boot_nonmoving_gc",
"BUILD_FLAVOUR": "validate+boot_nonmoving_gc",
"CONFIGURE_ARGS": "",
- "RUNTEST_ARGS": "--way=nonmoving --way=nonmoving_thr --way=nonmoving_thr_sanity",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output --way=nonmoving --way=nonmoving_thr --way=nonmoving_thr_sanity",
"TEST_ENV": "x86_64-linux-deb11-validate+boot_nonmoving_gc"
}
},
@@ -4498,7 +4643,8 @@
"expire_in": "2 weeks",
"paths": [
"ghc-x86_64-linux-fedora33-release.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -4544,6 +4690,7 @@
"CONFIGURE_ARGS": "",
"LLC": "/bin/false",
"OPT": "/bin/false",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-linux-fedora33-release"
}
},
@@ -4557,7 +4704,8 @@
"expire_in": "2 weeks",
"paths": [
"ghc-x86_64-windows-validate.tar.xz",
- "junit.xml"
+ "junit.xml",
+ "unexpected-output"
],
"reports": {
"junit": "junit.xml"
@@ -4602,6 +4750,7 @@
"HADRIAN_ARGS": "--docs=no-sphinx",
"LANG": "en_US.UTF-8",
"MSYSTEM": "CLANG64",
+ "RUNTEST_ARGS": "--unexpected-output-dir=./unexpected-output",
"TEST_ENV": "x86_64-windows-validate"
}
}
=====================================
testsuite/driver/runtests.py
=====================================
@@ -71,6 +71,7 @@ parser.add_argument("--config", action='append', help="config field")
parser.add_argument("--rootdir", action='append', help="root of tree containing tests (default: .)")
parser.add_argument("--metrics-file", help="file in which to save (append) the performance test metrics. If omitted, git notes will be used.")
parser.add_argument("--summary-file", help="file in which to save the (human-readable) summary")
+parser.add_argument("--unexpected-output-dir", help="directory in which to place unexpected output")
parser.add_argument("--target-wrapper", help="wrapper executable to use when executing binaries compiled for the target")
parser.add_argument("--no-print-summary", action="store_true", help="should we print the summary?")
parser.add_argument("--only", action="append", help="just this test (can be give multiple --only= flags)")
@@ -128,6 +129,9 @@ if args.top:
if args.test_package_db:
config.test_package_db = args.test_package_db
+if args.unexpected_output_dir:
+ config.unexpected_output_dir = Path(args.unexpected_output_dir)
+
if args.only:
config.only = args.only
config.run_only_some_tests = True
=====================================
testsuite/driver/testglobals.py
=====================================
@@ -203,7 +203,7 @@ class TestConfig:
self.baseline_commit = None # type: Optional[GitRef]
# Additional package dbs to inspect for test dependencies.
- self.test_package_db = [] # type: [PathToPackageDb]
+ self.test_package_db = [] # type: List[PathToPackageDb]
# Should we skip performance tests
self.skip_perf_tests = False
@@ -225,6 +225,10 @@ class TestConfig:
# See Note [Haddock runtime stats files] at the bottom of this file.
self.stats_files_dir = Path('/please_set_stats_files_dir')
+ # Where to place observed output files on when unexpected output
+ # is observed.
+ self.unexpected_output_dir = None # type: Optional[Path]
+
# Should we cleanup after test runs?
self.cleanup = True
=====================================
testsuite/driver/testlib.py
=====================================
@@ -2196,11 +2196,15 @@ async def check_prof_ok(name: TestName, way: WayName) -> bool:
# new output. Returns true if output matched or was accepted, false
# otherwise. See Note [Output comparison] for the meaning of the
# normaliser and whitespace_normaliser parameters.
-async def compare_outputs(way: WayName,
- kind: str,
- normaliser: OutputNormalizer,
- expected_file, actual_file, diff_file=None,
- whitespace_normaliser: OutputNormalizer=lambda x:x) -> bool:
+async def compare_outputs(
+ way: WayName,
+ kind: str,
+ normaliser: OutputNormalizer,
+ expected_file: Path,
+ actual_file: Path,
+ diff_file: Optional[Path]=None,
+ whitespace_normaliser: OutputNormalizer=lambda x:x) -> bool:
+
# Respect ignore_stdout and ignore_stderr options
if kind == 'stderr' and getTestOpts().ignore_stderr:
return True
@@ -2234,6 +2238,11 @@ async def compare_outputs(way: WayName,
if expected_normalised_path != Path(os.devnull):
write_file(expected_normalised_path, expected_str)
+ if config.unexpected_output_dir is not None:
+ out = config.unexpected_output_dir / expected_path.relative_to(config.top)
+ out.parent.mkdir(exist_ok=True, parents=True)
+ write_file(out, expected_str)
+
actual_normalised_path = add_suffix(actual_path, 'normalised')
write_file(actual_normalised_path, actual_str)
@@ -2331,6 +2340,15 @@ def grep_output(normaliser: OutputNormalizer, pattern_file, actual_file, is_subs
# squash all whitespace, making the diff unreadable. Instead we rely
# on the `diff` program to ignore whitespace changes as much as
# possible (#10152).
+#
+# In addition, to aid CI users we will optionally collect all
+# of the unexpected output that we encountered in the
+# directory at config.unexpected_output_dir. The intent here is for this
+# directory to be preserved as a CI artifact which can then
+# be downloaded by the user and committed to their branch
+# to address CI failures on platforms which they cannot
+# test locally.
+
# Note [Null device handling]
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~
View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/compare/c0722dd1938bdd723fd6eb00acd7a9c55cb1f8c6...46f435d2649ea67049db51213fd009b45253f28f
--
View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/compare/c0722dd1938bdd723fd6eb00acd7a9c55cb1f8c6...46f435d2649ea67049db51213fd009b45253f28f
You're receiving this email because of your account on gitlab.haskell.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mail.haskell.org/pipermail/ghc-commits/attachments/20230525/6c363096/attachment-0001.html>
More information about the ghc-commits
mailing list