[Git][ghc/ghc][wip/testsuite-output-patch] 4 commits: gitlab-ci: Refactor compilation of gen_ci

Ben Gamari (@bgamari) gitlab at gitlab.haskell.org
Thu Jun 29 01:42:16 UTC 2023



Ben Gamari pushed to branch wip/testsuite-output-patch at Glasgow Haskell Compiler / GHC


Commits:
126a6be4 by Ben Gamari at 2023-06-28T21:32:30-04:00
gitlab-ci: Refactor compilation of gen_ci

Flakify and document it, making it far less sensitive to the build
environment.

- - - - -
234b519f by Ben Gamari at 2023-06-28T21:40:46-04:00
Drop circle-ci-job.sh

- - - - -
9bcda707 by Ben Gamari at 2023-06-28T21:40:46-04:00
testsuite: Allow preservation of unexpected output

Here we introduce a new flag to the testsuite driver,
--unexpected-output-dir=<dir>, which allows the user to ask
the driver to preserve unexpected output from tests. The intent is for
this to be used in CI to allow users to more easily fix unexpected
platform-dependent output.

- - - - -
7540d340 by Ben Gamari at 2023-06-28T21:41:24-04:00
gitlab-ci: Preserve unexpected output

Here we enable use of the testsuite driver's `--unexpected-output-dir`
flag by CI, preserving the result as an artifact for use by users.

- - - - -


16 changed files:

- .gitlab-ci.yml
- .gitlab/ci.sh
- − .gitlab/circle-ci-job.sh
- + .gitlab/generate-ci/LICENSE
- + .gitlab/generate-ci/README.mkd
- + .gitlab/generate-ci/flake.lock
- + .gitlab/generate-ci/flake.nix
- .gitlab/gen_ci.hs → .gitlab/generate-ci/gen_ci.hs
- + .gitlab/generate-ci/generate-ci.cabal
- + .gitlab/generate-ci/update-ci
- − .gitlab/generate_job_metadata
- − .gitlab/generate_jobs
- .gitlab/jobs.yaml
- testsuite/driver/runtests.py
- testsuite/driver/testglobals.py
- testsuite/driver/testlib.py


Changes:

=====================================
.gitlab-ci.yml
=====================================
@@ -274,9 +274,8 @@ lint-ci-config:
     GIT_SUBMODULE_STRATEGY: none
   before_script:
     - echo "experimental-features = nix-command flakes" >> /etc/nix/nix.conf
-    - nix-channel --update
   script:
-    - .gitlab/generate_jobs
+    - nix run .gitlab/generate-ci#update-ci
     # 1 if .gitlab/generate_jobs changed the output of the generated config
     - nix shell nixpkgs#git -c git diff --exit-code
     # And run this to just make sure that works


=====================================
.gitlab/ci.sh
=====================================
@@ -673,12 +673,14 @@ function test_hadrian() {
       --summary-junit=./junit.xml \
       --test-have-intree-files \
       --test-compiler="${test_compiler}" \
-      "runtest.opts+=${RUNTEST_ARGS:-}" || fail "hadrian main testsuite"
+      "runtest.opts+=${RUNTEST_ARGS:-}" \
+      "runtest.opts+=--unexpected-output-dir=$TOP/unexpected-test-output" \
+      || fail "hadrian main testsuite"
 
+    tar -czf unexpected-test-output.tar.gz unexpected-test-output
     info "STAGE2_TEST=$?"
 
-    fi
-
+  fi
 }
 
 function summarise_hi_files() {


=====================================
.gitlab/circle-ci-job.sh deleted
=====================================
@@ -1,110 +0,0 @@
-# Circle CI "backend" for Gitlab CI
-# =================================
-#
-# Usage example:
-#   .gitlab/circle-ci-job.sh validate-x86_64-linux
-#
-# There are two things to configure to get artifacts to be
-# uploaded to gitlab properly:
-#
-# - At https://<gitlab host>/admin/application_settings, expand the
-#   Continuous Integration and Deployment section and set the
-#   "Maximum artifacts size (MB)" field to something large enough
-#   to contain the bindists (the test reports are tiny in comparison).
-#   500MB seems to work fine, but 200MB might be sufficient.
-#
-# - If gitlab is exposed behind some form of proxy (e.g nginx), make sure
-#   the maximum client request body size is large enough to contain all the
-#   artifacts of a build. For nginx, this would be the following configuration
-#   option: https://nginx.org/en/docs/http/ngx_http_core_module.html#client_max_body_size
-#   (which can be set with services.nginx.clientMaxBodySize on nixos).
-
-#!/usr/bin/env sh
-
-set -e
-
-GHCCI_URL="localhost:8888"
-
-[ $# -gt 0 ] || (echo You need to pass the Circle CI job type as argument to this script; exit 1)
-[ ${CI_RUNNER_ID:-} ] || (echo "CI_RUNNER_ID is not set"; exit 1)
-[ ${CI_JOB_ID:-} ] || (echo "CI_JOB_ID is not set"; exit 1)
-[ ${CI_COMMIT_SHA:-} ] || (echo "CI_COMMIT_SHA is not set"; exit 1)
-[ ${CI_REPOSITORY_URL:-} ] || (echo "CI_REPOSITORY_URL is not set"; exit 1)
-[ ${CI_PIPELINE_ID:-} ] || (echo "CI_PIPELINE_ID is not set"; exit 1)
-# the first argument to this script is the Circle CI job type:
-# validate-x86_64-linux, validate-i386-linux, ...
-CIRCLE_JOB="circleci-$1"
-
-gitlab_user=$(echo $CI_REPOSITORY_URL | cut -d/ -f4)
-gitlab_repo=$(echo $CI_REPOSITORY_URL | cut -d/ -f5 | cut -d. -f1)
-
-BODY="{ \"jobType\": \"$CIRCLE_JOB\", \"source\": { \"user\": \"$gitlab_user\", \"project\":\"$gitlab_repo\", \"commit\":\"$CI_COMMIT_SHA\" }, \"pipelineID\": $CI_PIPELINE_ID, \"runnerID\": $CI_RUNNER_ID, \"jobID\": $CI_JOB_ID }"
-
-
-RESP=$(curl -s -XPOST -H "Content-Type: application/json" -d "$BODY" \
-	    http://${GHCCI_URL}/job)
-
-if [ $? -eq 0 ]; then
-    build_num=$(echo $RESP | jq '.build_num')
-    circle_url=$(echo $RESP | jq '.url')
-else
-    echo "Couldn't submit job"
-    echo $RESP
-    exit 1
-fi
-
-echo Circle CI build number: $build_num
-echo Circle CI build page: $circle_url
-
-outcome="null"
-STATUS_URL="http://${GHCCI_URL}/job/${build_num}"
-STATUS_RESP=""
-
-while [ "$outcome" == "null" ]; do
-    sleep 30s
-    STATUS_RESP=$(curl -s $STATUS_URL)
-    if [ $? -eq 0 ]; then
-	new_outcome=$(echo $STATUS_RESP | jq '.outcome')
-	jq_exitcode=$?
-	if [ "$new_outcome" == "null" ] && [ $jq_exitcode -ne 0 ]; then
-	    echo "Couldn't read 'outcome' field in JSON:"
-	    echo $STATUS_RESP
-	    echo "Skipping"
-	else
-	    outcome="$new_outcome"
-	fi
-    else
-	echo "curl failed:"
-	echo $STATUS_RESP
-	echo "Skipping"
-    fi
-done
-
-if [ "$outcome" == "\"success\"" ]; then
-    echo The build passed
-    artifactsBody=$(curl -s http://${GHCCI_URL}/job/${build_num}/artifacts)
-    (echo $artifactsBody | jq '.[] | .url' | xargs wget -q) || echo "No artifacts"
-    exit 0
-else
-    echo The build failed
-
-    artifactsBody=$(curl -s http://${GHCCI_URL}/job/${build_num}/artifacts)
-    (echo $artifactsBody | jq '.[] | .url' | xargs wget -q) || echo "No artifacts"
-
-    failing_step=$(echo $STATUS_RESP | jq '.steps | .[] | .actions | .[] | select(.status != "success")')
-    failing_step_name=$(echo $failing_step | jq '.name' | sed -e 's/^"//' -e 's/"$//' -e 's/\\r\\n/\n/')
-    echo "Failing step: $failing_step_name"
-
-    failing_cmds=$(echo $failing_step | jq '.bash_command' | sed -e 's/^"//' -e 's/"$//' -e 's/\\r\\n/\n/')
-    echo "Failing command(s):"
-    echo $failing_cmds
-
-    log_url=$(echo $failing_step | jq '.output_url' | sed -e 's/^"//' -e 's/"$//' -e 's/\\r\\n/\n/')
-    echo "Log url: $log_url"
-
-    last_log_lines=$(curl -s $log_url | gunzip | jq '.[] | select(.type == "out") | .message' | sed -e 's/^"//' -e 's/"$//' -e 's/\\r\\n/\n/' | tail -50)
-    echo End of the build log:
-    echo $last_log_lines
-
-    exit 1
-fi


=====================================
.gitlab/generate-ci/LICENSE
=====================================
@@ -0,0 +1,30 @@
+Copyright (c) 2023, The GHC Developers
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+
+    * Redistributions in binary form must reproduce the above
+      copyright notice, this list of conditions and the following
+      disclaimer in the documentation and/or other materials provided
+      with the distribution.
+
+    * Neither the name of The GHC Developers nor the names of other
+      contributors may be used to endorse or promote products derived
+      from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.


=====================================
.gitlab/generate-ci/README.mkd
=====================================
@@ -0,0 +1,26 @@
+# generate-ci
+
+This is the generator for GHC's GitLab CI infrastructure. In particular, this
+generates `.gitlab/jobs.yaml`, which is a YAML (or, strictly speaking, JSON)
+file which defines the bulk of the validation, nightly, and release jobs of
+GHC's CI.
+
+
+## Modifying the CI configuration (nix)
+
+The jobs are defined in `gen_ci.hs`. After modifying this you can run
+```sh
+nix run .gitlab/generate-ci#update-ci
+```
+from the top of the GHC repository to update the generated configuration.
+
+
+## Modifying the CI configuration (without nix)
+
+One can run `update-ci` without Nix as follows (assuming one has `jq`,
+`cabal-install`, and GHC installed):
+```sh
+$ cabal build generate-ci
+$ PATH="$(dirname $(cabal list-bin generate-ci)):$PATH"
+$ ./update-ci
+```


=====================================
.gitlab/generate-ci/flake.lock
=====================================
@@ -0,0 +1,59 @@
+{
+  "nodes": {
+    "flake-utils": {
+      "inputs": {
+        "systems": "systems"
+      },
+      "locked": {
+        "lastModified": 1687709756,
+        "narHash": "sha256-Y5wKlQSkgEK2weWdOu4J3riRd+kV/VCgHsqLNTTWQ/0=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "dbabf0ca0c0c4bce6ea5eaf65af5cb694d2082c7",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1687886075,
+        "narHash": "sha256-PeayJDDDy+uw1Ats4moZnRdL1OFuZm1Tj+KiHlD67+o=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "a565059a348422af5af9026b5174dc5c0dcefdae",
+        "type": "github"
+      },
+      "original": {
+        "id": "nixpkgs",
+        "type": "indirect"
+      }
+    },
+    "root": {
+      "inputs": {
+        "flake-utils": "flake-utils",
+        "nixpkgs": "nixpkgs"
+      }
+    },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}


=====================================
.gitlab/generate-ci/flake.nix
=====================================
@@ -0,0 +1,33 @@
+{
+  description = "GHC CI Generator";
+
+  inputs.flake-utils.url = "github:numtide/flake-utils";
+
+  outputs = { self, nixpkgs, flake-utils }:
+    flake-utils.lib.eachDefaultSystem (system:
+      let pkgs = nixpkgs.legacyPackages.${system}; in
+      {
+        packages = rec {
+          update-ci = pkgs.runCommand "update-ci" {
+            nativeBuildInputs = with pkgs; [ makeWrapper ];
+          } ''
+            mkdir -p $out/bin
+            makeWrapper ${./update-ci} $out/bin/update-ci \
+              --prefix PATH : ${with pkgs; lib.makeBinPath [ generate-ci jq gitMinimal ]}
+          '';
+
+          generate-ci = pkgs.haskellPackages.callCabal2nix "generate-ci" ./. {};
+
+          default = update-ci;
+        };
+
+        apps = rec {
+          update-ci = flake-utils.lib.mkApp {
+            drv = self.packages.${system}.update-ci;
+          };
+
+          default = update-ci;
+        };
+      }
+    );
+}


=====================================
.gitlab/gen_ci.hs → .gitlab/generate-ci/gen_ci.hs
=====================================
@@ -3,9 +3,6 @@
 {-# LANGUAGE OverloadedStrings #-}
 {-# LANGUAGE DeriveFunctor #-}
 {-# LANGUAGE GeneralizedNewtypeDeriving #-}
-{- cabal:
-build-depends: base, aeson >= 1.8.1, containers, bytestring
--}
 
 import Data.Aeson as A
 import qualified Data.Map as Map
@@ -699,16 +696,19 @@ job arch opsys buildConfig = NamedJob { name = jobName, jobInfo = Job {..} }
           Emulator s       -> "CROSS_EMULATOR" =: s
           NoEmulatorNeeded -> mempty
       , if withNuma buildConfig then "ENABLE_NUMA" =: "1" else mempty
-      , if validateNonmovingGc buildConfig
-           then "RUNTEST_ARGS" =: "--way=nonmoving --way=nonmoving_thr --way=nonmoving_thr_sanity"
-           else mempty
+      , let runtestArgs =
+                [ "--way=nonmoving --way=nonmoving_thr --way=nonmoving_thr_sanity"
+                | validateNonmovingGc buildConfig
+                ]
+        in "RUNTEST_ARGS" =: unwords runtestArgs
       ]
 
     jobArtifacts = Artifacts
       { junitReport = "junit.xml"
       , expireIn = "2 weeks"
       , artifactPaths = [binDistName arch opsys buildConfig ++ ".tar.xz"
-                        ,"junit.xml"]
+                        ,"junit.xml"
+                        ,"unexpected-test-output.tar.gz"]
       , artifactsWhen = ArtifactsAlways
       }
 


=====================================
.gitlab/generate-ci/generate-ci.cabal
=====================================
@@ -0,0 +1,15 @@
+cabal-version:      3.0
+name:               generate-ci
+version:            0.1.0.0
+license:            BSD-3-Clause
+license-file:       LICENSE
+build-type:         Simple
+
+executable generate-ci
+    main-is:          gen_ci.hs
+    ghc-options:      -Wall
+    build-depends:    base,
+                      containers,
+                      bytestring,
+                      aeson >= 1.8.1
+    default-language: Haskell2010


=====================================
.gitlab/generate-ci/update-ci
=====================================
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+out_dir="$(git rev-parse --show-toplevel)/.gitlab"
+
+# Update job metadata for ghcup
+generate-ci metadata "$out_dir/jobs-metadata.json"
+echo "Updated $out_dir/jobs-metadata.json"
+
+# Update CI jobs
+tmp="$(mktemp)"
+generate-ci gitlab "$tmp"
+rm -f "$out_dir/jobs.yaml"
+echo "### THIS IS A GENERATED FILE, DO NOT MODIFY DIRECTLY" > "$out_dir/jobs.yaml"
+cat "$tmp" | jq >> "$out_dir/jobs.yaml"
+rm "$tmp"
+echo "Updated $out_dir/jobs.yaml"


=====================================
.gitlab/generate_job_metadata deleted
=====================================
@@ -1,5 +0,0 @@
-#! /usr/bin/env nix-shell
-#!nix-shell -i bash -p cabal-install "haskell.packages.ghc92.ghcWithPackages (pkgs: with pkgs; [aeson])" git jq
-
-cd "$(dirname "${BASH_SOURCE[0]}")"
-cabal run gen_ci -- metadata jobs-metadata.json


=====================================
.gitlab/generate_jobs deleted
=====================================
@@ -1,13 +0,0 @@
-#!/usr/bin/env nix-shell
-#!nix-shell -i bash -p cabal-install "haskell.packages.ghc92.ghcWithPackages (pkgs: with pkgs; [aeson])" git jq
-
-# shellcheck shell=bash
-
-set -euo pipefail
-
-cd "$(dirname "${BASH_SOURCE[0]}")"
-tmp=$(mktemp)
-cabal run gen_ci -- gitlab $tmp
-rm -f jobs.yaml
-echo "### THIS IS A GENERATED FILE, DO NOT MODIFY DIRECTLY" > jobs.yaml
-cat $tmp | jq | tee -a jobs.yaml


=====================================
.gitlab/jobs.yaml
=====================================
@@ -11,7 +11,8 @@
       "expire_in": "2 weeks",
       "paths": [
         "ghc-aarch64-darwin-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -59,6 +60,7 @@
       "LANG": "en_US.UTF-8",
       "MACOSX_DEPLOYMENT_TARGET": "11.0",
       "NIX_SYSTEM": "aarch64-darwin",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "aarch64-darwin-validate"
     }
   },
@@ -73,7 +75,8 @@
       "expire_in": "2 weeks",
       "paths": [
         "ghc-aarch64-linux-deb10-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -117,6 +120,7 @@
       "BIN_DIST_NAME": "ghc-aarch64-linux-deb10-validate",
       "BUILD_FLAVOUR": "validate",
       "CONFIGURE_ARGS": "",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "aarch64-linux-deb10-validate"
     }
   },
@@ -131,7 +135,8 @@
       "expire_in": "2 weeks",
       "paths": [
         "ghc-i386-linux-deb9-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -175,6 +180,7 @@
       "BIN_DIST_NAME": "ghc-i386-linux-deb9-validate",
       "BUILD_FLAVOUR": "validate",
       "CONFIGURE_ARGS": "",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "i386-linux-deb9-validate"
     }
   },
@@ -189,7 +195,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-aarch64-darwin-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -237,6 +244,7 @@
       "LANG": "en_US.UTF-8",
       "MACOSX_DEPLOYMENT_TARGET": "11.0",
       "NIX_SYSTEM": "aarch64-darwin",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "aarch64-darwin-validate",
       "XZ_OPT": "-9"
     }
@@ -252,7 +260,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-aarch64-linux-deb10-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -296,6 +305,7 @@
       "BIN_DIST_NAME": "ghc-aarch64-linux-deb10-validate",
       "BUILD_FLAVOUR": "validate",
       "CONFIGURE_ARGS": "",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "aarch64-linux-deb10-validate",
       "XZ_OPT": "-9"
     }
@@ -311,7 +321,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-aarch64-linux-deb10-validate+llvm.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -355,6 +366,7 @@
       "BIN_DIST_NAME": "ghc-aarch64-linux-deb10-validate+llvm",
       "BUILD_FLAVOUR": "validate+llvm",
       "CONFIGURE_ARGS": "",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "aarch64-linux-deb10-validate+llvm",
       "XZ_OPT": "-9"
     }
@@ -370,7 +382,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-i386-linux-deb9-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -414,6 +427,7 @@
       "BIN_DIST_NAME": "ghc-i386-linux-deb9-validate",
       "BUILD_FLAVOUR": "validate",
       "CONFIGURE_ARGS": "",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "i386-linux-deb9-validate",
       "XZ_OPT": "-9"
     }
@@ -429,7 +443,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-x86_64-darwin-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -477,6 +492,7 @@
       "LANG": "en_US.UTF-8",
       "MACOSX_DEPLOYMENT_TARGET": "10.10",
       "NIX_SYSTEM": "x86_64-darwin",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-darwin-validate",
       "XZ_OPT": "-9",
       "ac_cv_func_clock_gettime": "no",
@@ -495,7 +511,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-x86_64-freebsd13-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -541,6 +558,7 @@
       "CONFIGURE_ARGS": "--with-gmp-includes=/usr/local/include --with-gmp-libraries=/usr/local/lib --with-iconv-includes=/usr/local/include --with-iconv-libraries=/usr/local/lib ",
       "GHC_VERSION": "9.4.3",
       "HADRIAN_ARGS": "--docs=no-sphinx",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-freebsd13-validate",
       "XZ_OPT": "-9"
     }
@@ -556,7 +574,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-x86_64-linux-alpine3_12-int_native-validate+fully_static.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -603,6 +622,7 @@
       "CONFIGURE_ARGS": "--disable-ld-override ",
       "HADRIAN_ARGS": "--docs=no-sphinx",
       "INSTALL_CONFIGURE_ARGS": "--disable-ld-override",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-linux-alpine3_12-int_native-validate+fully_static",
       "XZ_OPT": "-9"
     }
@@ -618,7 +638,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-x86_64-linux-alpine3_12-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -665,6 +686,7 @@
       "CONFIGURE_ARGS": "--disable-ld-override ",
       "HADRIAN_ARGS": "--docs=no-sphinx",
       "INSTALL_CONFIGURE_ARGS": "--disable-ld-override",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-linux-alpine3_12-validate",
       "XZ_OPT": "-9"
     }
@@ -680,7 +702,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-x86_64-linux-alpine3_12-validate+fully_static.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -727,6 +750,7 @@
       "CONFIGURE_ARGS": "--disable-ld-override ",
       "HADRIAN_ARGS": "--docs=no-sphinx",
       "INSTALL_CONFIGURE_ARGS": "--disable-ld-override",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-linux-alpine3_12-validate+fully_static",
       "XZ_OPT": "-9"
     }
@@ -742,7 +766,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-x86_64-linux-alpine3_17-wasm-cross_wasm32-wasi-release+fully_static.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -788,6 +813,7 @@
       "CONFIGURE_ARGS": "--with-intree-gmp --with-system-libffi",
       "CROSS_TARGET": "wasm32-wasi",
       "HADRIAN_ARGS": "--docs=none",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-linux-alpine3_17-wasm-cross_wasm32-wasi-release+fully_static",
       "XZ_OPT": "-9"
     }
@@ -803,7 +829,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-x86_64-linux-alpine3_17-wasm-int_native-cross_wasm32-wasi-release+fully_static.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -849,6 +876,7 @@
       "CONFIGURE_ARGS": "--with-intree-gmp --with-system-libffi",
       "CROSS_TARGET": "wasm32-wasi",
       "HADRIAN_ARGS": "--docs=none",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-linux-alpine3_17-wasm-int_native-cross_wasm32-wasi-release+fully_static",
       "XZ_OPT": "-9"
     }
@@ -864,7 +892,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-x86_64-linux-alpine3_17-wasm-unreg-cross_wasm32-wasi-release+fully_static.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -910,6 +939,7 @@
       "CONFIGURE_ARGS": "--enable-unregisterised --with-intree-gmp --with-system-libffi",
       "CROSS_TARGET": "wasm32-wasi",
       "HADRIAN_ARGS": "--docs=none",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-linux-alpine3_17-wasm-unreg-cross_wasm32-wasi-release+fully_static",
       "XZ_OPT": "-9"
     }
@@ -925,7 +955,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-x86_64-linux-centos7-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -970,6 +1001,7 @@
       "BUILD_FLAVOUR": "validate",
       "CONFIGURE_ARGS": "",
       "HADRIAN_ARGS": "--docs=no-sphinx",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-linux-centos7-validate",
       "XZ_OPT": "-9"
     }
@@ -985,7 +1017,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-x86_64-linux-deb10-int_native-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -1029,6 +1062,7 @@
       "BIN_DIST_NAME": "ghc-x86_64-linux-deb10-int_native-validate",
       "BUILD_FLAVOUR": "validate",
       "CONFIGURE_ARGS": "",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-linux-deb10-int_native-validate",
       "XZ_OPT": "-9"
     }
@@ -1044,7 +1078,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-x86_64-linux-deb10-no_tntc-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -1088,6 +1123,7 @@
       "BIN_DIST_NAME": "ghc-x86_64-linux-deb10-no_tntc-validate",
       "BUILD_FLAVOUR": "validate",
       "CONFIGURE_ARGS": "--disable-tables-next-to-code",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-linux-deb10-no_tntc-validate",
       "XZ_OPT": "-9"
     }
@@ -1103,7 +1139,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-x86_64-linux-deb10-numa-slow-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -1148,6 +1185,7 @@
       "BUILD_FLAVOUR": "slow-validate",
       "CONFIGURE_ARGS": "",
       "ENABLE_NUMA": "1",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-linux-deb10-numa-slow-validate",
       "XZ_OPT": "-9"
     }
@@ -1163,7 +1201,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-x86_64-linux-deb10-unreg-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -1207,6 +1246,7 @@
       "BIN_DIST_NAME": "ghc-x86_64-linux-deb10-unreg-validate",
       "BUILD_FLAVOUR": "validate",
       "CONFIGURE_ARGS": "--enable-unregisterised",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-linux-deb10-unreg-validate",
       "XZ_OPT": "-9"
     }
@@ -1222,7 +1262,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-x86_64-linux-deb10-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -1266,6 +1307,7 @@
       "BIN_DIST_NAME": "ghc-x86_64-linux-deb10-validate",
       "BUILD_FLAVOUR": "validate",
       "CONFIGURE_ARGS": "",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-linux-deb10-validate",
       "XZ_OPT": "-9"
     }
@@ -1281,7 +1323,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-x86_64-linux-deb10-validate+debug_info.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -1325,6 +1368,7 @@
       "BIN_DIST_NAME": "ghc-x86_64-linux-deb10-validate+debug_info",
       "BUILD_FLAVOUR": "validate+debug_info",
       "CONFIGURE_ARGS": "",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-linux-deb10-validate+debug_info",
       "XZ_OPT": "-9"
     }
@@ -1340,7 +1384,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-x86_64-linux-deb10-validate+llvm.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -1384,6 +1429,7 @@
       "BIN_DIST_NAME": "ghc-x86_64-linux-deb10-validate+llvm",
       "BUILD_FLAVOUR": "validate+llvm",
       "CONFIGURE_ARGS": "",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-linux-deb10-validate+llvm",
       "XZ_OPT": "-9"
     }
@@ -1399,7 +1445,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-x86_64-linux-deb10-validate+thread_sanitizer.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -1444,6 +1491,7 @@
       "BUILD_FLAVOUR": "validate+thread_sanitizer",
       "CONFIGURE_ARGS": "",
       "HADRIAN_ARGS": "--docs=none",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-linux-deb10-validate+thread_sanitizer",
       "TSAN_OPTIONS": "suppressions=$CI_PROJECT_DIR/rts/.tsan-suppressions",
       "XZ_OPT": "-9"
@@ -1460,7 +1508,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-x86_64-linux-deb11-cross_aarch64-linux-gnu-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -1506,6 +1555,7 @@
       "CONFIGURE_ARGS": "--with-intree-gmp",
       "CROSS_EMULATOR": "qemu-aarch64 -L /usr/aarch64-linux-gnu",
       "CROSS_TARGET": "aarch64-linux-gnu",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-linux-deb11-cross_aarch64-linux-gnu-validate",
       "XZ_OPT": "-9"
     }
@@ -1521,7 +1571,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-x86_64-linux-deb11-int_native-cross_javascript-unknown-ghcjs-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -1568,6 +1619,7 @@
       "CONFIGURE_WRAPPER": "emconfigure",
       "CROSS_EMULATOR": "js-emulator",
       "CROSS_TARGET": "javascript-unknown-ghcjs",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-linux-deb11-int_native-cross_javascript-unknown-ghcjs-validate",
       "XZ_OPT": "-9"
     }
@@ -1583,7 +1635,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-x86_64-linux-deb11-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -1627,6 +1680,7 @@
       "BIN_DIST_NAME": "ghc-x86_64-linux-deb11-validate",
       "BUILD_FLAVOUR": "validate",
       "CONFIGURE_ARGS": "",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-linux-deb11-validate",
       "XZ_OPT": "-9"
     }
@@ -1642,7 +1696,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-x86_64-linux-deb11-validate+boot_nonmoving_gc.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -1702,7 +1757,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-x86_64-linux-deb9-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -1746,6 +1802,7 @@
       "BIN_DIST_NAME": "ghc-x86_64-linux-deb9-validate",
       "BUILD_FLAVOUR": "validate",
       "CONFIGURE_ARGS": "",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-linux-deb9-validate",
       "XZ_OPT": "-9"
     }
@@ -1761,7 +1818,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-x86_64-linux-fedora33-release.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -1807,6 +1865,7 @@
       "CONFIGURE_ARGS": "",
       "LLC": "/bin/false",
       "OPT": "/bin/false",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-linux-fedora33-release",
       "XZ_OPT": "-9"
     }
@@ -1822,7 +1881,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-x86_64-linux-fedora33-release.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -1869,6 +1929,7 @@
       "HADRIAN_ARGS": "--haddock-base-url",
       "LLC": "/bin/false",
       "OPT": "/bin/false",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-linux-fedora33-release",
       "XZ_OPT": "-9"
     }
@@ -1884,7 +1945,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-x86_64-linux-fedora33-validate+debug_info.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -1930,6 +1992,7 @@
       "CONFIGURE_ARGS": "",
       "LLC": "/bin/false",
       "OPT": "/bin/false",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-linux-fedora33-validate+debug_info",
       "XZ_OPT": "-9"
     }
@@ -1945,7 +2008,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-x86_64-linux-rocky8-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -1990,6 +2054,7 @@
       "BUILD_FLAVOUR": "validate",
       "CONFIGURE_ARGS": "",
       "HADRIAN_ARGS": "--docs=no-sphinx",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-linux-rocky8-validate",
       "XZ_OPT": "-9"
     }
@@ -2005,7 +2070,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-x86_64-linux-ubuntu18_04-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -2049,6 +2115,7 @@
       "BIN_DIST_NAME": "ghc-x86_64-linux-ubuntu18_04-validate",
       "BUILD_FLAVOUR": "validate",
       "CONFIGURE_ARGS": "",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-linux-ubuntu18_04-validate",
       "XZ_OPT": "-9"
     }
@@ -2064,7 +2131,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-x86_64-linux-ubuntu20_04-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -2108,6 +2176,7 @@
       "BIN_DIST_NAME": "ghc-x86_64-linux-ubuntu20_04-validate",
       "BUILD_FLAVOUR": "validate",
       "CONFIGURE_ARGS": "",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-linux-ubuntu20_04-validate",
       "XZ_OPT": "-9"
     }
@@ -2122,7 +2191,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-x86_64-windows-int_native-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -2167,6 +2237,7 @@
       "HADRIAN_ARGS": "--docs=no-sphinx",
       "LANG": "en_US.UTF-8",
       "MSYSTEM": "CLANG64",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-windows-int_native-validate",
       "XZ_OPT": "-9"
     }
@@ -2181,7 +2252,8 @@
       "expire_in": "8 weeks",
       "paths": [
         "ghc-x86_64-windows-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -2226,6 +2298,7 @@
       "HADRIAN_ARGS": "--docs=no-sphinx",
       "LANG": "en_US.UTF-8",
       "MSYSTEM": "CLANG64",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-windows-validate",
       "XZ_OPT": "-9"
     }
@@ -2241,7 +2314,8 @@
       "expire_in": "1 year",
       "paths": [
         "ghc-aarch64-darwin-release.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -2290,6 +2364,7 @@
       "LANG": "en_US.UTF-8",
       "MACOSX_DEPLOYMENT_TARGET": "11.0",
       "NIX_SYSTEM": "aarch64-darwin",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "aarch64-darwin-release",
       "XZ_OPT": "-9"
     }
@@ -2305,7 +2380,8 @@
       "expire_in": "1 year",
       "paths": [
         "ghc-aarch64-linux-deb10-release+no_split_sections.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -2351,6 +2427,7 @@
       "CONFIGURE_ARGS": "",
       "HADRIAN_ARGS": "--hash-unit-ids",
       "IGNORE_PERF_FAILURES": "all",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "aarch64-linux-deb10-release+no_split_sections",
       "XZ_OPT": "-9"
     }
@@ -2366,7 +2443,8 @@
       "expire_in": "1 year",
       "paths": [
         "ghc-i386-linux-deb9-release+no_split_sections.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -2412,6 +2490,7 @@
       "CONFIGURE_ARGS": "",
       "HADRIAN_ARGS": "--hash-unit-ids",
       "IGNORE_PERF_FAILURES": "all",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "i386-linux-deb9-release+no_split_sections",
       "XZ_OPT": "-9"
     }
@@ -2427,7 +2506,8 @@
       "expire_in": "1 year",
       "paths": [
         "ghc-x86_64-darwin-release.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -2476,6 +2556,7 @@
       "LANG": "en_US.UTF-8",
       "MACOSX_DEPLOYMENT_TARGET": "10.10",
       "NIX_SYSTEM": "x86_64-darwin",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-darwin-release",
       "XZ_OPT": "-9",
       "ac_cv_func_clock_gettime": "no",
@@ -2494,7 +2575,8 @@
       "expire_in": "1 year",
       "paths": [
         "ghc-x86_64-linux-alpine3_12-int_native-release+fully_static.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -2542,6 +2624,7 @@
       "HADRIAN_ARGS": "--hash-unit-ids --docs=no-sphinx",
       "IGNORE_PERF_FAILURES": "all",
       "INSTALL_CONFIGURE_ARGS": "--disable-ld-override",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-linux-alpine3_12-int_native-release+fully_static",
       "XZ_OPT": "-9"
     }
@@ -2557,7 +2640,8 @@
       "expire_in": "1 year",
       "paths": [
         "ghc-x86_64-linux-alpine3_12-release+fully_static+no_split_sections.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -2605,6 +2689,7 @@
       "HADRIAN_ARGS": "--hash-unit-ids --docs=no-sphinx",
       "IGNORE_PERF_FAILURES": "all",
       "INSTALL_CONFIGURE_ARGS": "--disable-ld-override",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-linux-alpine3_12-release+fully_static+no_split_sections",
       "XZ_OPT": "-9"
     }
@@ -2620,7 +2705,8 @@
       "expire_in": "1 year",
       "paths": [
         "ghc-x86_64-linux-alpine3_12-release+no_split_sections.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -2668,6 +2754,7 @@
       "HADRIAN_ARGS": "--hash-unit-ids --docs=no-sphinx",
       "IGNORE_PERF_FAILURES": "all",
       "INSTALL_CONFIGURE_ARGS": "--disable-ld-override",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-linux-alpine3_12-release+no_split_sections",
       "XZ_OPT": "-9"
     }
@@ -2683,7 +2770,8 @@
       "expire_in": "1 year",
       "paths": [
         "ghc-x86_64-linux-centos7-release+no_split_sections.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -2729,6 +2817,7 @@
       "CONFIGURE_ARGS": "",
       "HADRIAN_ARGS": "--hash-unit-ids --docs=no-sphinx",
       "IGNORE_PERF_FAILURES": "all",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-linux-centos7-release+no_split_sections",
       "XZ_OPT": "-9"
     }
@@ -2744,7 +2833,8 @@
       "expire_in": "1 year",
       "paths": [
         "ghc-x86_64-linux-deb10-release.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -2790,6 +2880,7 @@
       "CONFIGURE_ARGS": "",
       "HADRIAN_ARGS": "--hash-unit-ids",
       "IGNORE_PERF_FAILURES": "all",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-linux-deb10-release",
       "XZ_OPT": "-9"
     }
@@ -2805,7 +2896,8 @@
       "expire_in": "1 year",
       "paths": [
         "ghc-x86_64-linux-deb10-release+debug_info.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -2851,6 +2943,7 @@
       "CONFIGURE_ARGS": "",
       "HADRIAN_ARGS": "--hash-unit-ids",
       "IGNORE_PERF_FAILURES": "all",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-linux-deb10-release+debug_info",
       "XZ_OPT": "-9"
     }
@@ -2866,7 +2959,8 @@
       "expire_in": "1 year",
       "paths": [
         "ghc-x86_64-linux-deb11-release.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -2912,6 +3006,7 @@
       "CONFIGURE_ARGS": "",
       "HADRIAN_ARGS": "--hash-unit-ids",
       "IGNORE_PERF_FAILURES": "all",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-linux-deb11-release",
       "XZ_OPT": "-9"
     }
@@ -2927,7 +3022,8 @@
       "expire_in": "1 year",
       "paths": [
         "ghc-x86_64-linux-deb11-release+boot_nonmoving_gc.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -2989,7 +3085,8 @@
       "expire_in": "1 year",
       "paths": [
         "ghc-x86_64-linux-deb9-release+no_split_sections.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -3035,6 +3132,7 @@
       "CONFIGURE_ARGS": "",
       "HADRIAN_ARGS": "--hash-unit-ids",
       "IGNORE_PERF_FAILURES": "all",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-linux-deb9-release+no_split_sections",
       "XZ_OPT": "-9"
     }
@@ -3050,7 +3148,8 @@
       "expire_in": "1 year",
       "paths": [
         "ghc-x86_64-linux-fedora33-release.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -3098,6 +3197,7 @@
       "IGNORE_PERF_FAILURES": "all",
       "LLC": "/bin/false",
       "OPT": "/bin/false",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-linux-fedora33-release",
       "XZ_OPT": "-9"
     }
@@ -3113,7 +3213,8 @@
       "expire_in": "1 year",
       "paths": [
         "ghc-x86_64-linux-fedora33-release+debug_info.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -3161,6 +3262,7 @@
       "IGNORE_PERF_FAILURES": "all",
       "LLC": "/bin/false",
       "OPT": "/bin/false",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-linux-fedora33-release+debug_info",
       "XZ_OPT": "-9"
     }
@@ -3176,7 +3278,8 @@
       "expire_in": "1 year",
       "paths": [
         "ghc-x86_64-linux-fedora33-release.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -3224,6 +3327,7 @@
       "IGNORE_PERF_FAILURES": "all",
       "LLC": "/bin/false",
       "OPT": "/bin/false",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-linux-fedora33-release",
       "XZ_OPT": "-9"
     }
@@ -3239,7 +3343,8 @@
       "expire_in": "1 year",
       "paths": [
         "ghc-x86_64-linux-rocky8-release.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -3285,6 +3390,7 @@
       "CONFIGURE_ARGS": "",
       "HADRIAN_ARGS": "--hash-unit-ids --docs=no-sphinx",
       "IGNORE_PERF_FAILURES": "all",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-linux-rocky8-release",
       "XZ_OPT": "-9"
     }
@@ -3300,7 +3406,8 @@
       "expire_in": "1 year",
       "paths": [
         "ghc-x86_64-linux-ubuntu18_04-release.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -3346,6 +3453,7 @@
       "CONFIGURE_ARGS": "",
       "HADRIAN_ARGS": "--hash-unit-ids",
       "IGNORE_PERF_FAILURES": "all",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-linux-ubuntu18_04-release",
       "XZ_OPT": "-9"
     }
@@ -3361,7 +3469,8 @@
       "expire_in": "1 year",
       "paths": [
         "ghc-x86_64-linux-ubuntu20_04-release.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -3407,6 +3516,7 @@
       "CONFIGURE_ARGS": "",
       "HADRIAN_ARGS": "--hash-unit-ids",
       "IGNORE_PERF_FAILURES": "all",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-linux-ubuntu20_04-release",
       "XZ_OPT": "-9"
     }
@@ -3421,7 +3531,8 @@
       "expire_in": "1 year",
       "paths": [
         "ghc-x86_64-windows-int_native-release+no_split_sections.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -3467,6 +3578,7 @@
       "IGNORE_PERF_FAILURES": "all",
       "LANG": "en_US.UTF-8",
       "MSYSTEM": "CLANG64",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-windows-int_native-release+no_split_sections",
       "XZ_OPT": "-9"
     }
@@ -3481,7 +3593,8 @@
       "expire_in": "1 year",
       "paths": [
         "ghc-x86_64-windows-release+no_split_sections.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -3527,6 +3640,7 @@
       "IGNORE_PERF_FAILURES": "all",
       "LANG": "en_US.UTF-8",
       "MSYSTEM": "CLANG64",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-windows-release+no_split_sections",
       "XZ_OPT": "-9"
     }
@@ -3542,7 +3656,8 @@
       "expire_in": "2 weeks",
       "paths": [
         "ghc-x86_64-darwin-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -3590,6 +3705,7 @@
       "LANG": "en_US.UTF-8",
       "MACOSX_DEPLOYMENT_TARGET": "10.10",
       "NIX_SYSTEM": "x86_64-darwin",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-darwin-validate",
       "ac_cv_func_clock_gettime": "no",
       "ac_cv_func_futimens": "no",
@@ -3607,7 +3723,8 @@
       "expire_in": "2 weeks",
       "paths": [
         "ghc-x86_64-freebsd13-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -3653,6 +3770,7 @@
       "CONFIGURE_ARGS": "--with-gmp-includes=/usr/local/include --with-gmp-libraries=/usr/local/lib --with-iconv-includes=/usr/local/include --with-iconv-libraries=/usr/local/lib ",
       "GHC_VERSION": "9.4.3",
       "HADRIAN_ARGS": "--docs=no-sphinx",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-freebsd13-validate"
     }
   },
@@ -3667,7 +3785,8 @@
       "expire_in": "2 weeks",
       "paths": [
         "ghc-x86_64-linux-alpine3_12-validate+fully_static.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -3714,6 +3833,7 @@
       "CONFIGURE_ARGS": "--disable-ld-override ",
       "HADRIAN_ARGS": "--docs=no-sphinx",
       "INSTALL_CONFIGURE_ARGS": "--disable-ld-override",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-linux-alpine3_12-validate+fully_static"
     }
   },
@@ -3728,7 +3848,8 @@
       "expire_in": "2 weeks",
       "paths": [
         "ghc-x86_64-linux-alpine3_17-wasm-cross_wasm32-wasi-release+fully_static.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -3774,6 +3895,7 @@
       "CONFIGURE_ARGS": "--with-intree-gmp --with-system-libffi",
       "CROSS_TARGET": "wasm32-wasi",
       "HADRIAN_ARGS": "--docs=none",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-linux-alpine3_17-wasm-cross_wasm32-wasi-release+fully_static"
     }
   },
@@ -3788,7 +3910,8 @@
       "expire_in": "2 weeks",
       "paths": [
         "ghc-x86_64-linux-alpine3_17-wasm-int_native-cross_wasm32-wasi-release+fully_static.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -3835,6 +3958,7 @@
       "CONFIGURE_ARGS": "--with-intree-gmp --with-system-libffi",
       "CROSS_TARGET": "wasm32-wasi",
       "HADRIAN_ARGS": "--docs=none",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-linux-alpine3_17-wasm-int_native-cross_wasm32-wasi-release+fully_static"
     }
   },
@@ -3849,7 +3973,8 @@
       "expire_in": "2 weeks",
       "paths": [
         "ghc-x86_64-linux-alpine3_17-wasm-unreg-cross_wasm32-wasi-release+fully_static.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -3896,6 +4021,7 @@
       "CONFIGURE_ARGS": "--enable-unregisterised --with-intree-gmp --with-system-libffi",
       "CROSS_TARGET": "wasm32-wasi",
       "HADRIAN_ARGS": "--docs=none",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-linux-alpine3_17-wasm-unreg-cross_wasm32-wasi-release+fully_static"
     }
   },
@@ -3910,7 +4036,8 @@
       "expire_in": "2 weeks",
       "paths": [
         "ghc-x86_64-linux-deb10-int_native-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -3954,6 +4081,7 @@
       "BIN_DIST_NAME": "ghc-x86_64-linux-deb10-int_native-validate",
       "BUILD_FLAVOUR": "validate",
       "CONFIGURE_ARGS": "",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-linux-deb10-int_native-validate"
     }
   },
@@ -3968,7 +4096,8 @@
       "expire_in": "2 weeks",
       "paths": [
         "ghc-x86_64-linux-deb10-no_tntc-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -4013,6 +4142,7 @@
       "BIN_DIST_NAME": "ghc-x86_64-linux-deb10-no_tntc-validate",
       "BUILD_FLAVOUR": "validate",
       "CONFIGURE_ARGS": "--disable-tables-next-to-code",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-linux-deb10-no_tntc-validate"
     }
   },
@@ -4027,7 +4157,8 @@
       "expire_in": "2 weeks",
       "paths": [
         "ghc-x86_64-linux-deb10-numa-slow-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -4072,6 +4203,7 @@
       "BUILD_FLAVOUR": "slow-validate",
       "CONFIGURE_ARGS": "",
       "ENABLE_NUMA": "1",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-linux-deb10-numa-slow-validate"
     }
   },
@@ -4086,7 +4218,8 @@
       "expire_in": "2 weeks",
       "paths": [
         "ghc-x86_64-linux-deb10-unreg-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -4130,6 +4263,7 @@
       "BIN_DIST_NAME": "ghc-x86_64-linux-deb10-unreg-validate",
       "BUILD_FLAVOUR": "validate",
       "CONFIGURE_ARGS": "--enable-unregisterised",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-linux-deb10-unreg-validate"
     }
   },
@@ -4144,7 +4278,8 @@
       "expire_in": "2 weeks",
       "paths": [
         "ghc-x86_64-linux-deb10-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -4188,6 +4323,7 @@
       "BIN_DIST_NAME": "ghc-x86_64-linux-deb10-validate",
       "BUILD_FLAVOUR": "validate",
       "CONFIGURE_ARGS": "--enable-ipe-data-compression",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-linux-deb10-validate"
     }
   },
@@ -4202,7 +4338,8 @@
       "expire_in": "2 weeks",
       "paths": [
         "ghc-x86_64-linux-deb10-validate+debug_info.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -4246,6 +4383,7 @@
       "BIN_DIST_NAME": "ghc-x86_64-linux-deb10-validate+debug_info",
       "BUILD_FLAVOUR": "validate+debug_info",
       "CONFIGURE_ARGS": "",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-linux-deb10-validate+debug_info"
     }
   },
@@ -4260,7 +4398,8 @@
       "expire_in": "2 weeks",
       "paths": [
         "ghc-x86_64-linux-deb10-validate+llvm.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -4304,6 +4443,7 @@
       "BIN_DIST_NAME": "ghc-x86_64-linux-deb10-validate+llvm",
       "BUILD_FLAVOUR": "validate+llvm",
       "CONFIGURE_ARGS": "",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-linux-deb10-validate+llvm"
     }
   },
@@ -4318,7 +4458,8 @@
       "expire_in": "2 weeks",
       "paths": [
         "ghc-x86_64-linux-deb10-validate+thread_sanitizer.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -4364,6 +4505,7 @@
       "BUILD_FLAVOUR": "validate+thread_sanitizer",
       "CONFIGURE_ARGS": "",
       "HADRIAN_ARGS": "--docs=none",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-linux-deb10-validate+thread_sanitizer",
       "TSAN_OPTIONS": "suppressions=$CI_PROJECT_DIR/rts/.tsan-suppressions"
     }
@@ -4379,7 +4521,8 @@
       "expire_in": "2 weeks",
       "paths": [
         "ghc-x86_64-linux-deb11-cross_aarch64-linux-gnu-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -4425,6 +4568,7 @@
       "CONFIGURE_ARGS": "--with-intree-gmp",
       "CROSS_EMULATOR": "qemu-aarch64 -L /usr/aarch64-linux-gnu",
       "CROSS_TARGET": "aarch64-linux-gnu",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-linux-deb11-cross_aarch64-linux-gnu-validate"
     }
   },
@@ -4439,7 +4583,8 @@
       "expire_in": "2 weeks",
       "paths": [
         "ghc-x86_64-linux-deb11-int_native-cross_javascript-unknown-ghcjs-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -4486,6 +4631,7 @@
       "CONFIGURE_WRAPPER": "emconfigure",
       "CROSS_EMULATOR": "js-emulator",
       "CROSS_TARGET": "javascript-unknown-ghcjs",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-linux-deb11-int_native-cross_javascript-unknown-ghcjs-validate"
     }
   },
@@ -4500,7 +4646,8 @@
       "expire_in": "2 weeks",
       "paths": [
         "ghc-x86_64-linux-deb11-validate+boot_nonmoving_gc.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -4559,7 +4706,8 @@
       "expire_in": "2 weeks",
       "paths": [
         "ghc-x86_64-linux-fedora33-release.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -4605,6 +4753,7 @@
       "CONFIGURE_ARGS": "",
       "LLC": "/bin/false",
       "OPT": "/bin/false",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-linux-fedora33-release"
     }
   },
@@ -4618,7 +4767,8 @@
       "expire_in": "2 weeks",
       "paths": [
         "ghc-x86_64-windows-validate.tar.xz",
-        "junit.xml"
+        "junit.xml",
+        "unexpected-test-output.tar.gz"
       ],
       "reports": {
         "junit": "junit.xml"
@@ -4663,6 +4813,7 @@
       "HADRIAN_ARGS": "--docs=no-sphinx",
       "LANG": "en_US.UTF-8",
       "MSYSTEM": "CLANG64",
+      "RUNTEST_ARGS": "",
       "TEST_ENV": "x86_64-windows-validate"
     }
   }


=====================================
testsuite/driver/runtests.py
=====================================
@@ -71,6 +71,7 @@ parser.add_argument("--config", action='append', help="config field")
 parser.add_argument("--rootdir", action='append', help="root of tree containing tests (default: .)")
 parser.add_argument("--metrics-file", help="file in which to save (append) the performance test metrics. If omitted, git notes will be used.")
 parser.add_argument("--summary-file", help="file in which to save the (human-readable) summary")
+parser.add_argument("--unexpected-output-dir", help="directory in which to place unexpected output")
 parser.add_argument("--target-wrapper", help="wrapper executable to use when executing binaries compiled for the target")
 parser.add_argument("--no-print-summary", action="store_true", help="should we print the summary?")
 parser.add_argument("--only", action="append", help="just this test (can be give multiple --only= flags)")
@@ -128,6 +129,9 @@ if args.top:
 if args.test_package_db:
     config.test_package_db = args.test_package_db
 
+if args.unexpected_output_dir:
+    config.unexpected_output_dir = Path(args.unexpected_output_dir)
+
 if args.only:
     config.only = args.only
     config.run_only_some_tests = True


=====================================
testsuite/driver/testglobals.py
=====================================
@@ -203,7 +203,7 @@ class TestConfig:
         self.baseline_commit = None # type: Optional[GitRef]
 
         # Additional package dbs to inspect for test dependencies.
-        self.test_package_db = [] # type: [PathToPackageDb]
+        self.test_package_db = [] # type: List[PathToPackageDb]
 
         # Should we skip performance tests
         self.skip_perf_tests = False
@@ -225,6 +225,10 @@ class TestConfig:
         # See Note [Haddock runtime stats files] at the bottom of this file.
         self.stats_files_dir = Path('/please_set_stats_files_dir')
 
+        # Where to place observed output files on when unexpected output
+        # is observed.
+        self.unexpected_output_dir = None # type: Optional[Path]
+
         # Should we cleanup after test runs?
         self.cleanup = True
 


=====================================
testsuite/driver/testlib.py
=====================================
@@ -2257,11 +2257,15 @@ async def check_prof_ok(name: TestName, way: WayName) -> bool:
 # new output. Returns true if output matched or was accepted, false
 # otherwise. See Note [Output comparison] for the meaning of the
 # normaliser and whitespace_normaliser parameters.
-async def compare_outputs(way: WayName,
-                    kind: str,
-                    normaliser: OutputNormalizer,
-                    expected_file, actual_file, diff_file=None,
-                    whitespace_normaliser: OutputNormalizer=lambda x:x) -> bool:
+async def compare_outputs(
+        way: WayName,
+        kind: str,
+        normaliser: OutputNormalizer,
+        expected_file: Path,
+        actual_file: Path,
+        diff_file: Optional[Path]=None,
+        whitespace_normaliser: OutputNormalizer=lambda x:x) -> bool:
+
     # Respect ignore_stdout and ignore_stderr options
     if kind == 'stderr' and getTestOpts().ignore_stderr:
         return True
@@ -2338,6 +2342,12 @@ async def compare_outputs(way: WayName,
             expected_path.unlink()
             return True
         else:
+            if config.unexpected_output_dir is not None:
+                ghc_root = expected_path.relative_to(config.top.parent)
+                out = config.unexpected_output_dir / ghc_root
+                out.parent.mkdir(exist_ok=True, parents=True)
+                write_file(out, actual_raw)
+
             return False
 
 # Checks that each line from pattern_file is present in actual_file as
@@ -2392,6 +2402,15 @@ def grep_output(normaliser: OutputNormalizer, pattern_file, actual_file, is_subs
 #    squash all whitespace, making the diff unreadable. Instead we rely
 #    on the `diff` program to ignore whitespace changes as much as
 #    possible (#10152).
+#
+# In addition, to aid CI users we will optionally collect all
+# of the unexpected output that we encountered in the
+# directory at config.unexpected_output_dir. The intent here is for this
+# directory to be preserved as a CI artifact which can then
+# be downloaded by the user and committed to their branch
+# to address CI failures on platforms which they cannot
+# test locally.
+
 
 # Note [Null device handling]
 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~



View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/compare/02d785a052ace58eb5ff163b05e78d1a4218795a...7540d3405b0f97575071e70149e72caedc302bec

-- 
View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/compare/02d785a052ace58eb5ff163b05e78d1a4218795a...7540d3405b0f97575071e70149e72caedc302bec
You're receiving this email because of your account on gitlab.haskell.org.


-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mail.haskell.org/pipermail/ghc-commits/attachments/20230628/692fe345/attachment-0001.html>


More information about the ghc-commits mailing list