[Git][ghc/ghc][wip/backports-9.0] gitlab-ci: Backport ci.sh from master
Ben Gamari
gitlab at gitlab.haskell.org
Mon Nov 30 00:49:23 UTC 2020
Ben Gamari pushed to branch wip/backports-9.0 at Glasgow Haskell Compiler / GHC
Commits:
46e8a41a by Ben Gamari at 2020-11-29T19:49:17-05:00
gitlab-ci: Backport ci.sh from master
- - - - -
3 changed files:
- .gitlab-ci.yml
- .gitlab/ci.sh
- + .gitlab/common.sh
Changes:
=====================================
.gitlab-ci.yml
=====================================
@@ -17,14 +17,14 @@ variables:
GIT_SUBMODULE_STRATEGY: "recursive"
stages:
- - lint # Source linting
- - quick-build # A very quick smoke-test to weed out broken commits
- - build # A quick smoke-test to weed out broken commits
- - full-build # Build all the things
- - cleanup # See Note [Cleanup after the shell executor]
- - packaging # Source distribution, etc.
- - testing # head.hackage correctness and compiler performance testing
- - deploy # push documentation
+ - tool-lint # Source linting of the tools
+ - quick-build # A very quick smoke-test to weed out broken commits
+ - lint # Source linting of GHC
+ - build # A quick smoke-test to weed out broken commits
+ - full-build # Build all the things
+ - packaging # Source distribution, etc.
+ - testing # head.hackage correctness and compiler performance testing
+ - deploy # push documentation
# Note [The CI Story]
# ~~~~~~~~~~~~~~~~~~~
@@ -47,6 +47,7 @@ workflow:
rules:
- if: $CI_MERGE_REQUEST_ID
- if: $CI_COMMIT_TAG
+ - if: '$CI_COMMIT_BRANCH == "master"'
- if: '$CI_COMMIT_BRANCH == "wip/marge_bot_batch_merge_job"'
- if: '$CI_COMMIT_BRANCH =~ /ghc-[0-9]+\.[0-9]+/'
- if: '$CI_PIPELINE_SOURCE == "web"'
@@ -61,7 +62,6 @@ workflow:
.release: &release
variables:
BUILD_FLAVOUR: "perf"
- FLAVOUR: "perf"
artifacts:
when: always
expire_in: 1 year
@@ -81,11 +81,11 @@ workflow:
############################################################
-# Linting
+# tool linting
############################################################
ghc-linters:
- stage: lint
+ stage: tool-lint
image: "registry.gitlab.haskell.org/ghc/ci-images/linters:$DOCKER_REV"
script:
- git fetch "$CI_MERGE_REQUEST_PROJECT_URL" $CI_MERGE_REQUEST_TARGET_BRANCH_NAME
@@ -105,7 +105,7 @@ ghc-linters:
# Run mypy Python typechecker on linter scripts.
lint-linters:
- stage: lint
+ stage: tool-lint
image: "registry.gitlab.haskell.org/ghc/ci-images/linters:$DOCKER_REV"
script:
- mypy .gitlab/linters/*.py
@@ -115,17 +115,17 @@ lint-linters:
# Check that .T files all parse by listing broken tests.
lint-testsuite:
- stage: lint
+ stage: tool-lint
image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
script:
- - make -Ctestsuite list_broken TEST_HC=ghc
+ - make -Ctestsuite list_broken TEST_HC=$GHC
dependencies: []
tags:
- lint
# Run mypy Python typechecker on testsuite driver
typecheck-testsuite:
- stage: lint
+ stage: tool-lint
image: "registry.gitlab.haskell.org/ghc/ci-images/linters:$DOCKER_REV"
script:
- mypy testsuite/driver/runtests.py
@@ -137,7 +137,7 @@ typecheck-testsuite:
# accommodate, e.g., haddock changes not yet upstream) but not on `master` or
# Marge jobs.
.lint-submods:
- stage: lint
+ stage: tool-lint
image: "registry.gitlab.haskell.org/ghc/ci-images/linters:$DOCKER_REV"
script:
- git fetch "$CI_MERGE_REQUEST_PROJECT_URL" $CI_MERGE_REQUEST_TARGET_BRANCH_NAME
@@ -169,7 +169,7 @@ lint-submods-branch:
- if: '$CI_COMMIT_BRANCH =~ /ghc-[0.9]+\.[0-9]+/'
.lint-changelogs:
- stage: lint
+ stage: tool-lint
image: "registry.gitlab.haskell.org/ghc/ci-images/linters:$DOCKER_REV"
dependencies: []
tags:
@@ -189,14 +189,13 @@ lint-release-changelogs:
rules:
- if: '$CI_COMMIT_BRANCH =~ /ghc-[0.9]+\.[0-9]+/'
-
############################################################
# Validation via Pipelines (hadrian)
############################################################
.validate-hadrian:
variables:
- FLAVOUR: "validate"
+ BUILD_FLAVOUR: "validate"
script:
- .gitlab/ci.sh setup
- .gitlab/ci.sh configure
@@ -212,7 +211,7 @@ lint-release-changelogs:
junit: junit.xml
expire_in: 2 week
paths:
- - ghc.tar.xz
+ - "$BIN_DIST_NAME.tar.xz"
- junit.xml
.validate-linux-hadrian:
@@ -220,6 +219,7 @@ lint-release-changelogs:
image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
variables:
TEST_ENV: "x86_64-linux-deb9-hadrian"
+ BIN_DIST_NAME: "ghc-x86_64-deb9-linux"
before_script:
# workaround for docker permissions
- sudo chown ghc:ghc -R .
@@ -228,23 +228,65 @@ lint-release-changelogs:
- git checkout .gitmodules
- "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
after_script:
+ - cp -Rf $HOME/.cabal cabal-cache
- .gitlab/ci.sh clean
tags:
- x86_64-linux
+# Verify that Hadrian builds with stack. Note that we don't actually perform a
+# build of GHC itself; we merely test that the Hadrian executable builds and
+# works (by invoking `hadrian --version`).
+stack-hadrian-build:
+ extends: .validate-linux-hadrian
+ stage: build
+ script:
+ - .gitlab/ci.sh setup
+ - .gitlab/ci.sh configure
+ - hadrian/build-stack --version
+
validate-x86_64-linux-deb9-hadrian:
extends: .validate-linux-hadrian
+ needs: [hadrian-ghc-in-ghci]
stage: build
validate-x86_64-linux-deb9-unreg-hadrian:
extends: .validate-linux-hadrian
+ needs: [validate-x86_64-linux-deb9-hadrian]
stage: full-build
variables:
CONFIGURE_ARGS: --enable-unregisterised
TEST_ENV: "x86_64-linux-deb9-unreg-hadrian"
+validate-x86_64-linux-deb10-hadrian-cross-aarch64:
+ <<: *nightly
+ extends: .validate-linux-hadrian
+ stage: full-build
+ image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb10:$DOCKER_REV"
+ variables:
+ BIN_DIST_NAME: "ghc-x86_64-deb9-linux"
+ rules:
+ - if: '$CI_MERGE_REQUEST_LABELS =~ /.*cross-compilation.*/'
+ variables:
+ CONFIGURE_ARGS: --with-intree-gmp
+ CROSS_TARGET: "aarch64-linux-gnu"
+
+nightly-x86_64-linux-deb10-hadrian-cross-aarch64:
+ <<: *nightly
+ extends: .validate-linux-hadrian
+ stage: full-build
+ variables:
+ CONFIGURE_ARGS: --with-intree-gmp
+ CROSS_TARGET: "aarch64-linux-gnu"
+
+
+
+############################################################
+# GHC-in-GHCi (Hadrian)
+############################################################
+
hadrian-ghc-in-ghci:
stage: quick-build
+ needs: [lint-linters, lint-submods]
image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
before_script:
# workaround for docker permissions
@@ -257,20 +299,56 @@ hadrian-ghc-in-ghci:
tags:
- x86_64-linux
script:
+ - .gitlab/ci.sh setup
- cabal update
- - cd hadrian; cabal new-build --project-file=ci.project; cd ..
+ - cd hadrian; cabal new-build -j`../mk/detect-cpu-count.sh` --with-compiler=$GHC --project-file=ci.project; cd ..
- git clean -xdf && git submodule foreach git clean -xdf
- - .gitlab/ci.sh setup
- - if [[ -d ./cabal-cache ]]; then cp -R ./.cabal-cache ~/.cabal-cache; fi
- ./boot
- ./configure $CONFIGURE_ARGS
# Load ghc-in-ghci then immediately exit and check the modules loaded
- echo ":q" | hadrian/ghci -j`mk/detect-cpu-count.sh`| tail -n2 | grep "Ok,"
+ after_script:
+ - cp -Rf $HOME/.cabal cabal-cache
cache:
key: hadrian-ghci
paths:
- cabal-cache
+############################################################
+# GHC source code linting
+############################################################
+
+.lint-params:
+ stage: lint
+ needs: [lint-submods]
+ tags:
+ - lint
+ image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
+ before_script:
+ - export PATH="/opt/toolchain/bin:$PATH"
+ # workaround for docker permissions
+ - sudo chown ghc:ghc -R .
+ - git submodule sync --recursive
+ - git submodule update --init --recursive
+ - git checkout .gitmodules
+ - .gitlab/ci.sh setup
+ - cabal update
+ - cd hadrian; cabal new-build -j`../mk/detect-cpu-count.sh` --with-compiler=$GHC --project-file=ci.project; cd ..
+ after_script:
+ - cp -Rf $HOME/.cabal cabal-cache
+ variables:
+ GHC_FLAGS: -Werror
+ cache:
+ key: lint
+ paths:
+ - cabal-cache
+
+lint-base:
+ extends: .lint-params
+ script:
+ - hadrian/build -c -j stage1:lib:base
+ - hadrian/build -j lint:base
+
############################################################
# Validation via Pipelines (make)
############################################################
@@ -299,7 +377,9 @@ hadrian-ghc-in-ghci:
#################################
.build-x86_64-freebsd:
+ stage: full-build
extends: .validate
+ needs: [validate-x86_64-linux-deb9-hadrian]
tags:
- x86_64-freebsd
allow_failure: true
@@ -329,23 +409,19 @@ hadrian-ghc-in-ghci:
# Conditional due to lack of builder capacity
validate-x86_64-freebsd:
extends: .build-x86_64-freebsd
- stage: full-build
rules:
- if: '$CI_MERGE_REQUEST_LABELS =~ /.*FreeBSD.*/'
nightly-x86_64-freebsd:
<<: *nightly
extends: .build-x86_64-freebsd
- stage: full-build
release-x86_64-freebsd:
<<: *release
extends: .build-x86_64-freebsd
- stage: full-build
.build-x86_64-freebsd-hadrian:
extends: .validate-hadrian
- stage: full-build
tags:
- x86_64-freebsd
allow_failure: true
@@ -354,9 +430,9 @@ release-x86_64-freebsd:
HADRIAN_ARGS: "--docs=no-sphinx"
GHC_VERSION: 8.6.3
CABAL_INSTALL_VERSION: 3.0.0.0
- BIN_DIST_PREP_TAR_COMP: "ghc-x86_64-portbld-freebsd.tar.xz"
+ BIN_DIST_NAME: "ghc-x86_64-portbld-freebsd"
TEST_ENV: "x86_64-freebsd-hadrian"
- FLAVOUR: "validate"
+ BUILD_FLAVOUR: "validate"
after_script:
- cp -Rf $HOME/.cabal cabal-cache
- .gitlab/ci.sh clean
@@ -372,7 +448,6 @@ release-x86_64-freebsd:
# Disabled due to lack of builder capacity
.validate-x86_64-freebsd-hadrian:
extends: .build-x86_64-freebsd-hadrian
- stage: full-build
#################################
# x86_64-darwin
@@ -380,6 +455,7 @@ release-x86_64-freebsd:
validate-x86_64-darwin:
extends: .validate
+ needs: [validate-x86_64-linux-deb9-hadrian]
stage: full-build
tags:
- x86_64-darwin
@@ -409,6 +485,7 @@ validate-x86_64-darwin:
# Disabled because of OS X CI capacity
.validate-x86_64-darwin-hadrian:
stage: full-build
+ needs: [validate-x86_64-linux-deb9-hadrian]
tags:
- x86_64-darwin
variables:
@@ -418,7 +495,8 @@ validate-x86_64-darwin:
LANG: "en_US.UTF-8"
CONFIGURE_ARGS: --with-intree-gmp
TEST_ENV: "x86_64-darwin-hadrian"
- FLAVOUR: "validate"
+ BIN_DIST_NAME: "ghc-x86_64-apple-darwin"
+ BUILD_FLAVOUR: "validate"
script:
- .gitlab/ci.sh setup
- .gitlab/ci.sh configure
@@ -433,11 +511,12 @@ validate-x86_64-darwin:
reports:
junit: junit.xml
paths:
- - ghc.tar.xz
+ - "$BIN_DIST_NAME.tar.xz"
- junit.xml
.validate-linux:
extends: .validate
+ needs: [validate-x86_64-linux-deb9-hadrian]
tags:
- x86_64-linux
variables:
@@ -460,66 +539,74 @@ validate-x86_64-darwin:
- toolchain
#################################
-# aarch64-linux-deb9
+# aarch64-linux-deb10
#################################
-.build-aarch64-linux-deb9:
+.build-aarch64-linux-deb10:
extends: .validate-linux
stage: full-build
- image: "registry.gitlab.haskell.org/ghc/ci-images/aarch64-linux-deb9:$DOCKER_REV"
+ image: "registry.gitlab.haskell.org/ghc/ci-images/aarch64-linux-deb10:$DOCKER_REV"
allow_failure: true
variables:
- TEST_ENV: "aarch64-linux-deb9"
- BIN_DIST_PREP_TAR_COMP: "ghc-aarch64-linux-deb9.tar.xz"
+ TEST_ENV: "aarch64-linux-deb10"
+ BIN_DIST_PREP_TAR_COMP: "ghc-aarch64-linux-deb10.tar.xz"
cache:
- key: linux-aarch64-deb9
+ key: linux-aarch64-deb10
+ tags:
+ - aarch64-linux
+
+.build-aarch64-linux-deb10-llvm:
+ extends: .build-aarch64-linux-deb10
+ stage: full-build
+ variables:
+ BUILD_FLAVOUR: perf-llvm
tags:
- aarch64-linux
-validate-aarch64-linux-deb9:
- extends: .build-aarch64-linux-deb9
+validate-aarch64-linux-deb10:
+ extends: .build-aarch64-linux-deb10
artifacts:
when: always
expire_in: 2 week
-nightly-aarch64-linux-deb9:
+nightly-aarch64-linux-deb10:
<<: *nightly
- extends: .build-aarch64-linux-deb9
+ extends: .build-aarch64-linux-deb10
variables:
TEST_TYPE: slowtest
#################################
-# armv7-linux-deb9
+# armv7-linux-deb10
#################################
-.build-armv7-linux-deb9:
+.build-armv7-linux-deb10:
extends: .validate-linux
stage: full-build
- image: "registry.gitlab.haskell.org/ghc/ci-images/armv7-linux-deb9:$DOCKER_REV"
+ image: "registry.gitlab.haskell.org/ghc/ci-images/armv7-linux-deb10:$DOCKER_REV"
# Due to linker issues
allow_failure: true
variables:
- TEST_ENV: "armv7-linux-deb9"
- BIN_DIST_PREP_TAR_COMP: "ghc-armv7-linux-deb9.tar.xz"
+ TEST_ENV: "armv7-linux-deb10"
+ BIN_DIST_PREP_TAR_COMP: "ghc-armv7-linux-deb10.tar.xz"
CONFIGURE_ARGS: "--host=armv7-linux-gnueabihf --build=armv7-linux-gnueabihf --target=armv7-linux-gnueabihf"
# N.B. We disable ld.lld explicitly here because it appears to fail
# non-deterministically on ARMv7. See #18280.
LD: "ld.gold"
GccUseLdOpt: "-fuse-ld=gold"
cache:
- key: linux-armv7-deb9
+ key: linux-armv7-deb10
tags:
- armv7-linux
-validate-armv7-linux-deb9:
- extends: .build-armv7-linux-deb9
+validate-armv7-linux-deb10:
+ extends: .build-armv7-linux-deb10
artifacts:
when: always
expire_in: 2 week
-nightly-armv7-linux-deb9:
+nightly-armv7-linux-deb10:
<<: *nightly
- extends: .build-armv7-linux-deb9
+ extends: .build-armv7-linux-deb10
variables:
TEST_TYPE: slowtest
@@ -599,23 +686,6 @@ validate-x86_64-linux-deb9-debug:
when: always
expire_in: 2 week
-validate-x86_64-linux-deb9-llvm:
- extends: .build-x86_64-linux-deb9
- stage: full-build
- rules:
- - if: '$CI_MERGE_REQUEST_LABELS =~ /.*LLVM backend.*/'
- variables:
- BUILD_FLAVOUR: perf-llvm
- TEST_ENV: "x86_64-linux-deb9-llvm"
-
-nightly-x86_64-linux-deb9-llvm:
- <<: *nightly
- extends: .build-x86_64-linux-deb9
- stage: full-build
- variables:
- BUILD_FLAVOUR: perf-llvm
- TEST_ENV: "x86_64-linux-deb9-llvm"
-
validate-x86_64-linux-deb9-integer-simple:
extends: .build-x86_64-linux-deb9
stage: full-build
@@ -691,6 +761,61 @@ release-x86_64-linux-deb10:
<<: *release
extends: .build-x86_64-linux-deb10
+release-x86_64-linux-deb10-dwarf:
+ <<: *release
+ extends: .build-x86_64-linux-deb10
+ variables:
+ CONFIGURE_ARGS: "--enable-dwarf-unwind"
+ BUILD_FLAVOUR: dwarf
+ TEST_ENV: "x86_64-linux-deb10-dwarf"
+ BIN_DIST_PREP_TAR_COMP: "ghc-x86_64-deb10-linux-dwarf.tar.xz"
+
+validate-x86_64-linux-deb10-llvm:
+ extends: .build-x86_64-linux-deb10
+ stage: full-build
+ rules:
+ - if: '$CI_MERGE_REQUEST_LABELS =~ /.*LLVM backend.*/'
+ variables:
+ BUILD_FLAVOUR: perf-llvm
+ TEST_ENV: "x86_64-linux-deb10-llvm"
+
+nightly-x86_64-linux-deb10-llvm:
+ <<: *nightly
+ extends: .build-x86_64-linux-deb10
+ stage: full-build
+ variables:
+ BUILD_FLAVOUR: perf-llvm
+ TEST_ENV: "x86_64-linux-deb10-llvm"
+
+#################################
+# x86_64-linux-ubuntu 20.04
+#################################
+
+.build-x86_64-linux-ubuntu2004:
+ extends: .validate-linux
+ stage: full-build
+ image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-ubuntu20_04:$DOCKER_REV"
+ variables:
+ TEST_ENV: "x86_64-linux-ubuntu2004"
+ BIN_DIST_PREP_TAR_COMP: "./ghc-x86_64-ubuntu2004-linux.tar.xz"
+ cache:
+ key: linux-x86_64-ubuntu2004
+
+# Disabled to alleviate CI load
+.validate-x86_64-linux-ubuntu2004:
+ extends: .build-x86_64-linux-ubuntu2004
+ stage: full-build
+
+nightly-x86_64-linux-ubuntu2004:
+ <<: *nightly
+ extends: .build-x86_64-linux-ubuntu2004
+ variables:
+ TEST_TYPE: slowtest
+
+release-x86_64-linux-ubuntu2004:
+ <<: *release
+ extends: .build-x86_64-linux-ubuntu2004
+
#################################
# x86_64-linux-deb8
#################################
@@ -724,12 +849,12 @@ release-x86_64-linux-deb8:
.build-x86_64-linux-alpine-hadrian:
extends: .validate-linux-hadrian
stage: full-build
- image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-alpine:$DOCKER_REV"
+ image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-alpine3_12:$DOCKER_REV"
# There are currently a few failing tests
allow_failure: true
variables:
TEST_ENV: "x86_64-linux-alpine"
- BIN_DIST_PREP_TAR_COMP: "ghc-x86_64-alpine-linux.tar.xz"
+ BIN_DIST_NAME: "ghc-x86_64-alpine-linux"
# Can't use ld.gold due to #13958.
CONFIGURE_ARGS: "--disable-ld-override"
HADRIAN_ARGS: "--docs=no-sphinx"
@@ -777,13 +902,15 @@ release-x86_64-linux-centos7:
# x86_64-linux-fedora27
#################################
-validate-x86_64-linux-fedora27:
+.build-x86_64-linux-fedora27:
extends: .validate-linux
stage: full-build
image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-fedora27:$DOCKER_REV"
variables:
+ # LLVM 10 is not available for Fedora27
+ LLC: /bin/false
+ OPT: /bin/false
TEST_ENV: "x86_64-linux-fedora27"
- BIN_DIST_PREP_TAR_COMP: "ghc-x86_64-fedora27-linux.tar.xz"
cache:
key: linux-x86_64-fedora27
artifacts:
@@ -792,13 +919,26 @@ validate-x86_64-linux-fedora27:
# longer.
expire_in: 8 week
+validate-x86_64-linux-fedora27:
+ extends: .build-x86_64-linux-fedora27
+ variables:
+ BIN_DIST_PREP_TAR_COMP: "ghc-x86_64-fedora27-linux.tar.xz"
+
+release-x86_64-linux-fedora27-dwarf:
+ <<: *release
+ extends: .build-x86_64-linux-fedora27
+ variables:
+ CONFIGURE_ARGS: "--enable-dwarf-unwind"
+ BUILD_FLAVOUR: dwarf
+ BIN_DIST_PREP_TAR_COMP: "ghc-x86_64-fedora27-linux-dwarf.tar.xz"
+ TEST_ENV: "x86_64-linux-fedora27-dwarf"
+
############################################################
# Validation via Pipelines (Windows)
############################################################
.build-windows:
- # For the reasons given in #17777 this build isn't reliable.
- allow_failure: true
+ needs: [validate-x86_64-linux-deb9-hadrian]
before_script:
- git clean -xdf
@@ -825,7 +965,7 @@ validate-x86_64-linux-fedora27:
extends: .build-windows
stage: full-build
variables:
- FLAVOUR: "validate"
+ BUILD_FLAVOUR: "validate"
# skipping perf tests for now since we build a quick-flavoured GHC,
# which might result in some broken perf tests?
HADRIAN_ARGS: "--docs=no-sphinx --skip-perf"
@@ -843,7 +983,7 @@ validate-x86_64-linux-fedora27:
expire_in: 2 week
when: always
paths:
- - ghc.tar.xz
+ - "$BIN_DIST_NAME.tar.xz"
- junit.xml
.build-x86_64-windows-hadrian:
@@ -851,6 +991,7 @@ validate-x86_64-linux-fedora27:
variables:
MSYSTEM: MINGW64
TEST_ENV: "x86_64-windows"
+ BIN_DIST_NAME: "ghc-x86_64-unknown-mingw32"
cache:
key: "x86_64-windows-$WINDOWS_TOOLCHAIN_VERSION"
@@ -910,45 +1051,8 @@ release-x86_64-windows-integer-simple:
variables:
BIGNUM_BACKEND: native
BUILD_FLAVOUR: "perf"
+ BIN_DIST_NAME: "ghc-x86_64-mingw32-unknown-nogmp"
-############################################################
-# Cleanup
-############################################################
-
-# Note [Cleaning up after shell executor]
-# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-#
-# As noted in [1], gitlab-runner's shell executor doesn't clean up its working
-# directory after builds. Unfortunately, we are forced to use the shell executor
-# on Darwin. To avoid running out of disk space we add a stage at the end of
-# the build to remove the /.../GitLabRunner/builds directory. Since we only run a
-# single build at a time on Darwin this should be safe.
-#
-# We used to have a similar cleanup job on Windows as well however it ended up
-# being quite fragile as we have multiple Windows builders yet there is no
-# guarantee that the cleanup job is run on the same machine as the build itself
-# was run. Consequently we were forced to instead handle cleanup with a separate
-# cleanup cron job on Windows.
-#
-# [1] https://gitlab.com/gitlab-org/gitlab-runner/issues/3856
-
-# See Note [Cleanup after shell executor]
-cleanup-darwin:
- stage: cleanup
- tags:
- - x86_64-darwin
- when: always
- dependencies: []
- before_script:
- - echo "Time to clean up"
- script:
- - echo "Let's go"
- after_script:
- - BUILD_DIR=$CI_PROJECT_DIR
- - echo "Cleaning $BUILD_DIR"
- - cd $HOME
- - rm -Rf $BUILD_DIR/*
- - exit 0
############################################################
# Packaging
@@ -956,6 +1060,9 @@ cleanup-darwin:
doc-tarball:
stage: packaging
+ needs: [validate-x86_64-linux-deb9-debug, validate-x86_64-windows-hadrian, validate-x86_64-linux-deb9-unreg-hadrian]
+ # N.B. Documentation isn't correctly packaged in Hadrian bindists
+ allow_failure: true
tags:
- x86_64-linux
image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
@@ -964,9 +1071,7 @@ doc-tarball:
- validate-x86_64-windows-hadrian
variables:
LINUX_BINDIST: "ghc-x86_64-deb9-linux-debug.tar.xz"
- WINDOWS_BINDIST: "ghc-x86_64-mingw32.tar.xz"
- # Due to Windows allow_failure
- allow_failure: true
+ WINDOWS_BINDIST: "ghc-x86_64-unknown-mingw32.tar.xz"
artifacts:
paths:
- haddock.html.tar.xz
@@ -991,6 +1096,7 @@ doc-tarball:
source-tarball:
stage: packaging
+ needs: [validate-x86_64-linux-deb9-unreg-hadrian]
tags:
- x86_64-linux
image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
@@ -1025,6 +1131,7 @@ source-tarball:
.hackage:
stage: testing
+ needs: [doc-tarball]
image: ghcci/x86_64-linux-deb9:0.2
tags:
- x86_64-linux
@@ -1053,6 +1160,7 @@ nightly-hackage:
perf-nofib:
stage: testing
+ needs: [validate-x86_64-linux-deb9-dwarf, doc-tarball]
dependencies:
- validate-x86_64-linux-deb9-dwarf
image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
@@ -1072,8 +1180,8 @@ perf-nofib:
make install
popd
rm -Rf tmp
- - export BOOT_HC=$(which ghc)
- - cabal update; cabal install -w $BOOT_HC regex-compat
+ - export BOOT_HC=$GHC
+ - cabal update; cabal install -w "$BOOT_HC" --lib regex-compat
- export PATH=$root/bin:$PATH
- make -C nofib boot mode=fast -j$CPUS
- "make -C nofib EXTRA_RUNTEST_OPTS='-cachegrind +RTS -V0 -RTS' NoFibRuns=1 mode=fast -j$CPUS 2>&1 | tee nofib.log"
@@ -1083,17 +1191,52 @@ perf-nofib:
paths:
- nofib.log
+############################################################
+# Ad-hoc performance testing
+############################################################
+
+perf:
+ stage: testing
+ needs: [validate-x86_64-linux-deb9-dwarf, doc-tarball]
+ dependencies:
+ - validate-x86_64-linux-deb9-dwarf
+ image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
+ rules:
+ - if: $CI_MERGE_REQUEST_ID
+ - if: '$CI_COMMIT_BRANCH == "master"'
+ - if: '$CI_COMMIT_BRANCH =~ /ghc-[0.9]+\.[0-9]+/'
+ tags:
+ - x86_64-linux-perf
+ script:
+ - root=$(pwd)/ghc
+ - |
+ mkdir tmp
+ tar -xf ghc-x86_64-deb9-linux-dwarf.tar.xz -C tmp
+ pushd tmp/ghc-*/
+ ./configure --prefix=$root
+ make install
+ popd
+ rm -Rf tmp
+ - export BOOT_HC=$(which ghc)
+ - export HC=$root/bin/ghc
+ - .gitlab/ci.sh perf_test
+ artifacts:
+ expire_in: 12 week
+ when: always
+ paths:
+ - out
+
+
############################################################
# Documentation deployment via GitLab Pages
############################################################
pages:
stage: deploy
+ needs: [doc-tarball]
dependencies:
- doc-tarball
image: ghcci/x86_64-linux-deb9:0.2
- # Due to Windows allow_failure
- allow_failure: true
tags:
- x86_64-linux
script:
@@ -1109,7 +1252,9 @@ pages:
EOF
- cp -f index.html public/doc
rules:
- - if: '$CI_COMMIT_BRANCH == "master"'
+ # N.B. only run this on ghc/ghc since the deployed pages are quite large
+ # and we only serve GitLab Pages for ghc/ghc.
+ - if: '$CI_COMMIT_BRANCH == "master" && $CI_PROJECT_NAMESPACE == "ghc"'
artifacts:
paths:
- public
=====================================
.gitlab/ci.sh
=====================================
@@ -2,62 +2,78 @@
# shellcheck disable=SC2230
# This is the primary driver of the GitLab CI infrastructure.
+# Run `ci.sh usage` for usage information.
+
set -e -o pipefail
# Configuration:
-hackage_index_state="@1579718451"
-
+hackage_index_state="2020-09-14T19:30:43Z"
+MIN_HAPPY_VERSION="1.20"
MIN_ALEX_VERSION="3.2"
-# Colors
-BLACK="0;30"
-GRAY="1;30"
-RED="0;31"
-LT_RED="1;31"
-BROWN="0;33"
-LT_BROWN="1;33"
-GREEN="0;32"
-LT_GREEN="1;32"
-BLUE="0;34"
-LT_BLUE="1;34"
-PURPLE="0;35"
-LT_PURPLE="1;35"
-CYAN="0;36"
-LT_CYAN="1;36"
-WHITE="1;37"
-LT_GRAY="0;37"
-
-# GitLab Pipelines log section delimiters
-# https://gitlab.com/gitlab-org/gitlab-foss/issues/14664
-start_section() {
- name="$1"
- echo -e "section_start:$(date +%s):$name\015\033[0K"
-}
+TOP="$(pwd)"
+if [ ! -d "$TOP/.gitlab" ]; then
+ echo "This script expects to be run from the root of a ghc checkout"
+fi
-end_section() {
- name="$1"
- echo -e "section_end:$(date +%s):$name\015\033[0K"
-}
+source $TOP/.gitlab/common.sh
-echo_color() {
- local color="$1"
- local msg="$2"
- echo -e "\033[${color}m${msg}\033[0m"
-}
+function usage() {
+ cat <<EOF
+$0 - GHC continuous integration driver
-error() { echo_color "${RED}" "$1"; }
-warn() { echo_color "${LT_BROWN}" "$1"; }
-info() { echo_color "${LT_BLUE}" "$1"; }
+Common Modes:
-fail() { error "error: $1"; exit 1; }
+ usage Show this usage message.
+ setup Prepare environment for a build.
+ configure Run ./configure.
+ clean Clean the tree
+ shell Run an interactive shell with a configured build environment.
-function run() {
- info "Running $*..."
- "$@" || ( error "$* failed"; return 1; )
-}
+Make build system:
-TOP="$(pwd)"
+ build_make Build GHC via the make build system
+ test_make Test GHC via the make build system
+
+Hadrian build system
+ build_hadrian Build GHC via the Hadrian build system
+ test_hadrian Test GHC via the Hadrian build system
+
+Environment variables affecting both build systems:
+
+ CROSS_TARGET Triple of cross-compilation target.
+ VERBOSE Set to non-empty for verbose build output
+ MSYSTEM (Windows-only) Which platform to build form (MINGW64 or MINGW32).
+
+Environment variables determining build configuration of Make system:
+
+ BUILD_FLAVOUR Which flavour to build.
+ BUILD_SPHINX_HTML Whether to build Sphinx HTML documentation.
+ BUILD_SPHINX_PDF Whether to build Sphinx PDF documentation.
+ INTEGER_LIBRARY Which integer library to use (integer-simple or integer-gmp).
+ HADDOCK_HYPERLINKED_SOURCES
+ Whether to build hyperlinked Haddock sources.
+ TEST_TYPE Which test rule to run.
+
+Environment variables determining build configuration of Hadrian system:
+
+ BUILD_FLAVOUR Which flavour to build.
+
+Environment variables determining bootstrap toolchain (Linux):
+
+ GHC Path of GHC executable to use for bootstrapping.
+ CABAL Path of cabal-install executable to use for bootstrapping.
+ ALEX Path of alex executable to use for bootstrapping.
+ HAPPY Path of alex executable to use for bootstrapping.
+
+Environment variables determining bootstrap toolchain (non-Linux):
+
+ GHC_VERSION Which GHC version to fetch for bootstrapping.
+ CABAL_INSTALL_VERSION
+ Cabal-install version to fetch for bootstrapping.
+EOF
+}
function setup_locale() {
# Musl doesn't provide locale support at all...
@@ -95,11 +111,11 @@ function setup_locale() {
function mingw_init() {
case "$MSYSTEM" in
MINGW32)
- triple="i386-unknown-mingw32"
+ target_triple="i386-unknown-mingw32"
boot_triple="i386-unknown-mingw32" # triple of bootstrap GHC
;;
MINGW64)
- triple="x86_64-unknown-mingw32"
+ target_triple="x86_64-unknown-mingw32"
boot_triple="x86_64-unknown-mingw32" # triple of bootstrap GHC
;;
*)
@@ -154,21 +170,24 @@ function show_tool() {
function set_toolchain_paths() {
needs_toolchain=1
case "$(uname)" in
- Linux) needs_toolchain="" ;;
+ Linux) needs_toolchain="0" ;;
*) ;;
esac
- if [[ -n "$needs_toolchain" ]]; then
+ if [[ "$needs_toolchain" = 1 ]]; then
# These are populated by setup_toolchain
GHC="$toolchain/bin/ghc$exe"
CABAL="$toolchain/bin/cabal$exe"
HAPPY="$toolchain/bin/happy$exe"
ALEX="$toolchain/bin/alex$exe"
else
- GHC="$(which ghc)"
- CABAL="/usr/local/bin/cabal"
- HAPPY="$HOME/.cabal/bin/happy"
- ALEX="$HOME/.cabal/bin/alex"
+ # These are generally set by the Docker image but
+ # we provide these handy fallbacks in case the
+ # script isn't run from within a GHC CI docker image.
+ if [ -z "$GHC" ]; then GHC="$(which ghc)"; fi
+ if [ -z "$CABAL" ]; then GHC="$(which cabal)"; fi
+ if [ -z "$HAPPY" ]; then GHC="$(which happy)"; fi
+ if [ -z "$ALEX" ]; then GHC="$(which alex)"; fi
fi
export GHC
@@ -207,12 +226,12 @@ function setup() {
}
function fetch_ghc() {
- local v="$GHC_VERSION"
- if [[ -z "$v" ]]; then
- fail "GHC_VERSION is not set"
- fi
-
if [ ! -e "$GHC" ]; then
+ local v="$GHC_VERSION"
+ if [[ -z "$v" ]]; then
+ fail "neither GHC nor GHC_VERSION are not set"
+ fi
+
start_section "fetch GHC"
url="https://downloads.haskell.org/~ghc/${GHC_VERSION}/ghc-${GHC_VERSION}-${boot_triple}.tar.xz"
info "Fetching GHC binary distribution from $url..."
@@ -236,12 +255,12 @@ function fetch_ghc() {
}
function fetch_cabal() {
- local v="$CABAL_INSTALL_VERSION"
- if [[ -z "$v" ]]; then
- fail "CABAL_INSTALL_VERSION is not set"
- fi
-
if [ ! -e "$CABAL" ]; then
+ local v="$CABAL_INSTALL_VERSION"
+ if [[ -z "$v" ]]; then
+ fail "neither CABAL nor CABAL_INSTALL_VERSION are not set"
+ fi
+
start_section "fetch GHC"
case "$(uname)" in
# N.B. Windows uses zip whereas all others use .tar.xz
@@ -284,6 +303,7 @@ function setup_toolchain() {
fetch_cabal
cabal_install="$CABAL v2-install \
+ --with-compiler=$GHC \
--index-state=$hackage_index_state \
--installdir=$toolchain/bin \
--overwrite-policy=always"
@@ -297,7 +317,7 @@ function setup_toolchain() {
cabal update
info "Building happy..."
- $cabal_install happy --constraint="happy==1.19.*"
+ $cabal_install happy --constraint="happy>=$MIN_HAPPY_VERSION"
info "Building alex..."
$cabal_install alex --constraint="alex>=$MIN_ALEX_VERSION"
@@ -358,8 +378,8 @@ function configure() {
end_section "booting"
local target_args=""
- if [[ -n "$triple" ]]; then
- target_args="--target=$triple"
+ if [[ -n "$target_triple" ]]; then
+ target_args="--target=$target_triple"
fi
start_section "configuring"
@@ -379,6 +399,11 @@ function build_make() {
if [[ -z "$BIN_DIST_PREP_TAR_COMP" ]]; then
fail "BIN_DIST_PREP_TAR_COMP is not set"
fi
+ if [[ -n "$VERBOSE" ]]; then
+ MAKE_ARGS="$MAKE_ARGS V=1"
+ else
+ MAKE_ARGS="$MAKE_ARGS V=0"
+ fi
echo "include mk/flavours/${BUILD_FLAVOUR}.mk" > mk/build.mk
echo 'GhcLibHcOpts+=-haddock' >> mk/build.mk
@@ -405,6 +430,11 @@ function determine_metric_baseline() {
}
function test_make() {
+ if [ -n "$CROSS_TARGET" ]; then
+ info "Can't test cross-compiled build."
+ return
+ fi
+
run "$MAKE" test_bindist TEST_PREP=YES
run "$MAKE" V=0 test \
THREADS="$cores" \
@@ -412,16 +442,24 @@ function test_make() {
}
function build_hadrian() {
- if [ -z "$FLAVOUR" ]; then
- fail "FLAVOUR not set"
+ if [ -z "$BUILD_FLAVOUR" ]; then
+ fail "BUILD_FLAVOUR not set"
+ fi
+ if [ -z "$BIN_DIST_NAME" ]; then
+ fail "BIN_DIST_NAME not set"
fi
run_hadrian binary-dist
- mv _build/bindist/ghc*.tar.xz ghc.tar.xz
+ mv _build/bindist/ghc*.tar.xz $BIN_DIST_NAME.tar.xz
}
function test_hadrian() {
+ if [ -n "$CROSS_TARGET" ]; then
+ info "Can't test cross-compiled build."
+ return
+ fi
+
cd _build/bindist/ghc-*/
run ./configure --prefix="$TOP"/_build/install
run "$MAKE" install
@@ -433,6 +471,34 @@ function test_hadrian() {
--test-compiler="$TOP"/_build/install/bin/ghc
}
+function cabal_test() {
+ if [ -z "$OUT" ]; then
+ fail "OUT not set"
+ fi
+
+ start_section "Cabal test: $OUT"
+ mkdir -p "$OUT"
+ run "$HC" \
+ -hidir tmp -odir tmp -fforce-recomp \
+ -ddump-to-file -dumpdir "$OUT/dumps" -ddump-timings \
+ +RTS --machine-readable "-t$OUT/rts.log" -RTS \
+ -package mtl -ilibraries/Cabal/Cabal libraries/Cabal/Cabal/Setup.hs \
+ $@
+ rm -Rf tmp
+ end_section "Cabal test: $OUT"
+}
+
+function run_perf_test() {
+ if [ -z "$HC" ]; then
+ fail "HC not set"
+ fi
+
+ mkdir -p out
+ OUT=out/Cabal-O0 cabal_test -O0
+ OUT=out/Cabal-O1 cabal_test -O1
+ OUT=out/Cabal-O2 cabal_test -O2
+}
+
function clean() {
rm -R tmp
run "$MAKE" --quiet clean || true
@@ -441,8 +507,9 @@ function clean() {
function run_hadrian() {
if [ -z "$BIGNUM_BACKEND" ]; then BIGNUM_BACKEND="gmp"; fi
+ if [ -n "$VERBOSE" ]; then HADRIAN_ARGS="$HADRIAN_ARGS -V"; fi
run hadrian/build-cabal \
- --flavour="$FLAVOUR" \
+ --flavour="$BUILD_FLAVOUR" \
-j"$cores" \
--broken-test="$BROKEN_TESTS" \
--bignum=$BIGNUM_BACKEND \
@@ -480,9 +547,15 @@ case "$(uname)" in
*) fail "uname $(uname) is not supported" ;;
esac
+if [ -n "$CROSS_TARGET" ]; then
+ info "Cross-compiling for $CROSS_TARGET..."
+ target_triple="$CROSS_TARGET"
+fi
+
set_toolchain_paths
case $1 in
+ usage) usage ;;
setup) setup && cleanup_submodules ;;
configure) configure ;;
build_make) build_make ;;
@@ -503,6 +576,7 @@ case $1 in
push_perf_notes
exit $res ;;
run_hadrian) run_hadrian $@ ;;
+ perf_test) run_perf_test ;;
clean) clean ;;
shell) shell $@ ;;
*) fail "unknown mode $1" ;;
=====================================
.gitlab/common.sh
=====================================
@@ -0,0 +1,50 @@
+# Common bash utilities
+# ----------------------
+
+# Colors
+BLACK="0;30"
+GRAY="1;30"
+RED="0;31"
+LT_RED="1;31"
+BROWN="0;33"
+LT_BROWN="1;33"
+GREEN="0;32"
+LT_GREEN="1;32"
+BLUE="0;34"
+LT_BLUE="1;34"
+PURPLE="0;35"
+LT_PURPLE="1;35"
+CYAN="0;36"
+LT_CYAN="1;36"
+WHITE="1;37"
+LT_GRAY="0;37"
+
+# GitLab Pipelines log section delimiters
+# https://gitlab.com/gitlab-org/gitlab-foss/issues/14664
+start_section() {
+ name="$1"
+ echo -e "section_start:$(date +%s):$name\015\033[0K"
+}
+
+end_section() {
+ name="$1"
+ echo -e "section_end:$(date +%s):$name\015\033[0K"
+}
+
+echo_color() {
+ local color="$1"
+ local msg="$2"
+ echo -e "\033[${color}m${msg}\033[0m"
+}
+
+error() { echo_color "${RED}" "$1"; }
+warn() { echo_color "${LT_BROWN}" "$1"; }
+info() { echo_color "${LT_BLUE}" "$1"; }
+
+fail() { error "error: $1"; exit 1; }
+
+function run() {
+ info "Running $*..."
+ "$@" || ( error "$* failed"; return 1; )
+}
+
View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/commit/46e8a41ab3d9f0d7c7eae69f74689d6bf1f77e27
--
View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/commit/46e8a41ab3d9f0d7c7eae69f74689d6bf1f77e27
You're receiving this email because of your account on gitlab.haskell.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mail.haskell.org/pipermail/ghc-commits/attachments/20201129/5c3cac85/attachment-0001.html>
More information about the ghc-commits
mailing list