[Git][ghc/ghc][wip/refactor-gen-ci] gitlab-ci: Refactor compilation of gen_ci
Ben Gamari (@bgamari)
gitlab at gitlab.haskell.org
Thu Jun 29 14:46:05 UTC 2023
Ben Gamari pushed to branch wip/refactor-gen-ci at Glasgow Haskell Compiler / GHC
Commits:
8f6b7716 by Ben Gamari at 2023-06-29T10:45:58-04:00
gitlab-ci: Refactor compilation of gen_ci
Flakify and document it, making it far less sensitive to the build
environment.
- - - - -
11 changed files:
- .gitignore
- .gitlab-ci.yml
- + .gitlab/generate-ci/LICENSE
- + .gitlab/generate-ci/README.mkd
- + .gitlab/generate-ci/flake.lock
- + .gitlab/generate-ci/flake.nix
- .gitlab/gen_ci.hs → .gitlab/generate-ci/gen_ci.hs
- + .gitlab/generate-ci/generate-ci.cabal
- + .gitlab/generate-ci/update-ci
- − .gitlab/generate_job_metadata
- − .gitlab/generate_jobs
Changes:
=====================================
.gitignore
=====================================
@@ -97,6 +97,7 @@ _darcs/
# -----------------------------------------------------------------------------
# specific generated files
+/.gitlab/jobs-metadata.json
/bindist-list
/bindist-list.uniq
/bindistprep/
=====================================
.gitlab-ci.yml
=====================================
@@ -274,13 +274,10 @@ lint-ci-config:
GIT_SUBMODULE_STRATEGY: none
before_script:
- echo "experimental-features = nix-command flakes" >> /etc/nix/nix.conf
- - nix-channel --update
script:
- - .gitlab/generate_jobs
+ - nix run .gitlab/generate-ci#update-ci
# 1 if .gitlab/generate_jobs changed the output of the generated config
- nix shell nixpkgs#git -c git diff --exit-code
- # And run this to just make sure that works
- - .gitlab/generate_job_metadata
dependencies: []
lint-submods:
@@ -1020,7 +1017,7 @@ project-version:
- PipelineYear="$(date -d $CI_PIPELINE_CREATED_AT +%Y)"
- nix shell nixpkgs#wget -c wget "https://ghc.gitlab.haskell.org/ghcup-metadata/ghcup-nightlies-$PipelineYear-0.0.7.yaml" -O ghcup-0.0.7.yaml
- - .gitlab/generate_job_metadata
+ - nix run .gitlab/generate-ci#update-ci
artifacts:
paths:
=====================================
.gitlab/generate-ci/LICENSE
=====================================
@@ -0,0 +1,30 @@
+Copyright (c) 2023, The GHC Developers
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+ * Redistributions in binary form must reproduce the above
+ copyright notice, this list of conditions and the following
+ disclaimer in the documentation and/or other materials provided
+ with the distribution.
+
+ * Neither the name of The GHC Developers nor the names of other
+ contributors may be used to endorse or promote products derived
+ from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
=====================================
.gitlab/generate-ci/README.mkd
=====================================
@@ -0,0 +1,34 @@
+# generate-ci
+
+This is the generator for GHC's GitLab CI infrastructure. In particular, this
+generates two outputs:
+
+ * `.gitlab/jobs.yaml`, which is a YAML (or, strictly speaking, JSON)
+ file which defines the bulk of the validation, nightly, and release jobs of
+ GHC's CI.
+
+ * `.gitlab/jobs-metadata.json`, which is a mapping between platforms and
+ produced binary distribution names used when producing `ghcup` metadata
+ for nightly pipeline artifacts.
+
+Both of these are generated by the `update-ci` script, although only
+`jobs.yaml` is committed in the repository.
+
+## Modifying the CI configuration (nix)
+
+The jobs are defined in `gen_ci.hs`. After modifying this you can run
+```sh
+nix run .gitlab/generate-ci#update-ci
+```
+from the top of the GHC repository to update the generated configuration.
+
+
+## Modifying the CI configuration (without nix)
+
+One can run `update-ci` without Nix as follows (assuming one has `jq`,
+`cabal-install`, and GHC installed):
+```sh
+$ cabal build generate-ci
+$ PATH="$(dirname $(cabal list-bin generate-ci)):$PATH"
+$ ./update-ci
+```
=====================================
.gitlab/generate-ci/flake.lock
=====================================
@@ -0,0 +1,59 @@
+{
+ "nodes": {
+ "flake-utils": {
+ "inputs": {
+ "systems": "systems"
+ },
+ "locked": {
+ "lastModified": 1687709756,
+ "narHash": "sha256-Y5wKlQSkgEK2weWdOu4J3riRd+kV/VCgHsqLNTTWQ/0=",
+ "owner": "numtide",
+ "repo": "flake-utils",
+ "rev": "dbabf0ca0c0c4bce6ea5eaf65af5cb694d2082c7",
+ "type": "github"
+ },
+ "original": {
+ "owner": "numtide",
+ "repo": "flake-utils",
+ "type": "github"
+ }
+ },
+ "nixpkgs": {
+ "locked": {
+ "lastModified": 1687886075,
+ "narHash": "sha256-PeayJDDDy+uw1Ats4moZnRdL1OFuZm1Tj+KiHlD67+o=",
+ "owner": "NixOS",
+ "repo": "nixpkgs",
+ "rev": "a565059a348422af5af9026b5174dc5c0dcefdae",
+ "type": "github"
+ },
+ "original": {
+ "id": "nixpkgs",
+ "type": "indirect"
+ }
+ },
+ "root": {
+ "inputs": {
+ "flake-utils": "flake-utils",
+ "nixpkgs": "nixpkgs"
+ }
+ },
+ "systems": {
+ "locked": {
+ "lastModified": 1681028828,
+ "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+ "owner": "nix-systems",
+ "repo": "default",
+ "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+ "type": "github"
+ },
+ "original": {
+ "owner": "nix-systems",
+ "repo": "default",
+ "type": "github"
+ }
+ }
+ },
+ "root": "root",
+ "version": 7
+}
=====================================
.gitlab/generate-ci/flake.nix
=====================================
@@ -0,0 +1,33 @@
+{
+ description = "GHC CI Generator";
+
+ inputs.flake-utils.url = "github:numtide/flake-utils";
+
+ outputs = { self, nixpkgs, flake-utils }:
+ flake-utils.lib.eachDefaultSystem (system:
+ let pkgs = nixpkgs.legacyPackages.${system}; in
+ {
+ packages = rec {
+ update-ci = pkgs.runCommand "update-ci" {
+ nativeBuildInputs = with pkgs; [ makeWrapper ];
+ } ''
+ mkdir -p $out/bin
+ makeWrapper ${./update-ci} $out/bin/update-ci \
+ --prefix PATH : ${with pkgs; lib.makeBinPath [ generate-ci jq gitMinimal ]}
+ '';
+
+ generate-ci = pkgs.haskellPackages.callCabal2nix "generate-ci" ./. {};
+
+ default = update-ci;
+ };
+
+ apps = rec {
+ update-ci = flake-utils.lib.mkApp {
+ drv = self.packages.${system}.update-ci;
+ };
+
+ default = update-ci;
+ };
+ }
+ );
+}
=====================================
.gitlab/gen_ci.hs → .gitlab/generate-ci/gen_ci.hs
=====================================
@@ -3,9 +3,6 @@
{-# LANGUAGE OverloadedStrings #-}
{-# LANGUAGE DeriveFunctor #-}
{-# LANGUAGE GeneralizedNewtypeDeriving #-}
-{- cabal:
-build-depends: base, aeson >= 1.8.1, containers, bytestring
--}
import Data.Aeson as A
import qualified Data.Map as Map
=====================================
.gitlab/generate-ci/generate-ci.cabal
=====================================
@@ -0,0 +1,15 @@
+cabal-version: 3.0
+name: generate-ci
+version: 0.1.0.0
+license: BSD-3-Clause
+license-file: LICENSE
+build-type: Simple
+
+executable generate-ci
+ main-is: gen_ci.hs
+ ghc-options: -Wall
+ build-depends: base,
+ containers,
+ bytestring,
+ aeson >= 1.8.1
+ default-language: Haskell2010
=====================================
.gitlab/generate-ci/update-ci
=====================================
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+out_dir="$(git rev-parse --show-toplevel)/.gitlab"
+
+# Update job metadata for ghcup
+generate-ci metadata "$out_dir/jobs-metadata.json"
+echo "Updated $out_dir/jobs-metadata.json"
+
+# Update CI jobs
+tmp="$(mktemp)"
+generate-ci gitlab "$tmp"
+rm -f "$out_dir/jobs.yaml"
+echo "### THIS IS A GENERATED FILE, DO NOT MODIFY DIRECTLY" > "$out_dir/jobs.yaml"
+cat "$tmp" | jq >> "$out_dir/jobs.yaml"
+rm "$tmp"
+echo "Updated $out_dir/jobs.yaml"
=====================================
.gitlab/generate_job_metadata deleted
=====================================
@@ -1,5 +0,0 @@
-#! /usr/bin/env nix-shell
-#!nix-shell -i bash -p cabal-install "haskell.packages.ghc92.ghcWithPackages (pkgs: with pkgs; [aeson])" git jq
-
-cd "$(dirname "${BASH_SOURCE[0]}")"
-cabal run gen_ci -- metadata jobs-metadata.json
=====================================
.gitlab/generate_jobs deleted
=====================================
@@ -1,13 +0,0 @@
-#!/usr/bin/env nix-shell
-#!nix-shell -i bash -p cabal-install "haskell.packages.ghc92.ghcWithPackages (pkgs: with pkgs; [aeson])" git jq
-
-# shellcheck shell=bash
-
-set -euo pipefail
-
-cd "$(dirname "${BASH_SOURCE[0]}")"
-tmp=$(mktemp)
-cabal run gen_ci -- gitlab $tmp
-rm -f jobs.yaml
-echo "### THIS IS A GENERATED FILE, DO NOT MODIFY DIRECTLY" > jobs.yaml
-cat $tmp | jq | tee -a jobs.yaml
View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/commit/8f6b771621922cab934ca7d84dd15b3ac4715e19
--
View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/commit/8f6b771621922cab934ca7d84dd15b3ac4715e19
You're receiving this email because of your account on gitlab.haskell.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mail.haskell.org/pipermail/ghc-commits/attachments/20230629/ad22955d/attachment-0001.html>
More information about the ghc-commits
mailing list