[Git][ghc/ghc][wip/refactor-gen-ci] gitlab-ci: Refactor compilation of gen_ci

Ben Gamari (@bgamari) gitlab at gitlab.haskell.org
Thu Jun 29 14:46:05 UTC 2023



Ben Gamari pushed to branch wip/refactor-gen-ci at Glasgow Haskell Compiler / GHC


Commits:
8f6b7716 by Ben Gamari at 2023-06-29T10:45:58-04:00
gitlab-ci: Refactor compilation of gen_ci

Flakify and document it, making it far less sensitive to the build
environment.

- - - - -


11 changed files:

- .gitignore
- .gitlab-ci.yml
- + .gitlab/generate-ci/LICENSE
- + .gitlab/generate-ci/README.mkd
- + .gitlab/generate-ci/flake.lock
- + .gitlab/generate-ci/flake.nix
- .gitlab/gen_ci.hs → .gitlab/generate-ci/gen_ci.hs
- + .gitlab/generate-ci/generate-ci.cabal
- + .gitlab/generate-ci/update-ci
- − .gitlab/generate_job_metadata
- − .gitlab/generate_jobs


Changes:

=====================================
.gitignore
=====================================
@@ -97,6 +97,7 @@ _darcs/
 # -----------------------------------------------------------------------------
 # specific generated files
 
+/.gitlab/jobs-metadata.json
 /bindist-list
 /bindist-list.uniq
 /bindistprep/


=====================================
.gitlab-ci.yml
=====================================
@@ -274,13 +274,10 @@ lint-ci-config:
     GIT_SUBMODULE_STRATEGY: none
   before_script:
     - echo "experimental-features = nix-command flakes" >> /etc/nix/nix.conf
-    - nix-channel --update
   script:
-    - .gitlab/generate_jobs
+    - nix run .gitlab/generate-ci#update-ci
     # 1 if .gitlab/generate_jobs changed the output of the generated config
     - nix shell nixpkgs#git -c git diff --exit-code
-    # And run this to just make sure that works
-    - .gitlab/generate_job_metadata
   dependencies: []
 
 lint-submods:
@@ -1020,7 +1017,7 @@ project-version:
     - PipelineYear="$(date -d $CI_PIPELINE_CREATED_AT +%Y)"
     - nix shell nixpkgs#wget -c wget "https://ghc.gitlab.haskell.org/ghcup-metadata/ghcup-nightlies-$PipelineYear-0.0.7.yaml" -O ghcup-0.0.7.yaml
 
-    - .gitlab/generate_job_metadata
+    - nix run .gitlab/generate-ci#update-ci
 
   artifacts:
     paths:


=====================================
.gitlab/generate-ci/LICENSE
=====================================
@@ -0,0 +1,30 @@
+Copyright (c) 2023, The GHC Developers
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+
+    * Redistributions in binary form must reproduce the above
+      copyright notice, this list of conditions and the following
+      disclaimer in the documentation and/or other materials provided
+      with the distribution.
+
+    * Neither the name of The GHC Developers nor the names of other
+      contributors may be used to endorse or promote products derived
+      from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.


=====================================
.gitlab/generate-ci/README.mkd
=====================================
@@ -0,0 +1,34 @@
+# generate-ci
+
+This is the generator for GHC's GitLab CI infrastructure. In particular, this
+generates two outputs:
+
+ * `.gitlab/jobs.yaml`, which is a YAML (or, strictly speaking, JSON)
+   file which defines the bulk of the validation, nightly, and release jobs of
+   GHC's CI.
+
+ * `.gitlab/jobs-metadata.json`, which is a mapping between platforms and
+   produced binary distribution names used when producing `ghcup` metadata
+   for nightly pipeline artifacts.
+
+Both of these are generated by the `update-ci` script, although only
+`jobs.yaml` is committed in the repository.
+
+## Modifying the CI configuration (nix)
+
+The jobs are defined in `gen_ci.hs`. After modifying this you can run
+```sh
+nix run .gitlab/generate-ci#update-ci
+```
+from the top of the GHC repository to update the generated configuration.
+
+
+## Modifying the CI configuration (without nix)
+
+One can run `update-ci` without Nix as follows (assuming one has `jq`,
+`cabal-install`, and GHC installed):
+```sh
+$ cabal build generate-ci
+$ PATH="$(dirname $(cabal list-bin generate-ci)):$PATH"
+$ ./update-ci
+```


=====================================
.gitlab/generate-ci/flake.lock
=====================================
@@ -0,0 +1,59 @@
+{
+  "nodes": {
+    "flake-utils": {
+      "inputs": {
+        "systems": "systems"
+      },
+      "locked": {
+        "lastModified": 1687709756,
+        "narHash": "sha256-Y5wKlQSkgEK2weWdOu4J3riRd+kV/VCgHsqLNTTWQ/0=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "dbabf0ca0c0c4bce6ea5eaf65af5cb694d2082c7",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1687886075,
+        "narHash": "sha256-PeayJDDDy+uw1Ats4moZnRdL1OFuZm1Tj+KiHlD67+o=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "a565059a348422af5af9026b5174dc5c0dcefdae",
+        "type": "github"
+      },
+      "original": {
+        "id": "nixpkgs",
+        "type": "indirect"
+      }
+    },
+    "root": {
+      "inputs": {
+        "flake-utils": "flake-utils",
+        "nixpkgs": "nixpkgs"
+      }
+    },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}


=====================================
.gitlab/generate-ci/flake.nix
=====================================
@@ -0,0 +1,33 @@
+{
+  description = "GHC CI Generator";
+
+  inputs.flake-utils.url = "github:numtide/flake-utils";
+
+  outputs = { self, nixpkgs, flake-utils }:
+    flake-utils.lib.eachDefaultSystem (system:
+      let pkgs = nixpkgs.legacyPackages.${system}; in
+      {
+        packages = rec {
+          update-ci = pkgs.runCommand "update-ci" {
+            nativeBuildInputs = with pkgs; [ makeWrapper ];
+          } ''
+            mkdir -p $out/bin
+            makeWrapper ${./update-ci} $out/bin/update-ci \
+              --prefix PATH : ${with pkgs; lib.makeBinPath [ generate-ci jq gitMinimal ]}
+          '';
+
+          generate-ci = pkgs.haskellPackages.callCabal2nix "generate-ci" ./. {};
+
+          default = update-ci;
+        };
+
+        apps = rec {
+          update-ci = flake-utils.lib.mkApp {
+            drv = self.packages.${system}.update-ci;
+          };
+
+          default = update-ci;
+        };
+      }
+    );
+}


=====================================
.gitlab/gen_ci.hs → .gitlab/generate-ci/gen_ci.hs
=====================================
@@ -3,9 +3,6 @@
 {-# LANGUAGE OverloadedStrings #-}
 {-# LANGUAGE DeriveFunctor #-}
 {-# LANGUAGE GeneralizedNewtypeDeriving #-}
-{- cabal:
-build-depends: base, aeson >= 1.8.1, containers, bytestring
--}
 
 import Data.Aeson as A
 import qualified Data.Map as Map


=====================================
.gitlab/generate-ci/generate-ci.cabal
=====================================
@@ -0,0 +1,15 @@
+cabal-version:      3.0
+name:               generate-ci
+version:            0.1.0.0
+license:            BSD-3-Clause
+license-file:       LICENSE
+build-type:         Simple
+
+executable generate-ci
+    main-is:          gen_ci.hs
+    ghc-options:      -Wall
+    build-depends:    base,
+                      containers,
+                      bytestring,
+                      aeson >= 1.8.1
+    default-language: Haskell2010


=====================================
.gitlab/generate-ci/update-ci
=====================================
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+out_dir="$(git rev-parse --show-toplevel)/.gitlab"
+
+# Update job metadata for ghcup
+generate-ci metadata "$out_dir/jobs-metadata.json"
+echo "Updated $out_dir/jobs-metadata.json"
+
+# Update CI jobs
+tmp="$(mktemp)"
+generate-ci gitlab "$tmp"
+rm -f "$out_dir/jobs.yaml"
+echo "### THIS IS A GENERATED FILE, DO NOT MODIFY DIRECTLY" > "$out_dir/jobs.yaml"
+cat "$tmp" | jq >> "$out_dir/jobs.yaml"
+rm "$tmp"
+echo "Updated $out_dir/jobs.yaml"


=====================================
.gitlab/generate_job_metadata deleted
=====================================
@@ -1,5 +0,0 @@
-#! /usr/bin/env nix-shell
-#!nix-shell -i bash -p cabal-install "haskell.packages.ghc92.ghcWithPackages (pkgs: with pkgs; [aeson])" git jq
-
-cd "$(dirname "${BASH_SOURCE[0]}")"
-cabal run gen_ci -- metadata jobs-metadata.json


=====================================
.gitlab/generate_jobs deleted
=====================================
@@ -1,13 +0,0 @@
-#!/usr/bin/env nix-shell
-#!nix-shell -i bash -p cabal-install "haskell.packages.ghc92.ghcWithPackages (pkgs: with pkgs; [aeson])" git jq
-
-# shellcheck shell=bash
-
-set -euo pipefail
-
-cd "$(dirname "${BASH_SOURCE[0]}")"
-tmp=$(mktemp)
-cabal run gen_ci -- gitlab $tmp
-rm -f jobs.yaml
-echo "### THIS IS A GENERATED FILE, DO NOT MODIFY DIRECTLY" > jobs.yaml
-cat $tmp | jq | tee -a jobs.yaml



View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/commit/8f6b771621922cab934ca7d84dd15b3ac4715e19

-- 
View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/commit/8f6b771621922cab934ca7d84dd15b3ac4715e19
You're receiving this email because of your account on gitlab.haskell.org.


-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mail.haskell.org/pipermail/ghc-commits/attachments/20230629/ad22955d/attachment-0001.html>


More information about the ghc-commits mailing list