[Git][ghc/ghc][wip/T22686] gitlab: Collect metadata about binary distributions
Matthew Pickering (@mpickering)
gitlab at gitlab.haskell.org
Mon Jan 30 10:18:40 UTC 2023
Matthew Pickering pushed to branch wip/T22686 at Glasgow Haskell Compiler / GHC
Commits:
1163681a by Ben Gamari at 2023-01-30T10:08:23+00:00
gitlab: Collect metadata about binary distributions
Fixes #22686.
- - - - -
4 changed files:
- + .gitlab/bindist_metadata.py
- .gitlab/ci.sh
- .gitlab/gen_ci.hs
- .gitlab/jobs.yaml
Changes:
=====================================
.gitlab/bindist_metadata.py
=====================================
@@ -0,0 +1,141 @@
+#!/usr/bin/env python3
+
+import sys
+import os
+import shutil
+import re
+import ast
+from pathlib import Path
+import subprocess
+import json
+from typing import Dict, List, Set
+
+def run(args: List[str]) -> str:
+ return subprocess.check_output(args, encoding='UTF-8')
+
+def parse_hadrian_cfg(cfg: str) -> Dict[str,str]:
+ res = {}
+ for l in cfg.split('\n'):
+ if l.startswith('#'):
+ continue
+ elif '=' in l:
+ i = l.find('=')
+ k = l[:i].strip()
+ v = l[i+1:].strip()
+ res[k] = v
+
+ return res
+
+def get_ghc_info(ghc: Path) -> Dict[str,str]:
+ import ast
+ out = run([ghc, '--info'])
+ pairs = ast.literal_eval(out.strip())
+ res = {}
+ for k,v in pairs:
+ if v == 'YES':
+ v = True
+ elif v == 'NO':
+ v = False
+ res[k] = v
+
+ return res
+
+def get_configure_cmdline() -> str:
+ r = Path('config.log').read_text()
+ m = re.search(r' $ .+', r)
+ return m
+
+def get_dpkg_packages() -> Dict[str, str]:
+ pkgs = {}
+ for l in run(['dpkg-query', '--show']).split('\n'):
+ parts = l.split()
+ if len(parts) == 2:
+ k = parts[0].strip()
+ v = parts[1].strip()
+ pkgs[k] = v
+
+ return pkgs
+
+def get_rpm_packages() -> Dict[str, str]:
+ pkgs = {}
+ for l in run(['rpm', '-qa', '--queryformat=%{NAME} %{VERSION}\n']).split('\n'):
+ parts = l.split()
+ if len(parts) == 2:
+ k = parts[0].strip()
+ v = parts[1].strip()
+ pkgs[k] = v
+
+ return pkgs
+
+def main() -> None:
+ ghc = Path('_build/stage1/bin/ghc')
+ ghc_pkg = Path('_build/stage1/bin/ghc-pkg')
+
+ metadata = {}
+
+ system_config = Path('.') / 'hadrian' / 'cfg' / 'system.config'
+ cfg = parse_hadrian_cfg(system_config.read_text())
+
+ ######
+ # GHC build configuration
+ ######
+ metadata['ghc_version'] = cfg['project-version']
+ metadata['git_commit_id'] = cfg['project-git-commit-id']
+ metadata['tables_next_to_code'] = cfg['tables-next-to-code']
+ metadata['unregisterised'] = cfg['ghc-unregisterised']
+ metadata['build_triple'] = cfg['build-platform']
+ metadata['host_triple'] = cfg['host-platform']
+ metadata['target_triple'] = cfg['target-platform']
+ metadata['build_flavour'] = os.environ.get('BUILD_FLAVOUR')
+ metadata['configure_cmdline'] = get_configure_cmdline()
+
+ ######
+ # Information about the bootstrapping environment
+ ######
+ try:
+ lsb_release = run(['lsb_release'])
+ except:
+ lsb_release = 'unknown'
+
+ metadata['bootstrap_environment'] = {
+ 'ghc': run([cfg['system-ghc'], '--version']).split('\n')[0],
+ 'cc': run([cfg['system-cc'], '--version']).split('\n')[0],
+ 'lsb_release': lsb_release,
+ }
+
+ ######
+ # Information about the bootstrapping environment's packages
+ ######
+ if shutil.which('dpkg'):
+ metadata['bootstrap_environment']['package_system'] = 'dpkg'
+ metadata['bootstrap_environment']['installed_packages'] = get_dpkg_packages()
+ elif shutil.which('rpm'):
+ metadata['bootstrap_environment']['package_system'] = 'rpm'
+ metadata['bootstrap_environment']['installed_packages'] = get_rpm_packages()
+ else:
+ metadata['bootstrap_environment']['package_system'] = 'unknown'
+ metadata['bootstrap_environment']['installed_packages'] = None
+
+ ######
+ # The contents of the compiler's global package database
+ ######
+ def call_ghc_pkg(args: List[str]) -> str:
+ return run([ghc_pkg, '--simple-output'] + args).strip()
+
+ metadata['global_packages'] = {
+ pkg: {
+ 'version': call_ghc_pkg(['field', pkg, 'version']),
+ 'extra-libraries': call_ghc_pkg(['field', pkg, 'extra-libraries']).split(),
+ }
+ for pkg in call_ghc_pkg(['list', '--names-only']).split()
+ }
+
+ ######
+ # Information about the resulting compiler
+ ######
+ metadata['inplace_ghc_info'] = get_ghc_info(ghc)
+
+ json.dump(metadata, sys.stdout, indent=2)
+
+if __name__ == '__main__':
+ main()
=====================================
.gitlab/ci.sh
=====================================
@@ -580,6 +580,7 @@ function install_bindist() {
function test_hadrian() {
check_msys2_deps _build/stage1/bin/ghc --version
check_release_build
+ $TOP/.gitlab/bindist_metadata.py > metadata.json
# Ensure that statically-linked builds are actually static
if [[ "${BUILD_FLAVOUR}" = *static* ]]; then
=====================================
.gitlab/gen_ci.hs
=====================================
@@ -681,6 +681,7 @@ job arch opsys buildConfig = NamedJob { name = jobName, jobInfo = Job {..} }
{ junitReport = "junit.xml"
, expireIn = "2 weeks"
, artifactPaths = [binDistName arch opsys buildConfig ++ ".tar.xz"
+ ,"metadata.json"
,"junit.xml"]
, artifactsWhen = ArtifactsAlways
}
=====================================
.gitlab/jobs.yaml
=====================================
@@ -11,6 +11,7 @@
"expire_in": "2 weeks",
"paths": [
"ghc-aarch64-darwin-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -73,6 +74,7 @@
"expire_in": "2 weeks",
"paths": [
"ghc-aarch64-linux-deb10-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -131,6 +133,7 @@
"expire_in": "2 weeks",
"paths": [
"ghc-i386-linux-deb9-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -189,6 +192,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-aarch64-darwin-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -252,6 +256,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-aarch64-linux-deb10-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -311,6 +316,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-aarch64-linux-deb10-validate+llvm.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -370,6 +376,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-i386-linux-deb9-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -429,6 +436,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-darwin-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -494,6 +502,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-freebsd13-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -555,6 +564,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-alpine3_12-cross_wasm32-wasi-release+fully_static.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -616,6 +626,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-alpine3_12-int_native-cross_wasm32-wasi-release+fully_static.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -677,6 +688,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-alpine3_12-int_native-validate+fully_static.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -740,6 +752,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-alpine3_12-unreg-cross_wasm32-wasi-release+fully_static.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -801,6 +814,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-alpine3_12-validate+fully_static.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -864,6 +878,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-centos7-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -924,6 +939,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-deb10-int_native-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -983,6 +999,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-deb10-no_tntc-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -1042,6 +1059,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-deb10-numa-slow-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -1102,6 +1120,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-deb10-unreg-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -1161,6 +1180,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-deb10-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -1220,6 +1240,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-deb10-validate+debug_info.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -1279,6 +1300,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-deb10-validate+llvm.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -1338,6 +1360,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-deb10-validate+thread_sanitizer.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -1399,6 +1422,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-deb11-cross_aarch64-linux-gnu-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -1460,6 +1484,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-deb11-int_native-cross_js-unknown-ghcjs-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -1521,6 +1546,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-deb11-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -1580,6 +1606,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-deb9-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -1639,6 +1666,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-fedora33-release.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -1700,6 +1728,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-fedora33-release.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -1762,6 +1791,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-fedora33-validate+debug_info.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -1823,6 +1853,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-rocky8-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -1883,6 +1914,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-ubuntu18_04-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -1942,6 +1974,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-ubuntu20_04-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -2000,6 +2033,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-windows-int_native-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -2059,6 +2093,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-windows-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -2119,6 +2154,7 @@
"expire_in": "1 year",
"paths": [
"ghc-aarch64-darwin-release.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -2183,6 +2219,7 @@
"expire_in": "1 year",
"paths": [
"ghc-aarch64-linux-deb10-release+no_split_sections.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -2243,6 +2280,7 @@
"expire_in": "1 year",
"paths": [
"ghc-i386-linux-deb9-release+no_split_sections.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -2303,6 +2341,7 @@
"expire_in": "1 year",
"paths": [
"ghc-x86_64-darwin-release.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -2369,6 +2408,7 @@
"expire_in": "1 year",
"paths": [
"ghc-x86_64-freebsd13-release.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -2431,6 +2471,7 @@
"expire_in": "1 year",
"paths": [
"ghc-x86_64-linux-alpine3_12-int_native-release+fully_static.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -2495,6 +2536,7 @@
"expire_in": "1 year",
"paths": [
"ghc-x86_64-linux-alpine3_12-release+fully_static+no_split_sections.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -2559,6 +2601,7 @@
"expire_in": "1 year",
"paths": [
"ghc-x86_64-linux-centos7-release+no_split_sections.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -2620,6 +2663,7 @@
"expire_in": "1 year",
"paths": [
"ghc-x86_64-linux-deb10-release.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -2680,6 +2724,7 @@
"expire_in": "1 year",
"paths": [
"ghc-x86_64-linux-deb10-release+debug_info.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -2740,6 +2785,7 @@
"expire_in": "1 year",
"paths": [
"ghc-x86_64-linux-deb11-release.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -2800,6 +2846,7 @@
"expire_in": "1 year",
"paths": [
"ghc-x86_64-linux-deb9-release+no_split_sections.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -2860,6 +2907,7 @@
"expire_in": "1 year",
"paths": [
"ghc-x86_64-linux-fedora33-release.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -2922,6 +2970,7 @@
"expire_in": "1 year",
"paths": [
"ghc-x86_64-linux-fedora33-release+debug_info.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -2984,6 +3033,7 @@
"expire_in": "1 year",
"paths": [
"ghc-x86_64-linux-fedora33-release.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -3047,6 +3097,7 @@
"expire_in": "1 year",
"paths": [
"ghc-x86_64-linux-rocky8-release.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -3108,6 +3159,7 @@
"expire_in": "1 year",
"paths": [
"ghc-x86_64-linux-ubuntu18_04-release.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -3168,6 +3220,7 @@
"expire_in": "1 year",
"paths": [
"ghc-x86_64-linux-ubuntu20_04-release.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -3227,6 +3280,7 @@
"expire_in": "1 year",
"paths": [
"ghc-x86_64-windows-int_native-release+no_split_sections.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -3287,6 +3341,7 @@
"expire_in": "1 year",
"paths": [
"ghc-x86_64-windows-release+no_split_sections.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -3348,6 +3403,7 @@
"expire_in": "2 weeks",
"paths": [
"ghc-x86_64-darwin-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -3412,6 +3468,7 @@
"expire_in": "2 weeks",
"paths": [
"ghc-x86_64-freebsd13-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -3472,6 +3529,7 @@
"expire_in": "2 weeks",
"paths": [
"ghc-x86_64-linux-alpine3_12-cross_wasm32-wasi-release+fully_static.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -3532,6 +3590,7 @@
"expire_in": "2 weeks",
"paths": [
"ghc-x86_64-linux-alpine3_12-validate+fully_static.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -3594,6 +3653,7 @@
"expire_in": "2 weeks",
"paths": [
"ghc-x86_64-linux-deb10-int_native-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -3652,6 +3712,7 @@
"expire_in": "2 weeks",
"paths": [
"ghc-x86_64-linux-deb10-no_tntc-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -3711,6 +3772,7 @@
"expire_in": "2 weeks",
"paths": [
"ghc-x86_64-linux-deb10-numa-slow-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -3770,6 +3832,7 @@
"expire_in": "2 weeks",
"paths": [
"ghc-x86_64-linux-deb10-unreg-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -3828,6 +3891,7 @@
"expire_in": "2 weeks",
"paths": [
"ghc-x86_64-linux-deb10-validate+debug_info.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -3886,6 +3950,7 @@
"expire_in": "2 weeks",
"paths": [
"ghc-x86_64-linux-deb10-validate+llvm.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -3944,6 +4009,7 @@
"expire_in": "2 weeks",
"paths": [
"ghc-x86_64-linux-deb10-validate+thread_sanitizer.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -4005,6 +4071,7 @@
"expire_in": "2 weeks",
"paths": [
"ghc-x86_64-linux-deb11-cross_aarch64-linux-gnu-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -4065,6 +4132,7 @@
"expire_in": "2 weeks",
"paths": [
"ghc-x86_64-linux-deb11-int_native-cross_js-unknown-ghcjs-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -4125,6 +4193,7 @@
"expire_in": "2 weeks",
"paths": [
"ghc-x86_64-linux-fedora33-release.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -4184,6 +4253,7 @@
"expire_in": "2 weeks",
"paths": [
"ghc-x86_64-windows-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/commit/1163681a7174cd317a3d8082f1820e5b2ecb6c82
--
View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/commit/1163681a7174cd317a3d8082f1820e5b2ecb6c82
You're receiving this email because of your account on gitlab.haskell.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mail.haskell.org/pipermail/ghc-commits/attachments/20230130/69fdad2a/attachment-0001.html>
More information about the ghc-commits
mailing list