[Git][ghc/ghc][wip/T22686] gitlab: Collect metadata about binary distributions
Ben Gamari (@bgamari)
gitlab at gitlab.haskell.org
Tue Feb 14 20:05:34 UTC 2023
Ben Gamari pushed to branch wip/T22686 at Glasgow Haskell Compiler / GHC
Commits:
f6426b6f by Ben Gamari at 2023-02-14T15:05:25-05:00
gitlab: Collect metadata about binary distributions
Fixes #22686.
- - - - -
4 changed files:
- + .gitlab/bindist_metadata.py
- .gitlab/ci.sh
- .gitlab/gen_ci.hs
- .gitlab/jobs.yaml
Changes:
=====================================
.gitlab/bindist_metadata.py
=====================================
@@ -0,0 +1,155 @@
+#!/usr/bin/env python3
+
+import sys
+import os
+import shutil
+import re
+import ast
+from pathlib import Path
+import subprocess
+import json
+from typing import Dict, List, Set, Optional, NamedTuple
+
+def run(args: List[str]) -> str:
+ return subprocess.check_output(args, encoding='UTF-8')
+
+def parse_hadrian_cfg(cfg: str) -> Dict[str,str]:
+ res = {}
+ for l in cfg.split('\n'):
+ if l.startswith('#'):
+ continue
+ elif '=' in l:
+ i = l.find('=')
+ k = l[:i].strip()
+ v = l[i+1:].strip()
+ res[k] = v
+
+ return res
+
+def get_ghc_info(ghc: Path) -> Dict[str,str]:
+ import ast
+ out = run([ghc, '--info'])
+ pairs = ast.literal_eval(out.strip())
+ res = {}
+ for k,v in pairs:
+ if v == 'YES':
+ v = True
+ elif v == 'NO':
+ v = False
+ res[k] = v
+
+ return res
+
+def get_dynamic_deps(objfile: Path) -> Set[Path]:
+ out = run(['ldd', objfile])
+ return { Path(m.group(1)) for m in re.finditer('=> *([^ ]+)', out) }
+
+def get_configure_cmdline() -> str:
+ r = Path('config.log').read_text()
+ m = re.search(r' $ .+', r)
+ return m
+
+class Package(NamedTuple):
+ name: str
+ version: str
+
+def find_providing_package(f: Path) -> Optional[Package]:
+ if shutil.which('dpkg'):
+ out = run(['dpkg-query', '--search', f]).strip()
+ pkg,_file = out.split(':')
+
+ out = run(['dpkg-query', '--show', pkg]).strip()
+ _pkg,version = out.split()
+ return Package(pkg, version)
+ elif shutil.which('rpm'):
+ out = run(['rpm', '-qf', f, '--queryformat=%{NAME} %{VERSION}\n']).strip()
+ pkg,version = out.split()
+ return Package(pkg, version)
+ elif shutil.which('apk'):
+ out = run(['apk', 'info', '--who-owns', f]).strip()
+ pkg = re.find('is owned by ([.+])', out)
+
+ # Determining the version of an installed package is far too difficult;
+ # some day perhaps upstream will address
+ # https://gitlab.alpinelinux.org/alpine/apk-tools/-/issues/10704
+ db = Path('/lib/apk/db/installed').read_text()
+ m = re.find(f'P:{pkg}\nV:([.+])\n', db)
+ version = m.group(1)
+ return Package(pkg, version)
+ else:
+ return None
+
+def main() -> None:
+ ghc = Path('_build/stage1/bin/ghc')
+ ghc_pkg = Path('_build/stage1/bin/ghc-pkg')
+
+ metadata = {}
+
+ system_config = Path('.') / 'hadrian' / 'cfg' / 'system.config'
+ cfg = parse_hadrian_cfg(system_config.read_text())
+
+ ######
+ # GHC build configuration
+ ######
+ metadata['ghc_version'] = cfg['project-version']
+ metadata['git_commit_id'] = cfg['project-git-commit-id']
+ metadata['tables_next_to_code'] = cfg['tables-next-to-code']
+ metadata['unregisterised'] = cfg['ghc-unregisterised']
+ metadata['build_triple'] = cfg['build-platform']
+ metadata['host_triple'] = cfg['host-platform']
+ metadata['target_triple'] = cfg['target-platform']
+ metadata['build_flavour'] = os.environ.get('BUILD_FLAVOUR')
+ metadata['configure_cmdline'] = get_configure_cmdline()
+
+ opsys = cfg['build-os']
+
+ ######
+ # Information about the bootstrapping environment
+ ######
+ lsb_release = None
+ if opsys == 'linux':
+ lsb_release = run(['lsb_release'])
+
+ metadata['bootstrap_environment'] = {
+ 'ghc': run([cfg['system-ghc'], '--version']).split('\n')[0],
+ 'cc': run([cfg['system-cc'], '--version']).split('\n')[0],
+ 'lsb_release': lsb_release,
+ }
+
+ ######
+ # Information about the bootstrapping environment's packages
+ ######
+ metadata['dynamic_deps'] = None
+ if opsys != 'darwin':
+ dyn_deps = get_dynamic_deps(ghc)
+ print(dyn_deps)
+ deps = {
+ dep.name: find_providing_package(dep)
+ for dep in dyn_deps
+ if not dep.is_relative_to(Path('.').resolve())
+ }
+ metadata['dynamic_deps'] = deps
+
+ ######
+ # The contents of the compiler's global package database
+ ######
+ def call_ghc_pkg(args: List[str]) -> str:
+ return run([ghc_pkg, '--simple-output'] + args).strip()
+
+ metadata['global_packages'] = {
+ pkg: {
+ 'version': call_ghc_pkg(['field', pkg, 'version']),
+ 'extra-libraries': call_ghc_pkg(['field', pkg, 'extra-libraries']).split(),
+ }
+ for pkg in call_ghc_pkg(['list', '--names-only']).split()
+ }
+
+ ######
+ # Information about the resulting compiler
+ ######
+ metadata['inplace_ghc_info'] = get_ghc_info(ghc)
+
+ json.dump(metadata, sys.stdout, indent=2)
+
+if __name__ == '__main__':
+ main()
=====================================
.gitlab/ci.sh
=====================================
@@ -478,6 +478,7 @@ function check_msys2_deps() {
# Ensure that GHC on Windows doesn't have any dynamic dependencies on msys2
case "$(uname)" in
MSYS_*|MINGW*)
+ info "Checking for unwanted msys2 dependencies..."
sysroot="$(cygpath "$SYSTEMROOT")"
PATH="$sysroot/System32:$sysroot;$sysroot/Wbem" $@ \
|| fail "'$@' failed; there may be unwanted dynamic dependencies."
@@ -584,6 +585,9 @@ function test_hadrian() {
check_msys2_deps _build/stage1/bin/ghc --version
check_release_build
+ info "Collecting binary distribution metadata..."
+ run "$TOP/.gitlab/bindist_metadata.py" > metadata.json
+
# Ensure that statically-linked builds are actually static
if [[ "${BUILD_FLAVOUR}" = *static* ]]; then
bad_execs=""
=====================================
.gitlab/gen_ci.hs
=====================================
@@ -680,6 +680,7 @@ job arch opsys buildConfig = NamedJob { name = jobName, jobInfo = Job {..} }
{ junitReport = "junit.xml"
, expireIn = "2 weeks"
, artifactPaths = [binDistName arch opsys buildConfig ++ ".tar.xz"
+ ,"metadata.json"
,"junit.xml"]
, artifactsWhen = ArtifactsAlways
}
=====================================
.gitlab/jobs.yaml
=====================================
@@ -11,6 +11,7 @@
"expire_in": "2 weeks",
"paths": [
"ghc-aarch64-darwin-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -73,6 +74,7 @@
"expire_in": "2 weeks",
"paths": [
"ghc-aarch64-linux-deb10-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -131,6 +133,7 @@
"expire_in": "2 weeks",
"paths": [
"ghc-i386-linux-deb9-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -189,6 +192,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-aarch64-darwin-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -252,6 +256,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-aarch64-linux-deb10-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -311,6 +316,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-aarch64-linux-deb10-validate+llvm.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -370,6 +376,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-i386-linux-deb9-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -429,6 +436,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-darwin-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -494,6 +502,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-freebsd13-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -555,6 +564,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-alpine3_12-cross_wasm32-wasi-release+fully_static.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -616,6 +626,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-alpine3_12-int_native-cross_wasm32-wasi-release+fully_static.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -677,6 +688,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-alpine3_12-int_native-validate+fully_static.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -740,6 +752,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-alpine3_12-unreg-cross_wasm32-wasi-release+fully_static.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -801,6 +814,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-alpine3_12-validate+fully_static.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -864,6 +878,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-centos7-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -924,6 +939,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-deb10-int_native-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -983,6 +999,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-deb10-no_tntc-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -1042,6 +1059,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-deb10-numa-slow-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -1102,6 +1120,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-deb10-unreg-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -1161,6 +1180,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-deb10-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -1220,6 +1240,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-deb10-validate+debug_info.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -1279,6 +1300,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-deb10-validate+llvm.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -1338,6 +1360,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-deb10-validate+thread_sanitizer.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -1399,6 +1422,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-deb11-cross_aarch64-linux-gnu-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -1460,6 +1484,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-deb11-int_native-cross_javascript-unknown-ghcjs-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -1522,6 +1547,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-deb11-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -1581,6 +1607,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-deb9-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -1640,6 +1667,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-fedora33-release.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -1701,6 +1729,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-fedora33-release.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -1763,6 +1792,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-fedora33-validate+debug_info.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -1824,6 +1854,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-rocky8-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -1884,6 +1915,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-ubuntu18_04-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -1943,6 +1975,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-linux-ubuntu20_04-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -2001,6 +2034,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-windows-int_native-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -2060,6 +2094,7 @@
"expire_in": "8 weeks",
"paths": [
"ghc-x86_64-windows-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -2120,6 +2155,7 @@
"expire_in": "1 year",
"paths": [
"ghc-aarch64-darwin-release.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -2184,6 +2220,7 @@
"expire_in": "1 year",
"paths": [
"ghc-aarch64-linux-deb10-release+no_split_sections.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -2244,6 +2281,7 @@
"expire_in": "1 year",
"paths": [
"ghc-i386-linux-deb9-release+no_split_sections.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -2304,6 +2342,7 @@
"expire_in": "1 year",
"paths": [
"ghc-x86_64-darwin-release.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -2370,6 +2409,7 @@
"expire_in": "1 year",
"paths": [
"ghc-x86_64-linux-alpine3_12-int_native-release+fully_static.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -2434,6 +2474,7 @@
"expire_in": "1 year",
"paths": [
"ghc-x86_64-linux-alpine3_12-release+fully_static+no_split_sections.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -2498,6 +2539,7 @@
"expire_in": "1 year",
"paths": [
"ghc-x86_64-linux-centos7-release+no_split_sections.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -2559,6 +2601,7 @@
"expire_in": "1 year",
"paths": [
"ghc-x86_64-linux-deb10-release.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -2619,6 +2662,7 @@
"expire_in": "1 year",
"paths": [
"ghc-x86_64-linux-deb10-release+debug_info.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -2679,6 +2723,7 @@
"expire_in": "1 year",
"paths": [
"ghc-x86_64-linux-deb11-release.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -2739,6 +2784,7 @@
"expire_in": "1 year",
"paths": [
"ghc-x86_64-linux-deb9-release+no_split_sections.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -2799,6 +2845,7 @@
"expire_in": "1 year",
"paths": [
"ghc-x86_64-linux-fedora33-release.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -2861,6 +2908,7 @@
"expire_in": "1 year",
"paths": [
"ghc-x86_64-linux-fedora33-release+debug_info.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -2923,6 +2971,7 @@
"expire_in": "1 year",
"paths": [
"ghc-x86_64-linux-fedora33-release.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -2986,6 +3035,7 @@
"expire_in": "1 year",
"paths": [
"ghc-x86_64-linux-rocky8-release.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -3047,6 +3097,7 @@
"expire_in": "1 year",
"paths": [
"ghc-x86_64-linux-ubuntu18_04-release.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -3107,6 +3158,7 @@
"expire_in": "1 year",
"paths": [
"ghc-x86_64-linux-ubuntu20_04-release.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -3166,6 +3218,7 @@
"expire_in": "1 year",
"paths": [
"ghc-x86_64-windows-int_native-release+no_split_sections.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -3226,6 +3279,7 @@
"expire_in": "1 year",
"paths": [
"ghc-x86_64-windows-release+no_split_sections.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -3287,6 +3341,7 @@
"expire_in": "2 weeks",
"paths": [
"ghc-x86_64-darwin-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -3351,6 +3406,7 @@
"expire_in": "2 weeks",
"paths": [
"ghc-x86_64-freebsd13-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -3411,6 +3467,7 @@
"expire_in": "2 weeks",
"paths": [
"ghc-x86_64-linux-alpine3_12-cross_wasm32-wasi-release+fully_static.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -3471,6 +3528,7 @@
"expire_in": "2 weeks",
"paths": [
"ghc-x86_64-linux-alpine3_12-validate+fully_static.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -3533,6 +3591,7 @@
"expire_in": "2 weeks",
"paths": [
"ghc-x86_64-linux-deb10-int_native-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -3591,6 +3650,7 @@
"expire_in": "2 weeks",
"paths": [
"ghc-x86_64-linux-deb10-no_tntc-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -3650,6 +3710,7 @@
"expire_in": "2 weeks",
"paths": [
"ghc-x86_64-linux-deb10-numa-slow-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -3709,6 +3770,7 @@
"expire_in": "2 weeks",
"paths": [
"ghc-x86_64-linux-deb10-unreg-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -3767,6 +3829,7 @@
"expire_in": "2 weeks",
"paths": [
"ghc-x86_64-linux-deb10-validate+debug_info.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -3825,6 +3888,7 @@
"expire_in": "2 weeks",
"paths": [
"ghc-x86_64-linux-deb10-validate+llvm.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -3883,6 +3947,7 @@
"expire_in": "2 weeks",
"paths": [
"ghc-x86_64-linux-deb10-validate+thread_sanitizer.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -3944,6 +4009,7 @@
"expire_in": "2 weeks",
"paths": [
"ghc-x86_64-linux-deb11-cross_aarch64-linux-gnu-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -4004,6 +4070,7 @@
"expire_in": "2 weeks",
"paths": [
"ghc-x86_64-linux-deb11-int_native-cross_javascript-unknown-ghcjs-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -4065,6 +4132,7 @@
"expire_in": "2 weeks",
"paths": [
"ghc-x86_64-linux-fedora33-release.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
@@ -4124,6 +4192,7 @@
"expire_in": "2 weeks",
"paths": [
"ghc-x86_64-windows-validate.tar.xz",
+ "metadata.json",
"junit.xml"
],
"reports": {
View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/commit/f6426b6f33926fada1a82a50b89d736f390f7666
--
View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/commit/f6426b6f33926fada1a82a50b89d736f390f7666
You're receiving this email because of your account on gitlab.haskell.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mail.haskell.org/pipermail/ghc-commits/attachments/20230214/19c457df/attachment-0001.html>
More information about the ghc-commits
mailing list