[Git][ghc/ghc][wip/text-simdutf] 2 commits: hadrian: add +text_simdutf flavour transformer to allow building text with simdutf
Cheng Shao (@TerrorJack)
gitlab at gitlab.haskell.org
Sun Jun 2 15:14:11 UTC 2024
Cheng Shao pushed to branch wip/text-simdutf at Glasgow Haskell Compiler / GHC
Commits:
b492664c by Cheng Shao at 2024-06-02T15:13:55+00:00
hadrian: add +text_simdutf flavour transformer to allow building text with simdutf
This patch adds a +text_simdutf flavour transformer to hadrian to
allow downstream packagers and users that build from source to opt-in
simdutf support for text, in order to benefit from SIMD speedup at
run-time. It's still disabled by default for the time being.
- - - - -
45085756 by Cheng Shao at 2024-06-02T15:13:59+00:00
ci: enable +text_simdutf flavour transformer for wasm jobs
This commit enables +text_simdutf flavour transformer for wasm jobs,
so text is now built with simdutf support for wasm.
- - - - -
8 changed files:
- .gitlab/generate-ci/gen_ci.hs
- .gitlab/jobs.yaml
- hadrian/doc/flavours.md
- hadrian/src/Flavour.hs
- hadrian/src/Flavour/Type.hs
- hadrian/src/Rules/Register.hs
- hadrian/src/Settings/Default.hs
- hadrian/src/Settings/Packages.hs
Changes:
=====================================
.gitlab/generate-ci/gen_ci.hs
=====================================
@@ -153,6 +153,7 @@ data BuildConfig
, threadSanitiser :: Bool
, noSplitSections :: Bool
, validateNonmovingGc :: Bool
+ , textWithSIMDUTF :: Bool
}
-- Extra arguments to pass to ./configure due to the BuildConfig
@@ -174,7 +175,8 @@ mkJobFlavour BuildConfig{..} = Flavour buildFlavour opts
[FullyStatic | fullyStatic] ++
[ThreadSanitiser | threadSanitiser] ++
[NoSplitSections | noSplitSections, buildFlavour == Release ] ++
- [BootNonmovingGc | validateNonmovingGc ]
+ [BootNonmovingGc | validateNonmovingGc ] ++
+ [TextWithSIMDUTF | textWithSIMDUTF]
data Flavour = Flavour BaseFlavour [FlavourTrans]
@@ -185,6 +187,7 @@ data FlavourTrans =
| ThreadSanitiser
| NoSplitSections
| BootNonmovingGc
+ | TextWithSIMDUTF
data BaseFlavour = Release | Validate | SlowValidate deriving Eq
@@ -211,6 +214,7 @@ vanilla = BuildConfig
, threadSanitiser = False
, noSplitSections = False
, validateNonmovingGc = False
+ , textWithSIMDUTF = False
}
splitSectionsBroken :: BuildConfig -> BuildConfig
@@ -344,6 +348,7 @@ flavourString (Flavour base trans) = base_string base ++ concatMap (("+" ++) . f
flavour_string ThreadSanitiser = "thread_sanitizer_cmm"
flavour_string NoSplitSections = "no_split_sections"
flavour_string BootNonmovingGc = "boot_nonmoving_gc"
+ flavour_string TextWithSIMDUTF = "text_simdutf"
-- The path to the docker image (just for linux builders)
dockerImage :: Arch -> Opsys -> Maybe String
@@ -1057,6 +1062,7 @@ job_groups =
{
fullyStatic = True
, buildFlavour = Release -- TODO: This needs to be validate but wasm backend doesn't pass yet
+ , textWithSIMDUTF = True
}
@@ -1082,10 +1088,10 @@ platform_mapping = Map.map go combined_result
, "x86_64-linux-fedora33-release"
, "x86_64-linux-deb11-cross_aarch64-linux-gnu-validate"
, "x86_64-windows-validate"
- , "nightly-x86_64-linux-alpine3_18-wasm-cross_wasm32-wasi-release+fully_static"
+ , "nightly-x86_64-linux-alpine3_18-wasm-cross_wasm32-wasi-release+fully_static+text_simdutf"
, "nightly-x86_64-linux-deb11-validate"
, "nightly-x86_64-linux-deb12-validate"
- , "x86_64-linux-alpine3_18-wasm-cross_wasm32-wasi-release+fully_static"
+ , "x86_64-linux-alpine3_18-wasm-cross_wasm32-wasi-release+fully_static+text_simdutf"
, "x86_64-linux-deb12-validate+thread_sanitizer_cmm"
, "nightly-aarch64-linux-deb10-validate"
, "nightly-x86_64-linux-alpine3_12-validate"
=====================================
.gitlab/jobs.yaml
=====================================
@@ -951,7 +951,7 @@
"XZ_OPT": "-9"
}
},
- "nightly-x86_64-linux-alpine3_18-wasm-cross_wasm32-wasi-release+fully_static": {
+ "nightly-x86_64-linux-alpine3_18-wasm-cross_wasm32-wasi-release+fully_static+text_simdutf": {
"after_script": [
".gitlab/ci.sh save_cache",
".gitlab/ci.sh save_test_output",
@@ -962,7 +962,7 @@
"artifacts": {
"expire_in": "8 weeks",
"paths": [
- "ghc-x86_64-linux-alpine3_18-wasm-cross_wasm32-wasi-release+fully_static.tar.xz",
+ "ghc-x86_64-linux-alpine3_18-wasm-cross_wasm32-wasi-release+fully_static+text_simdutf.tar.xz",
"junit.xml",
"unexpected-test-output.tar.gz"
],
@@ -1005,17 +1005,17 @@
],
"variables": {
"BIGNUM_BACKEND": "gmp",
- "BIN_DIST_NAME": "ghc-x86_64-linux-alpine3_18-wasm-cross_wasm32-wasi-release+fully_static",
- "BUILD_FLAVOUR": "release+fully_static",
+ "BIN_DIST_NAME": "ghc-x86_64-linux-alpine3_18-wasm-cross_wasm32-wasi-release+fully_static+text_simdutf",
+ "BUILD_FLAVOUR": "release+fully_static+text_simdutf",
"CONFIGURE_ARGS": "--with-intree-gmp --with-system-libffi --enable-strict-ghc-toolchain-check",
"CROSS_TARGET": "wasm32-wasi",
"HADRIAN_ARGS": "--docs=none",
"RUNTEST_ARGS": "",
- "TEST_ENV": "x86_64-linux-alpine3_18-wasm-cross_wasm32-wasi-release+fully_static",
+ "TEST_ENV": "x86_64-linux-alpine3_18-wasm-cross_wasm32-wasi-release+fully_static+text_simdutf",
"XZ_OPT": "-9"
}
},
- "nightly-x86_64-linux-alpine3_18-wasm-int_native-cross_wasm32-wasi-release+fully_static": {
+ "nightly-x86_64-linux-alpine3_18-wasm-int_native-cross_wasm32-wasi-release+fully_static+text_simdutf": {
"after_script": [
".gitlab/ci.sh save_cache",
".gitlab/ci.sh save_test_output",
@@ -1026,7 +1026,7 @@
"artifacts": {
"expire_in": "8 weeks",
"paths": [
- "ghc-x86_64-linux-alpine3_18-wasm-int_native-cross_wasm32-wasi-release+fully_static.tar.xz",
+ "ghc-x86_64-linux-alpine3_18-wasm-int_native-cross_wasm32-wasi-release+fully_static+text_simdutf.tar.xz",
"junit.xml",
"unexpected-test-output.tar.gz"
],
@@ -1069,17 +1069,17 @@
],
"variables": {
"BIGNUM_BACKEND": "native",
- "BIN_DIST_NAME": "ghc-x86_64-linux-alpine3_18-wasm-int_native-cross_wasm32-wasi-release+fully_static",
- "BUILD_FLAVOUR": "release+fully_static",
+ "BIN_DIST_NAME": "ghc-x86_64-linux-alpine3_18-wasm-int_native-cross_wasm32-wasi-release+fully_static+text_simdutf",
+ "BUILD_FLAVOUR": "release+fully_static+text_simdutf",
"CONFIGURE_ARGS": "--with-intree-gmp --with-system-libffi --enable-strict-ghc-toolchain-check",
"CROSS_TARGET": "wasm32-wasi",
"HADRIAN_ARGS": "--docs=none",
"RUNTEST_ARGS": "",
- "TEST_ENV": "x86_64-linux-alpine3_18-wasm-int_native-cross_wasm32-wasi-release+fully_static",
+ "TEST_ENV": "x86_64-linux-alpine3_18-wasm-int_native-cross_wasm32-wasi-release+fully_static+text_simdutf",
"XZ_OPT": "-9"
}
},
- "nightly-x86_64-linux-alpine3_18-wasm-unreg-cross_wasm32-wasi-release+fully_static": {
+ "nightly-x86_64-linux-alpine3_18-wasm-unreg-cross_wasm32-wasi-release+fully_static+text_simdutf": {
"after_script": [
".gitlab/ci.sh save_cache",
".gitlab/ci.sh save_test_output",
@@ -1090,7 +1090,7 @@
"artifacts": {
"expire_in": "8 weeks",
"paths": [
- "ghc-x86_64-linux-alpine3_18-wasm-unreg-cross_wasm32-wasi-release+fully_static.tar.xz",
+ "ghc-x86_64-linux-alpine3_18-wasm-unreg-cross_wasm32-wasi-release+fully_static+text_simdutf.tar.xz",
"junit.xml",
"unexpected-test-output.tar.gz"
],
@@ -1133,13 +1133,13 @@
],
"variables": {
"BIGNUM_BACKEND": "gmp",
- "BIN_DIST_NAME": "ghc-x86_64-linux-alpine3_18-wasm-unreg-cross_wasm32-wasi-release+fully_static",
- "BUILD_FLAVOUR": "release+fully_static",
+ "BIN_DIST_NAME": "ghc-x86_64-linux-alpine3_18-wasm-unreg-cross_wasm32-wasi-release+fully_static+text_simdutf",
+ "BUILD_FLAVOUR": "release+fully_static+text_simdutf",
"CONFIGURE_ARGS": "--enable-unregisterised --with-intree-gmp --with-system-libffi --enable-strict-ghc-toolchain-check",
"CROSS_TARGET": "wasm32-wasi",
"HADRIAN_ARGS": "--docs=none",
"RUNTEST_ARGS": "",
- "TEST_ENV": "x86_64-linux-alpine3_18-wasm-unreg-cross_wasm32-wasi-release+fully_static",
+ "TEST_ENV": "x86_64-linux-alpine3_18-wasm-unreg-cross_wasm32-wasi-release+fully_static+text_simdutf",
"XZ_OPT": "-9"
}
},
@@ -4465,7 +4465,7 @@
"TEST_ENV": "x86_64-linux-alpine3_12-validate+fully_static"
}
},
- "x86_64-linux-alpine3_18-wasm-cross_wasm32-wasi-release+fully_static": {
+ "x86_64-linux-alpine3_18-wasm-cross_wasm32-wasi-release+fully_static+text_simdutf": {
"after_script": [
".gitlab/ci.sh save_cache",
".gitlab/ci.sh save_test_output",
@@ -4476,7 +4476,7 @@
"artifacts": {
"expire_in": "2 weeks",
"paths": [
- "ghc-x86_64-linux-alpine3_18-wasm-cross_wasm32-wasi-release+fully_static.tar.xz",
+ "ghc-x86_64-linux-alpine3_18-wasm-cross_wasm32-wasi-release+fully_static+text_simdutf.tar.xz",
"junit.xml",
"unexpected-test-output.tar.gz"
],
@@ -4519,16 +4519,16 @@
],
"variables": {
"BIGNUM_BACKEND": "gmp",
- "BIN_DIST_NAME": "ghc-x86_64-linux-alpine3_18-wasm-cross_wasm32-wasi-release+fully_static",
- "BUILD_FLAVOUR": "release+fully_static",
+ "BIN_DIST_NAME": "ghc-x86_64-linux-alpine3_18-wasm-cross_wasm32-wasi-release+fully_static+text_simdutf",
+ "BUILD_FLAVOUR": "release+fully_static+text_simdutf",
"CONFIGURE_ARGS": "--with-intree-gmp --with-system-libffi --enable-strict-ghc-toolchain-check",
"CROSS_TARGET": "wasm32-wasi",
"HADRIAN_ARGS": "--docs=none",
"RUNTEST_ARGS": "",
- "TEST_ENV": "x86_64-linux-alpine3_18-wasm-cross_wasm32-wasi-release+fully_static"
+ "TEST_ENV": "x86_64-linux-alpine3_18-wasm-cross_wasm32-wasi-release+fully_static+text_simdutf"
}
},
- "x86_64-linux-alpine3_18-wasm-int_native-cross_wasm32-wasi-release+fully_static": {
+ "x86_64-linux-alpine3_18-wasm-int_native-cross_wasm32-wasi-release+fully_static+text_simdutf": {
"after_script": [
".gitlab/ci.sh save_cache",
".gitlab/ci.sh save_test_output",
@@ -4539,7 +4539,7 @@
"artifacts": {
"expire_in": "2 weeks",
"paths": [
- "ghc-x86_64-linux-alpine3_18-wasm-int_native-cross_wasm32-wasi-release+fully_static.tar.xz",
+ "ghc-x86_64-linux-alpine3_18-wasm-int_native-cross_wasm32-wasi-release+fully_static+text_simdutf.tar.xz",
"junit.xml",
"unexpected-test-output.tar.gz"
],
@@ -4583,16 +4583,16 @@
],
"variables": {
"BIGNUM_BACKEND": "native",
- "BIN_DIST_NAME": "ghc-x86_64-linux-alpine3_18-wasm-int_native-cross_wasm32-wasi-release+fully_static",
- "BUILD_FLAVOUR": "release+fully_static",
+ "BIN_DIST_NAME": "ghc-x86_64-linux-alpine3_18-wasm-int_native-cross_wasm32-wasi-release+fully_static+text_simdutf",
+ "BUILD_FLAVOUR": "release+fully_static+text_simdutf",
"CONFIGURE_ARGS": "--with-intree-gmp --with-system-libffi --enable-strict-ghc-toolchain-check",
"CROSS_TARGET": "wasm32-wasi",
"HADRIAN_ARGS": "--docs=none",
"RUNTEST_ARGS": "",
- "TEST_ENV": "x86_64-linux-alpine3_18-wasm-int_native-cross_wasm32-wasi-release+fully_static"
+ "TEST_ENV": "x86_64-linux-alpine3_18-wasm-int_native-cross_wasm32-wasi-release+fully_static+text_simdutf"
}
},
- "x86_64-linux-alpine3_18-wasm-unreg-cross_wasm32-wasi-release+fully_static": {
+ "x86_64-linux-alpine3_18-wasm-unreg-cross_wasm32-wasi-release+fully_static+text_simdutf": {
"after_script": [
".gitlab/ci.sh save_cache",
".gitlab/ci.sh save_test_output",
@@ -4603,7 +4603,7 @@
"artifacts": {
"expire_in": "2 weeks",
"paths": [
- "ghc-x86_64-linux-alpine3_18-wasm-unreg-cross_wasm32-wasi-release+fully_static.tar.xz",
+ "ghc-x86_64-linux-alpine3_18-wasm-unreg-cross_wasm32-wasi-release+fully_static+text_simdutf.tar.xz",
"junit.xml",
"unexpected-test-output.tar.gz"
],
@@ -4647,13 +4647,13 @@
],
"variables": {
"BIGNUM_BACKEND": "gmp",
- "BIN_DIST_NAME": "ghc-x86_64-linux-alpine3_18-wasm-unreg-cross_wasm32-wasi-release+fully_static",
- "BUILD_FLAVOUR": "release+fully_static",
+ "BIN_DIST_NAME": "ghc-x86_64-linux-alpine3_18-wasm-unreg-cross_wasm32-wasi-release+fully_static+text_simdutf",
+ "BUILD_FLAVOUR": "release+fully_static+text_simdutf",
"CONFIGURE_ARGS": "--enable-unregisterised --with-intree-gmp --with-system-libffi --enable-strict-ghc-toolchain-check",
"CROSS_TARGET": "wasm32-wasi",
"HADRIAN_ARGS": "--docs=none",
"RUNTEST_ARGS": "",
- "TEST_ENV": "x86_64-linux-alpine3_18-wasm-unreg-cross_wasm32-wasi-release+fully_static"
+ "TEST_ENV": "x86_64-linux-alpine3_18-wasm-unreg-cross_wasm32-wasi-release+fully_static+text_simdutf"
}
},
"x86_64-linux-deb10-int_native-validate": {
=====================================
hadrian/doc/flavours.md
=====================================
@@ -265,6 +265,10 @@ The supported transformers are listed below:
<td><code>native_bignum</code></td>
<td>Use the native <code>ghc-bignum</code> backend.</td>
</tr>
+ <tr>
+ <td><code>text_simdutf</code></td>
+ <td>Enable building the <code>text</code> package with <code>simdutf</code> support.</td>
+ </tr>
<tr>
<td><code>no_profiled_libs</code></td>
<td>Disables building of libraries in profiled build ways.</td>
=====================================
hadrian/src/Flavour.hs
=====================================
@@ -17,6 +17,7 @@ module Flavour
, enableHaddock
, enableHiCore
, useNativeBignum
+ , enableTextWithSIMDUTF
, omitPragmas
, completeSetting
@@ -53,6 +54,7 @@ flavourTransformers = M.fromList
, "no_dynamic_ghc" =: disableDynamicGhcPrograms
, "no_dynamic_libs" =: disableDynamicLibs
, "native_bignum" =: useNativeBignum
+ , "text_simdutf" =: enableTextWithSIMDUTF
, "no_profiled_libs" =: disableProfiledLibs
, "omit_pragmas" =: omitPragmas
, "ipe" =: enableIPE
@@ -292,6 +294,12 @@ useNativeBignum flavour =
flavour { bignumBackend = "native"
}
+-- | Enable building the @text@ package with @simdutf@ support.
+enableTextWithSIMDUTF :: Flavour -> Flavour
+enableTextWithSIMDUTF flavour = flavour {
+ textWithSIMDUTF = True
+}
+
-- | Build stage2 compiler with -fomit-interface-pragmas to reduce
-- recompilation.
omitPragmas :: Flavour -> Flavour
=====================================
hadrian/src/Flavour/Type.hs
=====================================
@@ -23,6 +23,9 @@ data Flavour = Flavour {
bignumBackend :: String,
-- | Check selected bignum backend against native backend
bignumCheck :: Bool,
+ -- | Build the @text@ package with @simdutf@ support. Disabled by
+ -- default due to packaging difficulties described in #20724.
+ textWithSIMDUTF :: Bool,
-- | Build libraries these ways.
libraryWays :: Ways,
-- | Build RTS these ways.
@@ -70,4 +73,3 @@ type DocTargets = Set DocTarget
-- distribution.
data DocTarget = Haddocks | SphinxHTML | SphinxPDFs | SphinxMan | SphinxInfo
deriving (Eq, Ord, Show, Bounded, Enum)
-
=====================================
hadrian/src/Rules/Register.hs
=====================================
@@ -7,6 +7,7 @@ module Rules.Register (
import Base
import Context
import Expression ( getContextData )
+import Flavour
import Oracles.Setting
import Hadrian.BuildPath
import Hadrian.Expression
@@ -51,6 +52,14 @@ configurePackageRules = do
isGmp <- (== "gmp") <$> interpretInContext ctx getBignumBackend
when isGmp $
need [buildP -/- "include/ghc-gmp.h"]
+ when (pkg == text) $ do
+ simdutf <- textWithSIMDUTF <$> flavour
+ when simdutf $ do
+ -- This is required, otherwise you get Error: hadrian:
+ -- Encountered missing or private dependencies:
+ -- system-cxx-std-lib ==1.0
+ cxxStdLib <- systemCxxStdLibConfPath $ PackageDbLoc stage Inplace
+ need [cxxStdLib]
Cabal.configurePackage ctx
root -/- "**/autogen/cabal_macros.h" %> \out -> do
=====================================
hadrian/src/Settings/Default.hs
=====================================
@@ -267,6 +267,7 @@ defaultFlavour = Flavour
, packages = defaultPackages
, bignumBackend = defaultBignumBackend
, bignumCheck = False
+ , textWithSIMDUTF = False
, libraryWays = defaultLibraryWays
, rtsWays = defaultRtsWays
, dynamicGhcPrograms = defaultDynamicGhcPrograms
=====================================
hadrian/src/Settings/Packages.hs
=====================================
@@ -191,12 +191,10 @@ packageArgs = do
builder (Cabal Flags) ? stage0 `cabalFlag` "bootstrap"
---------------------------------- text --------------------------------
- , package text ? mconcat
- -- Disable SIMDUTF by default due to packaging difficulties
- -- described in #20724.
- [ builder (Cabal Flags) ? arg "-simdutf"
- -- https://github.com/haskell/text/issues/415
- , builder Ghc ? input "**/Data/Text/Encoding.hs" ? arg "-Wno-unused-imports" ]
+ , package text ?
+ ifM (textWithSIMDUTF <$> expr flavour)
+ (builder (Cabal Flags) ? arg "+simdutf")
+ (builder (Cabal Flags) ? arg "-simdutf")
------------------------------- haskeline ------------------------------
-- Hadrian doesn't currently support packages containing both libraries
View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/compare/8442f707bd28d8de693b0bdd94c60a8c925e86c9...45085756712ec43415874d94aea082dca30fd44f
--
View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/compare/8442f707bd28d8de693b0bdd94c60a8c925e86c9...45085756712ec43415874d94aea082dca30fd44f
You're receiving this email because of your account on gitlab.haskell.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mail.haskell.org/pipermail/ghc-commits/attachments/20240602/02fb3a58/attachment-0001.html>
More information about the ghc-commits
mailing list