[Git][ghc/ghc][wip/freebsd-ci] 7 commits: rts/linker: Resolve iconv_* on FreeBSD

Ben Gamari (@bgamari) gitlab at gitlab.haskell.org
Mon Aug 8 03:58:01 UTC 2022



Ben Gamari pushed to branch wip/freebsd-ci at Glasgow Haskell Compiler / GHC


Commits:
aeff4cb7 by Ben Gamari at 2022-08-07T23:57:54-04:00
rts/linker: Resolve iconv_* on FreeBSD

FreeBSD's libiconv includes an implementation of the
iconv_* functions in libc. Unfortunately these can
only be resolved using dlvsym, which is how the RTS linker
usually resolves such functions. To fix this we include an ad-hoc
special case for iconv_*.

Fixes #20354.

- - - - -
987a19e8 by Ben Gamari at 2022-08-07T23:57:54-04:00
system-cxx-std-lib: Add support for FreeBSD libcxxrt

- - - - -
3e97d848 by Ben Gamari at 2022-08-07T23:57:54-04:00
gitlab-ci: Bump to use freebsd13 runners

- - - - -
720a8bfb by Ben Gamari at 2022-08-07T23:57:54-04:00
XXX: Bump up freebsd job

- - - - -
5ec8e9b2 by Ben Gamari at 2022-08-07T23:57:54-04:00
gitlab-ci: Use cabal-install-3.6.2.0 on FreeBSD

- - - - -
aba49071 by Ben Gamari at 2022-08-07T23:57:54-04:00
gitlab-ci: XXX temporary GHC bindist on FreeBSD

- - - - -
9de048a2 by Ben Gamari at 2022-08-07T23:57:54-04:00
Update jobs.yaml

- - - - -


5 changed files:

- .gitlab/ci.sh
- .gitlab/gen_ci.hs
- .gitlab/jobs.yaml
- m4/fp_find_cxx_std_lib.m4
- rts/Linker.c


Changes:

=====================================
.gitlab/ci.sh
=====================================
@@ -279,6 +279,9 @@ function fetch_ghc() {
 
       start_section "fetch GHC"
       url="https://downloads.haskell.org/~ghc/${GHC_VERSION}/ghc-${GHC_VERSION}-${boot_triple}.tar.xz"
+      if "$(uname)" = "FreeBSD"; then
+          url="http://home.smart-cactus.org/~ben/ghc/ghc-9.4.1-x86_64-portbld-freebsd.tar.xz"
+      fi
       info "Fetching GHC binary distribution from $url..."
       curl "$url" > ghc.tar.xz || fail "failed to fetch GHC binary distribution"
       $TAR -xJf ghc.tar.xz || fail "failed to extract GHC binary distribution"
@@ -326,8 +329,7 @@ function fetch_cabal() {
           case "$(uname)" in
             Darwin) cabal_url="$base_url/cabal-install-$v-x86_64-apple-darwin17.7.0.tar.xz" ;;
             FreeBSD)
-              #cabal_url="$base_url/cabal-install-$v-x86_64-portbld-freebsd.tar.xz" ;;
-              cabal_url="http://home.smart-cactus.org/~ben/ghc/cabal-install-3.0.0.0-x86_64-portbld-freebsd.tar.xz" ;;
+              cabal_url="$base_url/cabal-install-$v-x86_64-freebsd.tar.xz" ;;
             *) fail "don't know where to fetch cabal-install for $(uname)"
           esac
           echo "Fetching cabal-install from $cabal_url"


=====================================
.gitlab/gen_ci.hs
=====================================
@@ -92,7 +92,7 @@ names of jobs to update these other places.
 data Opsys
   = Linux LinuxDistro
   | Darwin
-  | FreeBSD
+  | FreeBSD13
   | Windows deriving (Eq)
 
 data LinuxDistro
@@ -210,7 +210,7 @@ runnerTag arch (Linux distro) =
 runnerTag AArch64 Darwin = "aarch64-darwin"
 runnerTag Amd64 Darwin = "x86_64-darwin-m1"
 runnerTag Amd64 Windows = "new-x86_64-windows"
-runnerTag Amd64 FreeBSD = "x86_64-freebsd"
+runnerTag Amd64 FreeBSD13 = "x86_64-freebsd13"
 
 tags :: Arch -> Opsys -> BuildConfig -> [String]
 tags arch opsys _bc = [runnerTag arch opsys] -- Tag for which runners we can use
@@ -229,7 +229,7 @@ distroName Alpine     = "alpine3_12"
 opsysName :: Opsys -> String
 opsysName (Linux distro) = "linux-" ++ distroName distro
 opsysName Darwin = "darwin"
-opsysName FreeBSD = "freebsd"
+opsysName FreeBSD13 = "freebsd13"
 opsysName Windows = "windows"
 
 archName :: Arch -> String
@@ -299,15 +299,15 @@ type Variables = M.MonoidalMap String [String]
 a =: b = M.singleton a [b]
 
 opsysVariables :: Arch -> Opsys -> Variables
-opsysVariables _ FreeBSD = mconcat
+opsysVariables _ FreeBSD13 = mconcat
   [ -- N.B. we use iconv from ports as I see linker errors when we attempt
     -- to use the "native" iconv embedded in libc as suggested by the
     -- porting guide [1].
     -- [1] https://www.freebsd.org/doc/en/books/porters-handbook/using-iconv.html)
     "CONFIGURE_ARGS" =:  "--with-gmp-includes=/usr/local/include --with-gmp-libraries=/usr/local/lib --with-iconv-includes=/usr/local/include --with-iconv-libraries=/usr/local/lib"
   , "HADRIAN_ARGS" =: "--docs=no-sphinx"
-  , "GHC_VERSION" =: "9.2.2"
-  , "CABAL_INSTALL_VERSION" =: "3.2.0.0"
+  , "GHC_VERSION" =: "9.4.1"
+  , "CABAL_INSTALL_VERSION" =: "3.6.2.0"
   ]
 opsysVariables ARMv7 (Linux distro) =
   distroVariables distro <>
@@ -475,12 +475,12 @@ instance ToJSON OnOffRules where
 
 -- | A Rule corresponds to some condition which must be satisifed in order to
 -- run the job.
-data Rule = FastCI -- ^ Run this job when the fast-ci label is set
-          | ReleaseOnly -- ^ Only run this job in a release pipeline
-          | Nightly     -- ^ Only run this job in the nightly pipeline
-          | LLVMBackend -- ^ Only run this job when the "LLVM backend" label is present
-          | FreeBSDTag  -- ^ Only run this job when the "FreeBSD" label is set.
-          | Disable     -- ^ Don't run this job.
+data Rule = FastCI       -- ^ Run this job when the fast-ci label is set
+          | ReleaseOnly  -- ^ Only run this job in a release pipeline
+          | Nightly      -- ^ Only run this job in the nightly pipeline
+          | LLVMBackend  -- ^ Only run this job when the "LLVM backend" label is present
+          | FreeBSDLabel -- ^ Only run this job when the "FreeBSD" label is set.
+          | Disable      -- ^ Don't run this job.
           deriving (Bounded, Enum, Ord, Eq)
 
 -- A constant evaluating to True because gitlab doesn't support "true" in the
@@ -498,8 +498,8 @@ ruleString On FastCI = true
 ruleString Off FastCI = "$CI_MERGE_REQUEST_LABELS !~ /.*fast-ci.*/"
 ruleString On LLVMBackend = "$CI_MERGE_REQUEST_LABELS =~ /.*LLVM backend.*/"
 ruleString Off LLVMBackend = true
-ruleString On FreeBSDTag = "$CI_MERGE_REQUEST_LABELS =~ /.*FreeBSD.*/"
-ruleString Off FreeBSDTag = true
+ruleString On FreeBSDLabel = "$CI_MERGE_REQUEST_LABELS =~ /.*FreeBSD.*/"
+ruleString Off FreeBSDLabel = true
 ruleString On ReleaseOnly = "$RELEASE_JOB == \"yes\""
 ruleString Off ReleaseOnly = "$RELEASE_JOB != \"yes\""
 ruleString On Nightly = "$NIGHTLY"
@@ -766,7 +766,7 @@ jobs = M.fromList $ concatMap flattenJobGroup $
      , fastCI (standardBuilds Amd64 Windows)
      , disableValidate (standardBuildsWithConfig Amd64 Windows nativeInt)
      , standardBuilds Amd64 Darwin
-     , allowFailureGroup (addValidateRule FreeBSDTag (standardBuilds Amd64 FreeBSD))
+     , modifyJobs (\x -> x {jobStage = "quick-build"}) $ allowFailureGroup (addValidateRule FreeBSDLabel (standardBuilds Amd64 FreeBSD13))
      , standardBuilds AArch64 Darwin
      , standardBuilds AArch64 (Linux Debian10)
      , allowFailureGroup (disableValidate (standardBuilds ARMv7 (Linux Debian10)))


=====================================
.gitlab/jobs.yaml
=====================================
@@ -541,7 +541,7 @@
       "ac_cv_func_utimensat": "no"
     }
   },
-  "nightly-x86_64-freebsd-validate": {
+  "nightly-x86_64-freebsd13-validate": {
     "after_script": [
       ".gitlab/ci.sh save_cache",
       ".gitlab/ci.sh clean",
@@ -551,7 +551,7 @@
     "artifacts": {
       "expire_in": "8 weeks",
       "paths": [
-        "ghc-x86_64-freebsd-validate.tar.xz",
+        "ghc-x86_64-freebsd13-validate.tar.xz",
         "junit.xml"
       ],
       "reports": {
@@ -560,7 +560,7 @@
       "when": "always"
     },
     "cache": {
-      "key": "x86_64-freebsd-$CACHE_REV",
+      "key": "x86_64-freebsd13-$CACHE_REV",
       "paths": [
         "cabal-cache",
         "toolchain"
@@ -586,19 +586,19 @@
       ".gitlab/ci.sh build_hadrian",
       ".gitlab/ci.sh test_hadrian"
     ],
-    "stage": "full-build",
+    "stage": "quick-build",
     "tags": [
-      "x86_64-freebsd"
+      "x86_64-freebsd13"
     ],
     "variables": {
       "BIGNUM_BACKEND": "gmp",
-      "BIN_DIST_NAME": "ghc-x86_64-freebsd-validate",
+      "BIN_DIST_NAME": "ghc-x86_64-freebsd13-validate",
       "BUILD_FLAVOUR": "validate",
-      "CABAL_INSTALL_VERSION": "3.2.0.0",
+      "CABAL_INSTALL_VERSION": "3.6.2.0",
       "CONFIGURE_ARGS": "--with-gmp-includes=/usr/local/include --with-gmp-libraries=/usr/local/lib --with-iconv-includes=/usr/local/include --with-iconv-libraries=/usr/local/lib ",
-      "GHC_VERSION": "9.2.2",
+      "GHC_VERSION": "9.4.1",
       "HADRIAN_ARGS": "--docs=no-sphinx",
-      "TEST_ENV": "x86_64-freebsd-validate",
+      "TEST_ENV": "x86_64-freebsd13-validate",
       "XZ_OPT": "-9"
     }
   },
@@ -2050,7 +2050,7 @@
       "ac_cv_func_utimensat": "no"
     }
   },
-  "release-x86_64-freebsd-release": {
+  "release-x86_64-freebsd13-release": {
     "after_script": [
       ".gitlab/ci.sh save_cache",
       ".gitlab/ci.sh clean",
@@ -2060,7 +2060,7 @@
     "artifacts": {
       "expire_in": "1 year",
       "paths": [
-        "ghc-x86_64-freebsd-release.tar.xz",
+        "ghc-x86_64-freebsd13-release.tar.xz",
         "junit.xml"
       ],
       "reports": {
@@ -2069,7 +2069,7 @@
       "when": "always"
     },
     "cache": {
-      "key": "x86_64-freebsd-$CACHE_REV",
+      "key": "x86_64-freebsd13-$CACHE_REV",
       "paths": [
         "cabal-cache",
         "toolchain"
@@ -2095,20 +2095,20 @@
       ".gitlab/ci.sh build_hadrian",
       ".gitlab/ci.sh test_hadrian"
     ],
-    "stage": "full-build",
+    "stage": "quick-build",
     "tags": [
-      "x86_64-freebsd"
+      "x86_64-freebsd13"
     ],
     "variables": {
       "BIGNUM_BACKEND": "gmp",
-      "BIN_DIST_NAME": "ghc-x86_64-freebsd-release",
+      "BIN_DIST_NAME": "ghc-x86_64-freebsd13-release",
       "BUILD_FLAVOUR": "release",
-      "CABAL_INSTALL_VERSION": "3.2.0.0",
+      "CABAL_INSTALL_VERSION": "3.6.2.0",
       "CONFIGURE_ARGS": "--with-gmp-includes=/usr/local/include --with-gmp-libraries=/usr/local/lib --with-iconv-includes=/usr/local/include --with-iconv-libraries=/usr/local/lib ",
-      "GHC_VERSION": "9.2.2",
+      "GHC_VERSION": "9.4.1",
       "HADRIAN_ARGS": "--docs=no-sphinx",
       "IGNORE_PERF_FAILURES": "all",
-      "TEST_ENV": "x86_64-freebsd-release",
+      "TEST_ENV": "x86_64-freebsd13-release",
       "XZ_OPT": "-9"
     }
   },
@@ -2970,7 +2970,7 @@
       "ac_cv_func_utimensat": "no"
     }
   },
-  "x86_64-freebsd-validate": {
+  "x86_64-freebsd13-validate": {
     "after_script": [
       ".gitlab/ci.sh save_cache",
       ".gitlab/ci.sh clean",
@@ -2980,7 +2980,7 @@
     "artifacts": {
       "expire_in": "2 weeks",
       "paths": [
-        "ghc-x86_64-freebsd-validate.tar.xz",
+        "ghc-x86_64-freebsd13-validate.tar.xz",
         "junit.xml"
       ],
       "reports": {
@@ -2989,7 +2989,7 @@
       "when": "always"
     },
     "cache": {
-      "key": "x86_64-freebsd-$CACHE_REV",
+      "key": "x86_64-freebsd13-$CACHE_REV",
       "paths": [
         "cabal-cache",
         "toolchain"
@@ -3015,19 +3015,19 @@
       ".gitlab/ci.sh build_hadrian",
       ".gitlab/ci.sh test_hadrian"
     ],
-    "stage": "full-build",
+    "stage": "quick-build",
     "tags": [
-      "x86_64-freebsd"
+      "x86_64-freebsd13"
     ],
     "variables": {
       "BIGNUM_BACKEND": "gmp",
-      "BIN_DIST_NAME": "ghc-x86_64-freebsd-validate",
+      "BIN_DIST_NAME": "ghc-x86_64-freebsd13-validate",
       "BUILD_FLAVOUR": "validate",
-      "CABAL_INSTALL_VERSION": "3.2.0.0",
+      "CABAL_INSTALL_VERSION": "3.6.2.0",
       "CONFIGURE_ARGS": "--with-gmp-includes=/usr/local/include --with-gmp-libraries=/usr/local/lib --with-iconv-includes=/usr/local/include --with-iconv-libraries=/usr/local/lib ",
-      "GHC_VERSION": "9.2.2",
+      "GHC_VERSION": "9.4.1",
       "HADRIAN_ARGS": "--docs=no-sphinx",
-      "TEST_ENV": "x86_64-freebsd-validate"
+      "TEST_ENV": "x86_64-freebsd13-validate"
     }
   },
   "x86_64-linux-alpine3_12-int_native-validate+fully_static": {


=====================================
m4/fp_find_cxx_std_lib.m4
=====================================
@@ -18,10 +18,44 @@ unknown
 #endif
 EOF
         AC_MSG_CHECKING([C++ standard library flavour])
-        if "$CXX" -E actest.cpp -o actest.out; then
-            if grep "libc++" actest.out >/dev/null; then
-                CXX_STD_LIB_LIBS="c++ c++abi"
-                p="`"$CXX" --print-file-name libc++.so`"
+        if ! "$CXX" -E actest.cpp -o actest.out; then
+            rm -f actest.cpp actest.out
+            AC_MSG_ERROR([Failed to compile test program])
+        fi
+
+        dnl Identify standard library type
+        if grep "libc++" actest.out >/dev/null; then
+            CXX_STD_LIB_FLAVOUR="c++"
+            AC_MSG_RESULT([libc++])
+        elif grep "libstdc++" actest.out >/dev/null; then
+            CXX_STD_LIB_FLAVOUR="stdc++"
+            AC_MSG_RESULT([libstdc++])
+        else
+            rm -f actest.cpp actest.out
+            AC_MSG_ERROR([Unknown C++ standard library implementation.])
+        fi
+        rm -f actest.cpp actest.out
+
+        dnl -----------------------------------------
+        dnl Figure out how to link...
+        dnl -----------------------------------------
+        cat >actest.cpp <<-EOF
+#include <iostream>
+int main(int argc, char** argv) {
+    std::cout << "hello world\n";
+    return 0;
+}
+EOF
+        if ! "$CXX" -c actest.cpp; then
+            AC_MSG_ERROR([Failed to compile test object])
+        fi
+
+        try_libs() {
+            dnl Try to link a plain object with CC manually
+            AC_MSG_CHECKING([for linkage against '${3}'])
+            if "$CC" -o actest actest.o ${1} 2>/dev/null; then
+                CXX_STD_LIB_LIBS="${3}"
+                p="`"$CXX" --print-file-name ${2}`"
                 d="`dirname "$p"`"
                 dnl On some platforms (e.g. Windows) the C++ standard library
                 dnl can be found in the system search path. In this case $CXX
@@ -31,24 +65,25 @@ EOF
                 if test "$d" = "."; then d=""; fi
                 CXX_STD_LIB_LIB_DIRS="$d"
                 CXX_STD_LIB_DYN_LIB_DIRS="$d"
-                AC_MSG_RESULT([libc++])
-            elif grep "libstdc++" actest.out >/dev/null; then
-                CXX_STD_LIB_LIBS="stdc++"
-                p="`"$CXX" --print-file-name libstdc++.so`"
-                d="`dirname "$p"`"
-                if test "$d" = "."; then d=""; fi
-                CXX_STD_LIB_LIB_DIRS="$d"
-                CXX_STD_LIB_DYN_LIB_DIRS="$d"
-                AC_MSG_RESULT([libstdc++])
+                AC_MSG_RESULT([success])
+                true
             else
-                rm -f actest.cpp actest.out
-                AC_MSG_ERROR([Unknown C++ standard library implementation.])
+                AC_MSG_RESULT([failed])
+                false
             fi
-            rm -f actest.cpp actest.out
-        else
-            rm -f actest.cpp actest.out
-            AC_MSG_ERROR([Failed to compile test program])
-        fi
+        }
+        case $CXX_STD_LIB_FLAVOUR in
+        c++)
+            try_libs "-lc++ -lc++abi" "libc++.so" "c++ c++abi" || \
+            try_libs "-lc++ -lcxxrt" "libc++.so" "c++ cxxrt" ||
+            AC_MSG_ERROR([Failed to find C++ standard library]) ;;
+        stdc++)
+            try_libs "-lstdc++" "libstdc++.so" "stdc++" || \
+            try_libs "-lstdc++ -lsupc++" "libstdc++.so" "stdc++ supc++" || \
+            AC_MSG_ERROR([Failed to find C++ standard library]) ;;
+        esac
+
+        rm -f actest.cpp actest.o actest
     fi
 
     AC_SUBST([CXX_STD_LIB_LIBS])


=====================================
rts/Linker.c
=====================================
@@ -80,6 +80,33 @@
 #if defined(dragonfly_HOST_OS)
 #include <sys/tls.h>
 #endif
+
+/*
+ * Note [iconv and FreeBSD]
+ * ~~~~~~~~~~~~~~~~~~~~~~~~
+ *
+ * On FreeBSD libc.so provides an implementation of the iconv_* family of
+ * functions. However, due to their implementation, these symbols cannot be
+ * resolved via dlsym(); rather, they can only be resolved using the
+ * explicitly-versioned dlvsym().
+ *
+ * This is problematic for the RTS linker since we may be asked to load
+ * an object that depends upon iconv. To handle this we include a set of
+ * fallback cases for these functions, allowing us to resolve them to the
+ * symbols provided by the libc against which the RTS is linked.
+ *
+ * See #20354.
+ */
+
+#if defined(freebsd_HOST_OS)
+extern void iconvctl();
+extern void iconv_open_into();
+extern void iconv_open();
+extern void iconv_close();
+extern void iconv_canonicalize();
+extern void iconv();
+#endif
+
 /*
    Note [runtime-linker-support]
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -637,6 +664,10 @@ internal_dlsym(const char *symbol) {
     }
     RELEASE_LOCK(&dl_mutex);
 
+    IF_DEBUG(linker, debugBelch("internal_dlsym: looking for symbol '%s' in special cases\n", symbol));
+#   define SPECIAL_SYMBOL(sym) \
+      if (strcmp(symbol, #sym) == 0) return (void*)&sym;
+
 #   if defined(HAVE_SYS_STAT_H) && defined(linux_HOST_OS) && defined(__GLIBC__)
     // HACK: GLIBC implements these functions with a great deal of trickery where
     //       they are either inlined at compile time to their corresponding
@@ -646,18 +677,28 @@ internal_dlsym(const char *symbol) {
     //       We borrow the approach that the LLVM JIT uses to resolve these
     //       symbols. See http://llvm.org/PR274 and #7072 for more info.
 
-    IF_DEBUG(linker, debugBelch("internal_dlsym: looking for symbol '%s' in GLIBC special cases\n", symbol));
+    SPECIAL_SYMBOL(stat);
+    SPECIAL_SYMBOL(fstat);
+    SPECIAL_SYMBOL(lstat);
+    SPECIAL_SYMBOL(stat64);
+    SPECIAL_SYMBOL(fstat64);
+    SPECIAL_SYMBOL(lstat64);
+    SPECIAL_SYMBOL(atexit);
+    SPECIAL_SYMBOL(mknod);
+#   endif
 
-    if (strcmp(symbol, "stat") == 0) return (void*)&stat;
-    if (strcmp(symbol, "fstat") == 0) return (void*)&fstat;
-    if (strcmp(symbol, "lstat") == 0) return (void*)&lstat;
-    if (strcmp(symbol, "stat64") == 0) return (void*)&stat64;
-    if (strcmp(symbol, "fstat64") == 0) return (void*)&fstat64;
-    if (strcmp(symbol, "lstat64") == 0) return (void*)&lstat64;
-    if (strcmp(symbol, "atexit") == 0) return (void*)&atexit;
-    if (strcmp(symbol, "mknod") == 0) return (void*)&mknod;
+    // See Note [iconv and FreeBSD]
+#   if defined(freebsd_HOST_OS)
+    SPECIAL_SYMBOL(iconvctl);
+    SPECIAL_SYMBOL(iconv_open_into);
+    SPECIAL_SYMBOL(iconv_open);
+    SPECIAL_SYMBOL(iconv_close);
+    SPECIAL_SYMBOL(iconv_canonicalize);
+    SPECIAL_SYMBOL(iconv);
 #   endif
 
+#undef SPECIAL_SYMBOL
+
     // we failed to find the symbol
     return NULL;
 }



View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/compare/a17b223015c09dfadcd6240d3884bdbf406769af...9de048a2aab2fe24fb5236642c3d6b149242d1eb

-- 
View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/compare/a17b223015c09dfadcd6240d3884bdbf406769af...9de048a2aab2fe24fb5236642c3d6b149242d1eb
You're receiving this email because of your account on gitlab.haskell.org.


-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mail.haskell.org/pipermail/ghc-commits/attachments/20220807/6654f6ce/attachment-0001.html>


More information about the ghc-commits mailing list