[Git][ghc/ghc][master] testsuite: expand size testing infrastructure

Marge Bot (@marge-bot) gitlab at gitlab.haskell.org
Thu May 2 19:41:49 UTC 2024



Marge Bot pushed to branch master at Glasgow Haskell Compiler / GHC


Commits:
9bae34d8 by doyougnu at 2024-05-02T15:41:08-04:00
testsuite: expand size testing infrastructure

- closes #24191
- adds windows_skip, wasm_skip, wasm_arch, find_so, _find_so
- path_from_ghcPkg, collect_size_ghc_pkg, collect_object_size, find_non_inplace functions to testsuite
- adds on_windows and req_dynamic_ghc predicate to testsuite

The design is to not make the testsuite too smart and simply offload to
ghc-pkg for locations of object files and directories.

- - - - -


3 changed files:

- testsuite/driver/testglobals.py
- testsuite/driver/testlib.py
- testsuite/tests/perf/size/all.T


Changes:

=====================================
testsuite/driver/testglobals.py
=====================================
@@ -221,6 +221,10 @@ class TestConfig:
         # I have no idea what this does
         self.package_conf_cache_file = None # type: Optional[Path]
 
+        # the libdir for the test compiler. Set by hadrian, see
+        # Setting.Builders.RunTest
+        self.libdir = ''
+
         # The extra hadrian dependencies we need for all configured tests
         self.hadrian_deps = set() # type: Set[str]
 


=====================================
testsuite/driver/testlib.py
=====================================
@@ -143,6 +143,15 @@ def js_skip( name, opts ):
     if js_arch():
         skip(name,opts)
 
+# disable test on WASM arch
+def wasm_skip( name, opts ):
+    if wasm_arch():
+        skip(name,opts)
+
+def windows_skip(name,opts):
+    if on_windows():
+        skip(name,opts)
+
 # expect broken for the JS backend
 def js_broken( bug: IssueNumber ):
     if js_arch():
@@ -239,6 +248,15 @@ def req_dynamic_hs( name, opts ):
     if not config.supports_dynamic_hs:
         opts.expect = 'fail'
 
+def req_dynamic_ghc( name, opts ):
+    '''
+    Require that the GHC is dynamically linked, if static then skip.
+    See tests/perf/size/all.T, specifically foo.so tests for use case
+    and example
+    '''
+    if not config.ghc_dynamic:
+        skip(name,opts)
+
 def req_interp( name, opts ):
     if not config.have_interp or needsTargetWrapper():
         opts.expect = 'fail'
@@ -611,15 +629,24 @@ def collect_size ( deviation, path ):
 
 def get_dir_size(path):
     total = 0
-    with os.scandir(path) as it:
-        for entry in it:
-            if entry.is_file():
-                total += entry.stat().st_size
-            elif entry.is_dir():
-                total += get_dir_size(entry.path)
-    return total
+    try:
+        with os.scandir(path) as it:
+            for entry in it:
+                if entry.is_file():
+                    total += entry.stat().st_size
+                elif entry.is_dir():
+                    total += get_dir_size(entry.path)
+        return total
+    except FileNotFoundError:
+        print("Exception: Could not find: " + path)
 
 def collect_size_dir ( deviation, path ):
+
+    ## os.path.join joins the path with slashes (not backslashes) on windows
+    ## CI...for some reason, so we manually detect it here
+    sep = r"/"
+    if on_windows():
+        sep = r"\\"
     return collect_generic_stat ( 'size', deviation, lambda way: get_dir_size(path) )
 
 # Read a number from a specific file
@@ -636,6 +663,91 @@ def collect_generic_stats ( metric_info ):
         return _collect_generic_stat(name, opts, metric_info)
     return f
 
+# wrap the call to collect_size_dir with path_from_ghcPkg in a function. Python
+# is call-by-value so if we placed the call in an all.T file then the python
+# interpreter would evaluate the call to path_from_ghcPkg
+def collect_size_ghc_pkg (deviation, library):
+    return collect_size_dir(deviation, path_from_ghcPkg(library, "library-dirs"))
+
+# same for collect_size and find_so
+def collect_object_size (deviation, library, use_non_inplace=False):
+    if use_non_inplace:
+        return collect_size(deviation, find_non_inplace_so(library))
+    else:
+        return collect_size(deviation, find_so(library))
+
+def path_from_ghcPkg (library, field):
+    """Find the field as a path for a library via a call to ghc-pkg. This is a
+    testsuite wrapper around a call to ghc-pkg field {library} {field}.
+    """
+
+    ### example output from ghc-pkg:
+    ###  $ ./ghc-pkg field Cabal library-dirs
+    ###    library-dirs: /home/doyougnu/programming/haskell/ghc/_build/stage1/lib/../lib/x86_64-linux-ghc-9.11.20240424/Cabal-3.11.0.0-inplace
+    ###    so we split the string and drop the 'library-dirs'
+    ghcPkgCmd = fr"{config.ghc_pkg} field {library} {field}"
+
+    try:
+        result = subprocess.run(ghcPkgCmd, capture_output=True, shell=True)
+
+        # check_returncode throws an exception if the return code is not 0.
+        result.check_returncode()
+
+        # if we get here then the call worked and we have the path we split by
+        # whitespace and then return the path which becomes the second element
+        # in the array
+        return re.split(r'\s+', result.stdout.decode("utf-8"))[1]
+    except Exception as e:
+        message = f"""
+        Attempt to find {field} of {library} using ghc-pkg failed.
+        ghc-pkg path: {config.ghc_pkg}
+        error" {e}
+        """
+        print(message)
+
+
+def _find_so(lib, directory, in_place):
+    """Find a shared object file (.so) for lib in directory. We deliberately
+    keep the regex simple, just removing the ghc version and project version.
+    Example:
+
+    _find_so("Cabal-syntax-3.11.0.0", path-from-ghc-pkg, True) ==>
+    /builds/ghc/ghc/_build/install/lib/ghc-9.11.20240410/lib/x86_64-linux-ghc-9.11.20240410/libHSCabal-syntax-3.11.0.0-inplace-ghc9.11.20240410.so
+    """
+
+    # produce the suffix for the CI operating system
+    suffix = "so"
+    if config.os == "mingw32":
+        suffix = "dll"
+    elif config.os == "darwin":
+        suffix = "dylib"
+
+    # Most artfacts are of the form foo-inplace, except for the rts.
+    if in_place:
+        to_match = r'libHS{}-\d+(\.\d+)+-inplace-\S+\.' + suffix
+    else:
+        to_match = r'libHS{}-\d+(\.\d+)+\S+\.' + suffix
+
+    matches = []
+    # wrap this in some exception handling, hadrian test will error out because
+    # these files don't exist yet, so we pass when this occurs
+    try:
+        for f in os.listdir(directory):
+            if f.endswith(suffix):
+                pattern = re.compile(to_match.format(re.escape(lib)))
+                match   = re.match(pattern, f)
+                if match:
+                    matches.append(match.group())
+        return os.path.join(directory, matches[0])
+    except:
+        failBecause('Could not find shared object file: ' + lib)
+
+def find_so(lib):
+    return _find_so(lib,path_from_ghcPkg(lib, "dynamic-library-dirs"),True)
+
+def find_non_inplace_so(lib):
+    return _find_so(lib,path_from_ghcPkg(lib, "dynamic-library-dirs"),False)
+
 # Define the a generic stat test, which computes the statistic by calling the function
 # given as the third argument.
 def collect_generic_stat ( metric, deviation, get_stat ):
@@ -804,9 +916,9 @@ KNOWN_OPERATING_SYSTEMS = set([
 ])
 
 def exe_extension() -> str:
-    if config.arch == 'wasm32':
+    if wasm_arch():
         return '.wasm'
-    elif config.os == "mingw32":
+    elif on_windows():
         return '.exe'
     return ''
 
@@ -829,6 +941,9 @@ def cygwin( ) -> bool:
 def js_arch() -> bool:
     return arch("javascript");
 
+def on_windows() -> bool:
+    return config.os == "mingw32"
+
 def wasm_arch() -> bool:
     return arch("wasm32")
 


=====================================
testsuite/tests/perf/size/all.T
=====================================
@@ -3,4 +3,77 @@ test('size_hello_obj', [collect_size(5, 'size_hello_obj.o')], compile, [''])
 test('size_hello_artifact', [collect_size(5, 'size_hello_artifact' + exe_extension())],
                              compile_artifact, [''])
 
-test('libdir',[collect_size_dir(10, config.libdir)], static_stats, [] )
+test('array_dir'           ,[collect_size_ghc_pkg(5 , 'array')]           , static_stats , [] )
+test('base_dir'            ,[collect_size_ghc_pkg(5 , 'base')]            , static_stats , [] )
+test('binary_dir'          ,[collect_size_ghc_pkg(5 , 'binary')]          , static_stats , [] )
+test('bytestring_dir'      ,[collect_size_ghc_pkg(5 , 'bytestring')]      , static_stats , [] )
+test('cabal_dir'           ,[collect_size_ghc_pkg(5 , 'Cabal')]           , static_stats , [] )
+test('cabal_syntax_dir'    ,[collect_size_ghc_pkg(5 , 'Cabal-syntax')]    , static_stats , [] )
+test('containers_dir'      ,[collect_size_ghc_pkg(5 , 'containers')]      , static_stats , [] )
+test('deepseq_dir'         ,[collect_size_ghc_pkg(5 , 'deepseq')]         , static_stats , [] )
+test('directory_dir'       ,[collect_size_ghc_pkg(5 , 'directory')]       , static_stats , [] )
+test('exceptions_dir'      ,[collect_size_ghc_pkg(5 , 'exceptions')]      , static_stats , [] )
+test('ghc_bignum_dir'      ,[collect_size_ghc_pkg(5 , 'ghc-bignum')]      , static_stats , [] )
+test('ghc_boot_dir'        ,[collect_size_ghc_pkg(5 , 'ghc-boot')]        , static_stats , [] )
+test('ghc_boot_th_dir'     ,[collect_size_ghc_pkg(5 , 'ghc-boot-th')]     , static_stats , [] )
+test('ghc_compact_dir'     ,[collect_size_ghc_pkg(5 , 'ghc-compact')]     , static_stats , [] )
+test('ghc_dir'             ,[collect_size_ghc_pkg(5 , 'ghc')]             , static_stats , [] )
+test('ghc_experimental_dir',[collect_size_ghc_pkg(5 , 'ghc-experimental')], static_stats , [] )
+test('ghc_heap_dir'        ,[collect_size_ghc_pkg(5 , 'ghc-heap')]        , static_stats , [] )
+test('ghc_internal_dir'    ,[collect_size_ghc_pkg(5 , 'ghc-internal')]    , static_stats , [] )
+test('ghc_platform_dir'    ,[collect_size_ghc_pkg(5 , 'ghc-platform')]    , static_stats , [] )
+test('ghc_prim_dir'        ,[collect_size_ghc_pkg(5 , 'ghc-prim')]        , static_stats , [] )
+test('ghc_toolchain_dir'   ,[collect_size_ghc_pkg(5 , 'ghc-toolchain')]   , static_stats , [] )
+test('haskeline_dir'       ,[collect_size_ghc_pkg(5 , 'haskeline')]       , static_stats , [] )
+test('hpc_dir'             ,[collect_size_ghc_pkg(5 , 'hpc')]             , static_stats , [] )
+test('integer_gmp_dir'     ,[collect_size_ghc_pkg(5 , 'integer-gmp')]     , static_stats , [] )
+test('mtl_dir'             ,[collect_size_ghc_pkg(5 , 'mtl')]             , static_stats , [] )
+test('os_string_dir'       ,[collect_size_ghc_pkg(5 , 'os-string')]       , static_stats , [] )
+test('parsec_dir'          ,[collect_size_ghc_pkg(5 , 'parsec')]          , static_stats , [] )
+test('pretty_dir'          ,[collect_size_ghc_pkg(5 , 'pretty')]          , static_stats , [] )
+test('process_dir'         ,[collect_size_ghc_pkg(5 , 'process')]         , static_stats , [] )
+test('time_dir'            ,[collect_size_ghc_pkg(5 , 'time')]            , static_stats , [] )
+test('xhtml_dir'           ,[collect_size_ghc_pkg(5 , 'xhtml')]           , static_stats , [] )
+
+# size of the entire libdir
+test('libdir'              ,[collect_size_dir(10, config.libdir)]                       , static_stats , [] )
+
+# skip these on windows
+test('unix_dir'     ,[windows_skip, collect_size_ghc_pkg(5, 'unix')]    , static_stats, [] )
+test('terminfo_dir' ,[windows_skip, js_skip, collect_size_ghc_pkg(5, 'terminfo')], static_stats, [] )
+
+# skip the shared object file tests on windows
+test('array_so'           ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "array")]            , static_stats, [] )
+test('base_so'            ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "base")]             , static_stats, [] )
+test('binary_so'          ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "binary")]           , static_stats, [] )
+test('bytestring_so'      ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "bytestring")]       , static_stats, [] )
+test('cabal_so'           ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "Cabal")]            , static_stats, [] )
+test('cabal_syntax_so'    ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "Cabal-syntax")]     , static_stats, [] )
+test('containers_so'      ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "containers")]       , static_stats, [] )
+test('deepseq_so'         ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "deepseq")]          , static_stats, [] )
+test('directory_so'       ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "directory")]        , static_stats, [] )
+test('exceptions_so'      ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "exceptions")]       , static_stats, [] )
+test('filepath_so'        ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "filepath")]         , static_stats, [] )
+test('ghc_bignum_so'      ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "ghc-bignum")]       , static_stats, [] )
+test('ghc_boot_so'        ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "ghc-boot")]         , static_stats, [] )
+test('ghc_boot_th_so'     ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "ghc-boot-th")]      , static_stats, [] )
+test('ghc_experimental_so',[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "ghc-experimental")] , static_stats, [] )
+test('ghc_heap_so'        ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "ghc-heap")]         , static_stats, [] )
+test('ghc_platform_so'    ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "ghc-platform")]     , static_stats, [] )
+test('ghc_prim_so'        ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "ghc-prim")]         , static_stats, [] )
+test('ghc_so'             ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "ghc")]              , static_stats, [] )
+test('ghc_toolchain_so'   ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "ghc-toolchain")]    , static_stats, [] )
+test('ghci_so'            ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "ghci")]             , static_stats, [] )
+test('haskeline_so'       ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "haskeline")]        , static_stats, [] )
+test('hpc_so'             ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "hpc")]              , static_stats, [] )
+test('mtl_so'             ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "mtl")]              , static_stats, [] )
+test('os_string_so'       ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "os-string")]        , static_stats, [] )
+test('parsec_so'          ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "parsec")]           , static_stats, [] )
+test('process_so'         ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "process")]          , static_stats, [] )
+test('rts_so'             ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "rts", True)]              , static_stats, [] )
+test('template_haskell_so',[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "template-haskell")] , static_stats, [] )
+test('terminfo_so'        ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "terminfo")]         , static_stats, [] )
+test('text_so'            ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "text")]             , static_stats, [] )
+test('time_so'            ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "time")]             , static_stats, [] )
+test('transformers_so'    ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "transformers")]     , static_stats, [] )
+test('xhtml_so'           ,[req_dynamic_ghc, js_skip, windows_skip, collect_object_size(5, "xhtml")]            , static_stats, [] )



View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/commit/9bae34d87f6c978e03031c549920071857c9080c

-- 
View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/commit/9bae34d87f6c978e03031c549920071857c9080c
You're receiving this email because of your account on gitlab.haskell.org.


-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mail.haskell.org/pipermail/ghc-commits/attachments/20240502/133d406a/attachment-0001.html>


More information about the ghc-commits mailing list