[Git][ghc/ghc][wip/unroll-evac] Evac: Try unrolling copying

[Ben Gamari]

Ben Gamari pushed to branch wip/unroll-evac at Glasgow Haskell Compiler / GHC


Commits:
3a120e07 by Ben Gamari at 2019-05-10T23:58:32Z
Evac: Try unrolling copying

- - - - -


1 changed file:

- rts/sm/Evac.c


Changes:

=====================================
rts/sm/Evac.c
=====================================
@@ -12,6 +12,7 @@
  * ---------------------------------------------------------------------------*/
 
 #include "PosixSource.h"
+#include <string.h>
 #include "Rts.h"
 
 #include "Evac.h"
@@ -88,20 +89,35 @@ alloc_for_copy (uint32_t size, uint32_t gen_no)
    The evacuate() code
    -------------------------------------------------------------------------- */
 
+/* Manually unroll copy for small closures */
+STATIC_INLINE GNUC_ATTR_HOT void
+copy_words(StgWord *from, StgWord *to, uint32_t n)
+{
+    switch (n) {
+      case 7: to[6] = from[6]; FALLTHROUGH;
+      case 6: to[5] = from[5]; FALLTHROUGH;
+      case 5: to[4] = from[4]; FALLTHROUGH;
+      case 4: to[3] = from[3]; FALLTHROUGH;
+      case 3: to[2] = from[2]; FALLTHROUGH;
+      case 2: to[1] = from[1]; FALLTHROUGH;
+      case 1: to[0] = from[0]; FALLTHROUGH;
+      case 0: break;
+      default:
+        memcpy(to, from, n * sizeof(StgWord));
+    }
+}
+
 STATIC_INLINE GNUC_ATTR_HOT void
 copy_tag(StgClosure **p, const StgInfoTable *info,
          StgClosure *src, uint32_t size, uint32_t gen_no, StgWord tag)
 {
     StgPtr to, from;
-    uint32_t i;
 
     to = alloc_for_copy(size,gen_no);
 
     from = (StgPtr)src;
     to[0] = (W_)info;
-    for (i = 1; i < size; i++) { // unroll for small i
-        to[i] = from[i];
-    }
+    copy_words(&from[1], &to[1], size-1);
 
 //  if (to+size+2 < bd->start + BLOCK_SIZE_W) {
 //      __builtin_prefetch(to + size + 2, 1);
@@ -154,9 +170,7 @@ copy_tag_nolock(StgClosure **p, const StgInfoTable *info,
 
     from = (StgPtr)src;
     to[0] = (W_)info;
-    for (i = 1; i < size; i++) { // unroll for small i
-        to[i] = from[i];
-    }
+    copy_words(&from[1], &to[1], size-1);
 
     // if somebody else reads the forwarding pointer, we better make
     // sure there's a closure at the end of it.
@@ -185,7 +199,6 @@ copyPart(StgClosure **p, StgClosure *src, uint32_t size_to_reserve,
          uint32_t size_to_copy, uint32_t gen_no)
 {
     StgPtr to, from;
-    uint32_t i;
     StgWord info;
 
 #if defined(PARALLEL_GC)
@@ -211,9 +224,7 @@ spin:
 
     from = (StgPtr)src;
     to[0] = info;
-    for (i = 1; i < size_to_copy; i++) { // unroll for small i
-        to[i] = from[i];
-    }
+    copy_words(&from[1], &to[1], size_to_copy-1);
 
     write_barrier();
     src->header.info = (const StgInfoTable*)MK_FORWARDING_PTR(to);



View it on GitLab: https://gitlab.haskell.org/ghc/ghc/commit/3a120e07a876c6e8e109b56c9da47632416b86f6

-- 
View it on GitLab: https://gitlab.haskell.org/ghc/ghc/commit/3a120e07a876c6e8e109b56c9da47632416b86f6
You're receiving this email because of your account on gitlab.haskell.org.


-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mail.haskell.org/pipermail/ghc-commits/attachments/20190510/f9619464/attachment-0001.html>