[Git][ghc/ghc][wip/gc/nonmoving-pinned] nonmoving: Teach allocatePinned() to allocate into nonmoving heap
Ben Gamari (@bgamari)
gitlab at gitlab.haskell.org
Wed Dec 21 20:24:47 UTC 2022
Ben Gamari pushed to branch wip/gc/nonmoving-pinned at Glasgow Haskell Compiler / GHC
Commits:
c5fa7410 by Ben Gamari at 2022-12-21T15:24:39-05:00
nonmoving: Teach allocatePinned() to allocate into nonmoving heap
The allocatePinned() function is used to allocate pinned memory (e.g.
for newPinnedByteArray#)
- - - - -
5 changed files:
- rts/sm/NonMoving.c
- rts/sm/NonMoving.h
- rts/sm/NonMovingMark.c
- rts/sm/NonMovingScav.c
- rts/sm/Storage.c
Changes:
=====================================
rts/sm/NonMoving.c
=====================================
@@ -492,6 +492,24 @@ Mutex concurrent_coll_finished_lock;
* remembered set during the preparatory GC. This allows us to safely skip the
* non-moving write barrier without jeopardizing the snapshot invariant.
*
+ *
+ * Note [Allocating pinned objects into the non-moving heap]
+ * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ *
+ * Under the moving collector small, pinned ByteArray#s are allocated by
+ * Storage.c:allocatePinned() into a per-capability accumulator block which is
+ * filled in a bump-pointer fashion. While this scheme is simple, it can lead
+ * to very poor fragmentation behavior as objects become unreachable: a single
+ * live ByteArray# can keep an entire block of memory alive.
+ *
+ * When the non-moving collector is in use we can do better by allocating small
+ * pinned objects directly into the non-moving heap.
+ *
+ * One wrinkle here is that pinned ByteArrays may have alignment requirements
+ * which requires that we insert padding zero-words before the beginning of the
+ * object. We must be certain to account for this padding when inspecting the
+ * object.
+ *
*/
memcount nonmoving_live_words = 0;
@@ -660,8 +678,8 @@ void *nonmovingAllocate(Capability *cap, StgWord sz)
unsigned int log_block_size = log2_ceil(sz * sizeof(StgWord));
unsigned int block_count = nonmovingBlockCountFromSize(log_block_size);
- // The max we ever allocate is 3276 bytes (anything larger is a large
- // object and not moved) which is covered by allocator 9.
+ // The max we ever allocate is NONMOVING_MAX_BLOCK_SZ bytes (anything
+ // larger is a large object and not moved) which is covered by allocator 9.
ASSERT(log_block_size < NONMOVING_ALLOCA0 + NONMOVING_ALLOCA_CNT);
struct NonmovingAllocator *alloca = nonmovingHeap.allocators[log_block_size - NONMOVING_ALLOCA0];
=====================================
rts/sm/NonMoving.h
=====================================
@@ -92,11 +92,17 @@ struct NonmovingAllocator {
// allocators cover block sizes of 2^NONMOVING_ALLOCA0 to
// 2^(NONMOVING_ALLOCA0 + NONMOVING_ALLOCA_CNT) (in bytes)
+// The largest allocator class must be at least LARGE_OBJECT_THRESHOLD in size
+// as Storage.c:allocatePinned will allocate small pinned allocations into the
+// non-moving heap.
#define NONMOVING_ALLOCA_CNT 12
// maximum number of free segments to hold on to
#define NONMOVING_MAX_FREE 16
+// block size of largest allocator in bytes.
+#define NONMOVING_MAX_BLOCK_SZ (1 << (NONMOVING_ALLOCA0 + NONMOVING_ALLOCA_CNT - 1))
+
struct NonmovingHeap {
struct NonmovingAllocator *allocators[NONMOVING_ALLOCA_CNT];
// free segment list. This is a cache where we keep up to
=====================================
rts/sm/NonMovingMark.c
=====================================
@@ -1380,6 +1380,11 @@ mark_closure (MarkQueue *queue, const StgClosure *p0, StgClosure **origin)
// Trace pointers
/////////////////////////////////////////////////////
+ // Find beginning of object.
+ // See Note [Allocating pinned objects into the non-moving heap].
+ while (*(StgPtr*) p == NULL)
+ p = (StgClosure *) ((StgPtr*) p + 1);
+
const StgInfoTable *info = get_itbl(p);
switch (info->type) {
=====================================
rts/sm/NonMovingScav.c
=====================================
@@ -84,9 +84,18 @@
*/
void
-nonmovingScavengeOne (StgClosure *q)
+nonmovingScavengeOne (StgClosure *q0)
{
+ StgClosure *q = q0;
+
+ // N.B. There may be a gap before the first word of the closure in the case
+ // of an aligned ByteArray# as allocated by allocatePinned().
+ // See Note [Allocating pinned objects into the non-moving heap].
+ while (*(StgPtr*) q == NULL)
+ q = (StgClosure *) ((StgPtr*) q + 1);
+
ASSERT(LOOKS_LIKE_CLOSURE_PTR(q));
+
StgPtr p = (StgPtr)q;
const StgInfoTable *info = get_itbl(q);
const bool saved_eager_promotion = gct->eager_promotion;
=====================================
rts/sm/Storage.c
=====================================
@@ -1248,6 +1248,25 @@ allocatePinned (Capability *cap, W_ n /*words*/, W_ alignment /*bytes*/, W_ alig
const StgWord alignment_w = alignment / sizeof(W_);
+ // If the non-moving collector is enabled then we can allocate small,
+ // pinned allocations directly into the non-moving heap. This is a bit more
+ // expensive up-front but reduces fragmentation and is worthwhile since
+ // pinned allocations are often long-lived..
+ //
+ // See Note [Allocating pinned objects into the non-moving heap].
+ if (RTS_UNLIKELY(RtsFlags.GcFlags.useNonmoving)
+ && (n + alignment_w) * sizeof(W_) < NONMOVING_MAX_BLOCK_SZ)
+ {
+ ACQUIRE_SM_LOCK;
+ p = nonmovingAllocate(cap, n + alignment_w);
+ RELEASE_SM_LOCK;
+ W_ off_w = ALIGN_WITH_OFF_W(p, alignment, align_off);
+ MEMSET_SLOP_W(p, 0, off_w);
+ p += off_w;
+ MEMSET_SLOP_W(p + n, 0, alignment_w - off_w - 1);
+ return p;
+ }
+
// If the request is for a large object, then allocate()
// will give us a pinned object anyway.
if (n >= LARGE_OBJECT_THRESHOLD/sizeof(W_)) {
View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/commit/c5fa7410baecd9d8bf7b07565de341fdc0710bff
--
View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/commit/c5fa7410baecd9d8bf7b07565de341fdc0710bff
You're receiving this email because of your account on gitlab.haskell.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mail.haskell.org/pipermail/ghc-commits/attachments/20221221/b90acf95/attachment-0001.html>
More information about the ghc-commits
mailing list