Skip to content

Power of two size-pools #2

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
eightbitraptor wants to merge 21 commits into from
Closed

Power of two size-pools #2

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
12cbd04
Use power-of-2 size pool slot sizes
eightbitraptor Feb 13, 2026
0e6ade3
Add a 32 byte size pool for small objects
eightbitraptor Feb 16, 2026
aca2778
Introduce RVALUE_SIZE to capture the size of most RVALUES
eightbitraptor Feb 16, 2026
f6f96d5
Fix unsigned underflow in shape capacity for small size pools
eightbitraptor Feb 17, 2026
44439e5
Fix shape capacity for heaps with no room for embedded IVs
eightbitraptor Feb 17, 2026
5e886e8
Ensure T_OBJECT always has room for external IV storage
eightbitraptor Feb 17, 2026
1a69884
Scale initial heap size by bytes rather than slot count
eightbitraptor Feb 18, 2026
415720b
Fix the zjit tests.
eightbitraptor Feb 24, 2026
1d62f38
Update gc.rb docs for 6 size pools
eightbitraptor Feb 24, 2026
a6d4b97
Fix STATIC_ASSERT for class embedding with power-of-2 pools
eightbitraptor Feb 24, 2026
b08f648
Fix MMTk BASE_SLOT_SIZE to match default GC
eightbitraptor Feb 24, 2026
34acfad
Remove stale MMTk test exclusion for test_dump_includes_slot_size
eightbitraptor Feb 25, 2026
01251a1
Use VALUE alignment for pointer checks
eightbitraptor Feb 27, 2026
3d9e5b0
Introduce slot size lookup table with 48-byte pool
eightbitraptor Feb 18, 2026
d70af8c
Fix re-embed test for 48-byte RVALUE pool
eightbitraptor Feb 18, 2026
28d69e1
Fix allocator and bitmap ops for non-power-of-two pools
eightbitraptor Feb 27, 2026
7cb2c66
Rename MMTk SIZE_POOL_COUNT to HEAP_COUNT
eightbitraptor Feb 27, 2026
e9dbd9d
Fix RVALUE size assertion for debug builds
eightbitraptor Feb 28, 2026
7ec8f54
Update ZJIT snapshot for 48-byte RVALUE pool
eightbitraptor Mar 1, 2026
98223e5
Use power-of-two pools on 32-bit for now
eightbitraptor Mar 2, 2026
000910c
Fix MMTk RVALUE_OVERHEAD and ractor belonging
eightbitraptor Mar 2, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 22 additions & 13 deletions gc.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -269,43 +269,52 @@ def self.stat hash_or_key = nil
# GC.stat_heap
# # =>
# {0 =>
# {slot_size: 40,
# {slot_size: 32,
# heap_eden_pages: 24,
# heap_eden_slots: 12288,
# total_allocated_pages: 24,
# force_major_gc_count: 0,
# force_incremental_marking_finish_count: 0,
# total_allocated_objects: 8450,
# total_freed_objects: 3120},
# 1 =>
# {slot_size: 64,
# heap_eden_pages: 246,
# heap_eden_slots: 402802,
# total_allocated_pages: 246,
# force_major_gc_count: 2,
# force_incremental_marking_finish_count: 1,
# total_allocated_objects: 33867152,
# total_freed_objects: 33520523},
# 1 =>
# {slot_size: 80,
# 2 =>
# {slot_size: 128,
# heap_eden_pages: 84,
# heap_eden_slots: 68746,
# total_allocated_pages: 84,
# force_major_gc_count: 1,
# force_incremental_marking_finish_count: 4,
# total_allocated_objects: 147491,
# total_freed_objects: 90699},
# 2 =>
# {slot_size: 160,
# 3 =>
# {slot_size: 256,
# heap_eden_pages: 157,
# heap_eden_slots: 64182,
# total_allocated_pages: 157,
# force_major_gc_count: 0,
# force_incremental_marking_finish_count: 0,
# total_allocated_objects: 211460,
# total_freed_objects: 190075},
# 3 =>
# {slot_size: 320,
# 4 =>
# {slot_size: 512,
# heap_eden_pages: 8,
# heap_eden_slots: 1631,
# total_allocated_pages: 8,
# force_major_gc_count: 0,
# force_incremental_marking_finish_count: 0,
# total_allocated_objects: 1422,
# total_freed_objects: 700},
# 4 =>
# {slot_size: 640,
# 5 =>
# {slot_size: 1024,
# heap_eden_pages: 16,
# heap_eden_slots: 1628,
# total_allocated_pages: 16,
Expand All @@ -316,17 +325,17 @@ def self.stat hash_or_key = nil
#
# In the example above, the keys in the outer hash are the heap identifiers:
#
# GC.stat_heap.keys # => [0, 1, 2, 3, 4]
# GC.stat_heap.keys # => [0, 1, 2, 3, 4, 5]
#
# On CRuby, each heap identifier is an integer;
# on other implementations, a heap identifier may be a string.
#
# With only argument +heap_id+ given,
# returns statistics for the given heap identifier:
#
# GC.stat_heap(2)
# GC.stat_heap(3)
# # =>
# {slot_size: 160,
# {slot_size: 256,
# heap_eden_pages: 157,
# heap_eden_slots: 64182,
# total_allocated_pages: 157,
Expand All @@ -338,7 +347,7 @@ def self.stat hash_or_key = nil
# With arguments +heap_id+ and +key+ given,
# returns the value for the given key in the given heap:
#
# GC.stat_heap(2, :slot_size) # => 160
# GC.stat_heap(3, :slot_size) # => 256
#
# With arguments +nil+ and +hash+ given,
# merges the statistics for all heaps into the given hash:
Expand Down
143 changes: 109 additions & 34 deletions gc/default/default.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -187,9 +187,68 @@ static RB_THREAD_LOCAL_SPECIFIER int malloc_increase_local;
#define USE_TICK_T (PRINT_ENTER_EXIT_TICK || PRINT_ROOT_TICKS)

#ifndef HEAP_COUNT
# define HEAP_COUNT 5
# if SIZEOF_VALUE >= 8
# define HEAP_COUNT 7
# else
# define HEAP_COUNT 6
# endif
#endif

static const unsigned short heap_slot_size_table[HEAP_COUNT] = {
#if SIZEOF_VALUE >= 8
32, 48, 64, 128, 256, 512, 1024
#else
16, 32, 64, 128, 256, 512
#endif
};

static const size_t heap_init_slots_table[HEAP_COUNT] = {
#if SIZEOF_VALUE >= 8
/* [0] 32B */ 2000,
/* [1] 48B */ GC_HEAP_INIT_SLOTS,
/* [2] 64B */ GC_HEAP_INIT_SLOTS / 2,
/* [3] 128B */ GC_HEAP_INIT_SLOTS / 4,
/* [4] 256B */ GC_HEAP_INIT_SLOTS / 8,
/* [5] 512B */ GC_HEAP_INIT_SLOTS / 16,
/* [6] 1024B*/ GC_HEAP_INIT_SLOTS / 32,
#else
GC_HEAP_INIT_SLOTS, GC_HEAP_INIT_SLOTS / 2,
GC_HEAP_INIT_SLOTS / 4, GC_HEAP_INIT_SLOTS / 8,
GC_HEAP_INIT_SLOTS / 16, GC_HEAP_INIT_SLOTS / 32,
#endif
};

/* Precomputed reciprocals for fast slot index calculation.
* For slot size d: reciprocal = ceil(2^48 / d).
* Then offset / d == (uint32_t)((offset * reciprocal) >> 48)
* for all offset < HEAP_PAGE_SIZE. */
#define SLOT_RECIPROCAL_SHIFT 48

static const uint64_t heap_slot_reciprocal_table[HEAP_COUNT] = {
#if SIZEOF_VALUE >= 8
/* 32 */ (1ULL << 48) / 32,
/* 48 */ (1ULL << 48) / 48 + 1,
/* 64 */ (1ULL << 48) / 64,
/* 128 */ (1ULL << 48) / 128,
/* 256 */ (1ULL << 48) / 256,
/* 512 */ (1ULL << 48) / 512,
/* 1024*/ (1ULL << 48) / 1024,
#else
/* 16 */ (1ULL << 48) / 16,
/* 32 */ (1ULL << 48) / 32,
/* 64 */ (1ULL << 48) / 64,
/* 128 */ (1ULL << 48) / 128,
/* 256 */ (1ULL << 48) / 256,
/* 512 */ (1ULL << 48) / 512,
#endif
};

static inline bool
slot_size_is_power_of_two(unsigned short slot_size)
{
return (slot_size & (slot_size - 1)) == 0;
}

typedef struct ractor_newobj_heap_cache {
struct free_slot *freelist;
struct heap_page *using_page;
Expand Down Expand Up @@ -687,7 +746,12 @@ size_t rb_gc_impl_obj_slot_size(VALUE obj);
# endif
#endif

#define BASE_SLOT_SIZE (sizeof(struct RBasic) + sizeof(VALUE[RBIMPL_RVALUE_EMBED_LEN_MAX]) + RVALUE_OVERHEAD)
#if SIZEOF_VALUE >= 8
#define BASE_SLOT_SIZE_LOG2 5
#else
#define BASE_SLOT_SIZE_LOG2 4
#endif
#define BASE_SLOT_SIZE (1 << BASE_SLOT_SIZE_LOG2)

#ifndef MAX
# define MAX(a, b) (((a) > (b)) ? (a) : (b))
Expand Down Expand Up @@ -763,8 +827,11 @@ struct free_slot {
};

struct heap_page {
/* Cache line 0: allocation fast path + SLOT_INDEX */
struct free_slot *freelist;
uintptr_t start;
uint64_t slot_size_reciprocal;
unsigned short slot_size;
uint32_t slot_div_magic;
unsigned short total_slots;
unsigned short free_slots;
unsigned short final_slots;
Expand All @@ -779,8 +846,6 @@ struct heap_page {

struct heap_page *free_next;
struct heap_page_body *body;
uintptr_t start;
struct free_slot *freelist;
struct ccan_list_node page_node;

bits_t wb_unprotected_bits[HEAP_PAGE_BITMAP_LIMIT];
Expand Down Expand Up @@ -841,15 +906,13 @@ heap_page_in_global_empty_pages_pool(rb_objspace_t *objspace, struct heap_page *
#define GET_PAGE_HEADER(x) (&GET_PAGE_BODY(x)->header)
#define GET_HEAP_PAGE(x) (GET_PAGE_HEADER(x)->page)

static uint32_t slot_div_magics[HEAP_COUNT];

static inline size_t
slot_index_for_offset(size_t offset, uint32_t div_magic)
slot_index_for_offset(size_t offset, uint64_t reciprocal)
{
return (size_t)(((uint64_t)offset * div_magic) >> 32);
return (uint32_t)(((uint64_t)offset * reciprocal) >> SLOT_RECIPROCAL_SHIFT);
}

#define SLOT_INDEX(page, p) slot_index_for_offset((uintptr_t)(p) - (page)->start, (page)->slot_div_magic)
#define SLOT_INDEX(page, p) slot_index_for_offset((uintptr_t)(p) - (page)->start, (page)->slot_size_reciprocal)
#define SLOT_BITMAP_INDEX(page, p) (SLOT_INDEX(page, p) / BITS_BITLENGTH)
#define SLOT_BITMAP_OFFSET(page, p) (SLOT_INDEX(page, p) & (BITS_BITLENGTH - 1))
#define SLOT_BITMAP_BIT(page, p) ((bits_t)1 << SLOT_BITMAP_OFFSET(page, p))
Expand Down Expand Up @@ -1636,7 +1699,7 @@ heap_page_add_freeobj(rb_objspace_t *objspace, struct heap_page *page, VALUE obj
/* obj should belong to page */
!(page->start <= (uintptr_t)obj &&
(uintptr_t)obj < ((uintptr_t)page->start + (page->total_slots * page->slot_size)) &&
obj % BASE_SLOT_SIZE == 0)) {
obj % sizeof(VALUE) == 0)) {
rb_bug("heap_page_add_freeobj: %p is not rvalue.", (void *)obj);
}

Expand Down Expand Up @@ -1977,19 +2040,17 @@ heap_add_page(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *page)
GC_ASSERT(!heap->sweeping_page);
GC_ASSERT(heap_page_in_global_empty_pages_pool(objspace, page));

/* Align start to the first slot_size boundary after the page header */
/* Align start to slot_size boundary */
uintptr_t start = (uintptr_t)page->body + sizeof(struct heap_page_header);
size_t remainder = start % heap->slot_size;
if (remainder != 0) {
start += heap->slot_size - remainder;
}
uintptr_t rem = start % heap->slot_size;
if (rem) start += heap->slot_size - rem;

int slot_count = (int)((HEAP_PAGE_SIZE - (start - (uintptr_t)page->body))/heap->slot_size);

page->start = start;
page->total_slots = slot_count;
page->slot_size = heap->slot_size;
page->slot_div_magic = slot_div_magics[heap - heaps];
page->slot_size_reciprocal = heap_slot_reciprocal_table[heap - heaps];
page->heap = heap;

memset(&page->wb_unprotected_bits[0], 0, HEAP_PAGE_BITMAP_SIZE);
Expand Down Expand Up @@ -2237,13 +2298,7 @@ heap_slot_size(unsigned char pool_id)
{
GC_ASSERT(pool_id < HEAP_COUNT);

size_t slot_size = (1 << pool_id) * BASE_SLOT_SIZE;

#if RGENGC_CHECK_MODE
rb_objspace_t *objspace = rb_gc_get_objspace();
GC_ASSERT(heaps[pool_id].slot_size == (short)slot_size);
#endif

size_t slot_size = heap_slot_size_table[pool_id];
slot_size -= RVALUE_OVERHEAD;

return slot_size;
Expand Down Expand Up @@ -2356,10 +2411,17 @@ heap_idx_for_size(size_t size)
{
size += RVALUE_OVERHEAD;

size_t slot_count = CEILDIV(size, BASE_SLOT_SIZE);
if (size <= BASE_SLOT_SIZE) return 0;
if (size <= heap_slot_size_table[1]) return 1;

/* heap_idx is ceil(log2(slot_count)) */
size_t heap_idx = 64 - nlz_int64(slot_count - 1);
#if SIZEOF_VALUE >= 8
/* Pools from index 2 onward are powers of two (64, 128, ...),
* so the log2 formula still works — add 1 to skip the 48B pool. */
size_t heap_idx = 64 - nlz_int64(size - 1) - BASE_SLOT_SIZE_LOG2 + 1;
#else
/* All pools are powers of two on 32-bit. */
size_t heap_idx = 64 - nlz_int64(size - 1) - BASE_SLOT_SIZE_LOG2;
#endif

if (heap_idx >= HEAP_COUNT) {
rb_bug("heap_idx_for_size: allocation size too large "
Expand Down Expand Up @@ -2589,7 +2651,7 @@ is_pointer_to_heap(rb_objspace_t *objspace, const void *ptr)
if (p < heap_pages_lomem || p > heap_pages_himem) return FALSE;
RB_DEBUG_COUNTER_INC(gc_isptr_range);

if (p % BASE_SLOT_SIZE != 0) return FALSE;
if (p % sizeof(VALUE) != 0) return FALSE;
RB_DEBUG_COUNTER_INC(gc_isptr_align);

page = heap_page_for_ptr(objspace, (uintptr_t)ptr);
Expand Down Expand Up @@ -3495,7 +3557,7 @@ gc_sweep_plane(rb_objspace_t *objspace, rb_heap_t *heap, uintptr_t p, bits_t bit

do {
VALUE vp = (VALUE)p;
GC_ASSERT(vp % BASE_SLOT_SIZE == 0);
GC_ASSERT(vp % sizeof(VALUE) == 0);

rb_asan_unpoison_object(vp, false);
if (bitset & 1) {
Expand Down Expand Up @@ -5594,7 +5656,7 @@ gc_compact_plane(rb_objspace_t *objspace, rb_heap_t *heap, uintptr_t p, bits_t b

do {
VALUE vp = (VALUE)p;
GC_ASSERT(vp % BASE_SLOT_SIZE == 0);
GC_ASSERT(vp % sizeof(VALUE) == 0);

if (bitset & 1) {
objspace->rcompactor.considered_count_table[BUILTIN_TYPE(vp)]++;
Expand Down Expand Up @@ -9515,11 +9577,15 @@ rb_gc_impl_objspace_init(void *objspace_ptr)
rb_bug("Could not preregister postponed job for GC");
}

/* A standard RVALUE (RBasic + embedded VALUEs + debug overhead) must fit
* in at least one pool. In debug builds RVALUE_OVERHEAD can push this
* beyond the 48-byte pool into the 64-byte pool, which is fine. */
GC_ASSERT(rb_gc_impl_size_allocatable_p(sizeof(struct RBasic) + sizeof(VALUE[RBIMPL_RVALUE_EMBED_LEN_MAX])));

for (int i = 0; i < HEAP_COUNT; i++) {
rb_heap_t *heap = &heaps[i];

heap->slot_size = (1 << i) * BASE_SLOT_SIZE;
slot_div_magics[i] = (uint32_t)((uint64_t)UINT32_MAX / heap->slot_size + 1);
heap->slot_size = heap_slot_size_table[i];

ccan_list_head_init(&heap->pages);
}
Expand All @@ -9541,8 +9607,10 @@ rb_gc_impl_objspace_init(void *objspace_ptr)
#endif
/* Set size pools allocatable pages. */
for (int i = 0; i < HEAP_COUNT; i++) {
/* Set the default value of heap_init_slots. */
gc_params.heap_init_slots[i] = GC_HEAP_INIT_SLOTS;
/* Set the default value of heap_init_slots using a pre-computed
* table. Pool 1 (48B on 64-bit) gets the most slots since it holds
* the busiest objects; larger pools scale down proportionally. */
gc_params.heap_init_slots[i] = heap_init_slots_table[i];
}

init_mark_stack(&objspace->mark_stack);
Expand All @@ -9557,6 +9625,13 @@ rb_gc_impl_init(void)
VALUE gc_constants = rb_hash_new();
rb_hash_aset(gc_constants, ID2SYM(rb_intern("DEBUG")), GC_DEBUG ? Qtrue : Qfalse);
rb_hash_aset(gc_constants, ID2SYM(rb_intern("BASE_SLOT_SIZE")), SIZET2NUM(BASE_SLOT_SIZE - RVALUE_OVERHEAD));
/* Minimum slot size for a standard RVALUE (RBasic + embedded VALUEs) */
size_t rvalue_min = sizeof(struct RBasic) + sizeof(VALUE[RBIMPL_RVALUE_EMBED_LEN_MAX]) + RVALUE_OVERHEAD;
size_t rvalue_slot = heap_slot_size_table[0];
for (int i = 1; i < HEAP_COUNT && rvalue_slot < rvalue_min; i++) {
rvalue_slot = heap_slot_size_table[i];
}
rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_SIZE")), SIZET2NUM(rvalue_slot - RVALUE_OVERHEAD));
rb_hash_aset(gc_constants, ID2SYM(rb_intern("RBASIC_SIZE")), SIZET2NUM(sizeof(struct RBasic)));
rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_OVERHEAD")), SIZET2NUM(RVALUE_OVERHEAD));
rb_hash_aset(gc_constants, ID2SYM(rb_intern("HEAP_PAGE_BITMAP_SIZE")), SIZET2NUM(HEAP_PAGE_BITMAP_SIZE));
Expand Down
Loading
Loading