diff --git a/gc.rb b/gc.rb index 59adcbc62f64d6..d65a37e8aac501 100644 --- a/gc.rb +++ b/gc.rb @@ -269,7 +269,16 @@ def self.stat hash_or_key = nil # GC.stat_heap # # => # {0 => - # {slot_size: 40, + # {slot_size: 32, + # heap_eden_pages: 24, + # heap_eden_slots: 12288, + # total_allocated_pages: 24, + # force_major_gc_count: 0, + # force_incremental_marking_finish_count: 0, + # total_allocated_objects: 8450, + # total_freed_objects: 3120}, + # 1 => + # {slot_size: 64, # heap_eden_pages: 246, # heap_eden_slots: 402802, # total_allocated_pages: 246, @@ -277,8 +286,8 @@ def self.stat hash_or_key = nil # force_incremental_marking_finish_count: 1, # total_allocated_objects: 33867152, # total_freed_objects: 33520523}, - # 1 => - # {slot_size: 80, + # 2 => + # {slot_size: 128, # heap_eden_pages: 84, # heap_eden_slots: 68746, # total_allocated_pages: 84, @@ -286,8 +295,8 @@ def self.stat hash_or_key = nil # force_incremental_marking_finish_count: 4, # total_allocated_objects: 147491, # total_freed_objects: 90699}, - # 2 => - # {slot_size: 160, + # 3 => + # {slot_size: 256, # heap_eden_pages: 157, # heap_eden_slots: 64182, # total_allocated_pages: 157, @@ -295,8 +304,8 @@ def self.stat hash_or_key = nil # force_incremental_marking_finish_count: 0, # total_allocated_objects: 211460, # total_freed_objects: 190075}, - # 3 => - # {slot_size: 320, + # 4 => + # {slot_size: 512, # heap_eden_pages: 8, # heap_eden_slots: 1631, # total_allocated_pages: 8, @@ -304,8 +313,8 @@ def self.stat hash_or_key = nil # force_incremental_marking_finish_count: 0, # total_allocated_objects: 1422, # total_freed_objects: 700}, - # 4 => - # {slot_size: 640, + # 5 => + # {slot_size: 1024, # heap_eden_pages: 16, # heap_eden_slots: 1628, # total_allocated_pages: 16, @@ -316,7 +325,7 @@ def self.stat hash_or_key = nil # # In the example above, the keys in the outer hash are the heap identifiers: # - # GC.stat_heap.keys # => [0, 1, 2, 3, 4] + # GC.stat_heap.keys # => [0, 1, 2, 3, 4, 5] # # On CRuby, each heap identifier is an integer; # on other implementations, a heap identifier may be a string. @@ -324,9 +333,9 @@ def self.stat hash_or_key = nil # With only argument +heap_id+ given, # returns statistics for the given heap identifier: # - # GC.stat_heap(2) + # GC.stat_heap(3) # # => - # {slot_size: 160, + # {slot_size: 256, # heap_eden_pages: 157, # heap_eden_slots: 64182, # total_allocated_pages: 157, @@ -338,7 +347,7 @@ def self.stat hash_or_key = nil # With arguments +heap_id+ and +key+ given, # returns the value for the given key in the given heap: # - # GC.stat_heap(2, :slot_size) # => 160 + # GC.stat_heap(3, :slot_size) # => 256 # # With arguments +nil+ and +hash+ given, # merges the statistics for all heaps into the given hash: diff --git a/gc/default/default.c b/gc/default/default.c index 1099d6e0dc11e5..3e8d3e16158503 100644 --- a/gc/default/default.c +++ b/gc/default/default.c @@ -187,9 +187,68 @@ static RB_THREAD_LOCAL_SPECIFIER int malloc_increase_local; #define USE_TICK_T (PRINT_ENTER_EXIT_TICK || PRINT_ROOT_TICKS) #ifndef HEAP_COUNT -# define HEAP_COUNT 5 +# if SIZEOF_VALUE >= 8 +# define HEAP_COUNT 7 +# else +# define HEAP_COUNT 6 +# endif #endif +static const unsigned short heap_slot_size_table[HEAP_COUNT] = { +#if SIZEOF_VALUE >= 8 + 32, 48, 64, 128, 256, 512, 1024 +#else + 16, 32, 64, 128, 256, 512 +#endif +}; + +static const size_t heap_init_slots_table[HEAP_COUNT] = { +#if SIZEOF_VALUE >= 8 + /* [0] 32B */ 2000, + /* [1] 48B */ GC_HEAP_INIT_SLOTS, + /* [2] 64B */ GC_HEAP_INIT_SLOTS / 2, + /* [3] 128B */ GC_HEAP_INIT_SLOTS / 4, + /* [4] 256B */ GC_HEAP_INIT_SLOTS / 8, + /* [5] 512B */ GC_HEAP_INIT_SLOTS / 16, + /* [6] 1024B*/ GC_HEAP_INIT_SLOTS / 32, +#else + GC_HEAP_INIT_SLOTS, GC_HEAP_INIT_SLOTS / 2, + GC_HEAP_INIT_SLOTS / 4, GC_HEAP_INIT_SLOTS / 8, + GC_HEAP_INIT_SLOTS / 16, GC_HEAP_INIT_SLOTS / 32, +#endif +}; + +/* Precomputed reciprocals for fast slot index calculation. + * For slot size d: reciprocal = ceil(2^48 / d). + * Then offset / d == (uint32_t)((offset * reciprocal) >> 48) + * for all offset < HEAP_PAGE_SIZE. */ +#define SLOT_RECIPROCAL_SHIFT 48 + +static const uint64_t heap_slot_reciprocal_table[HEAP_COUNT] = { +#if SIZEOF_VALUE >= 8 + /* 32 */ (1ULL << 48) / 32, + /* 48 */ (1ULL << 48) / 48 + 1, + /* 64 */ (1ULL << 48) / 64, + /* 128 */ (1ULL << 48) / 128, + /* 256 */ (1ULL << 48) / 256, + /* 512 */ (1ULL << 48) / 512, + /* 1024*/ (1ULL << 48) / 1024, +#else + /* 16 */ (1ULL << 48) / 16, + /* 32 */ (1ULL << 48) / 32, + /* 64 */ (1ULL << 48) / 64, + /* 128 */ (1ULL << 48) / 128, + /* 256 */ (1ULL << 48) / 256, + /* 512 */ (1ULL << 48) / 512, +#endif +}; + +static inline bool +slot_size_is_power_of_two(unsigned short slot_size) +{ + return (slot_size & (slot_size - 1)) == 0; +} + typedef struct ractor_newobj_heap_cache { struct free_slot *freelist; struct heap_page *using_page; @@ -687,7 +746,12 @@ size_t rb_gc_impl_obj_slot_size(VALUE obj); # endif #endif -#define BASE_SLOT_SIZE (sizeof(struct RBasic) + sizeof(VALUE[RBIMPL_RVALUE_EMBED_LEN_MAX]) + RVALUE_OVERHEAD) +#if SIZEOF_VALUE >= 8 +#define BASE_SLOT_SIZE_LOG2 5 +#else +#define BASE_SLOT_SIZE_LOG2 4 +#endif +#define BASE_SLOT_SIZE (1 << BASE_SLOT_SIZE_LOG2) #ifndef MAX # define MAX(a, b) (((a) > (b)) ? (a) : (b)) @@ -763,8 +827,11 @@ struct free_slot { }; struct heap_page { + /* Cache line 0: allocation fast path + SLOT_INDEX */ + struct free_slot *freelist; + uintptr_t start; + uint64_t slot_size_reciprocal; unsigned short slot_size; - uint32_t slot_div_magic; unsigned short total_slots; unsigned short free_slots; unsigned short final_slots; @@ -779,8 +846,6 @@ struct heap_page { struct heap_page *free_next; struct heap_page_body *body; - uintptr_t start; - struct free_slot *freelist; struct ccan_list_node page_node; bits_t wb_unprotected_bits[HEAP_PAGE_BITMAP_LIMIT]; @@ -841,15 +906,13 @@ heap_page_in_global_empty_pages_pool(rb_objspace_t *objspace, struct heap_page * #define GET_PAGE_HEADER(x) (&GET_PAGE_BODY(x)->header) #define GET_HEAP_PAGE(x) (GET_PAGE_HEADER(x)->page) -static uint32_t slot_div_magics[HEAP_COUNT]; - static inline size_t -slot_index_for_offset(size_t offset, uint32_t div_magic) +slot_index_for_offset(size_t offset, uint64_t reciprocal) { - return (size_t)(((uint64_t)offset * div_magic) >> 32); + return (uint32_t)(((uint64_t)offset * reciprocal) >> SLOT_RECIPROCAL_SHIFT); } -#define SLOT_INDEX(page, p) slot_index_for_offset((uintptr_t)(p) - (page)->start, (page)->slot_div_magic) +#define SLOT_INDEX(page, p) slot_index_for_offset((uintptr_t)(p) - (page)->start, (page)->slot_size_reciprocal) #define SLOT_BITMAP_INDEX(page, p) (SLOT_INDEX(page, p) / BITS_BITLENGTH) #define SLOT_BITMAP_OFFSET(page, p) (SLOT_INDEX(page, p) & (BITS_BITLENGTH - 1)) #define SLOT_BITMAP_BIT(page, p) ((bits_t)1 << SLOT_BITMAP_OFFSET(page, p)) @@ -1636,7 +1699,7 @@ heap_page_add_freeobj(rb_objspace_t *objspace, struct heap_page *page, VALUE obj /* obj should belong to page */ !(page->start <= (uintptr_t)obj && (uintptr_t)obj < ((uintptr_t)page->start + (page->total_slots * page->slot_size)) && - obj % BASE_SLOT_SIZE == 0)) { + obj % sizeof(VALUE) == 0)) { rb_bug("heap_page_add_freeobj: %p is not rvalue.", (void *)obj); } @@ -1977,19 +2040,17 @@ heap_add_page(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *page) GC_ASSERT(!heap->sweeping_page); GC_ASSERT(heap_page_in_global_empty_pages_pool(objspace, page)); - /* Align start to the first slot_size boundary after the page header */ + /* Align start to slot_size boundary */ uintptr_t start = (uintptr_t)page->body + sizeof(struct heap_page_header); - size_t remainder = start % heap->slot_size; - if (remainder != 0) { - start += heap->slot_size - remainder; - } + uintptr_t rem = start % heap->slot_size; + if (rem) start += heap->slot_size - rem; int slot_count = (int)((HEAP_PAGE_SIZE - (start - (uintptr_t)page->body))/heap->slot_size); page->start = start; page->total_slots = slot_count; page->slot_size = heap->slot_size; - page->slot_div_magic = slot_div_magics[heap - heaps]; + page->slot_size_reciprocal = heap_slot_reciprocal_table[heap - heaps]; page->heap = heap; memset(&page->wb_unprotected_bits[0], 0, HEAP_PAGE_BITMAP_SIZE); @@ -2237,13 +2298,7 @@ heap_slot_size(unsigned char pool_id) { GC_ASSERT(pool_id < HEAP_COUNT); - size_t slot_size = (1 << pool_id) * BASE_SLOT_SIZE; - -#if RGENGC_CHECK_MODE - rb_objspace_t *objspace = rb_gc_get_objspace(); - GC_ASSERT(heaps[pool_id].slot_size == (short)slot_size); -#endif - + size_t slot_size = heap_slot_size_table[pool_id]; slot_size -= RVALUE_OVERHEAD; return slot_size; @@ -2356,10 +2411,17 @@ heap_idx_for_size(size_t size) { size += RVALUE_OVERHEAD; - size_t slot_count = CEILDIV(size, BASE_SLOT_SIZE); + if (size <= BASE_SLOT_SIZE) return 0; + if (size <= heap_slot_size_table[1]) return 1; - /* heap_idx is ceil(log2(slot_count)) */ - size_t heap_idx = 64 - nlz_int64(slot_count - 1); +#if SIZEOF_VALUE >= 8 + /* Pools from index 2 onward are powers of two (64, 128, ...), + * so the log2 formula still works — add 1 to skip the 48B pool. */ + size_t heap_idx = 64 - nlz_int64(size - 1) - BASE_SLOT_SIZE_LOG2 + 1; +#else + /* All pools are powers of two on 32-bit. */ + size_t heap_idx = 64 - nlz_int64(size - 1) - BASE_SLOT_SIZE_LOG2; +#endif if (heap_idx >= HEAP_COUNT) { rb_bug("heap_idx_for_size: allocation size too large " @@ -2589,7 +2651,7 @@ is_pointer_to_heap(rb_objspace_t *objspace, const void *ptr) if (p < heap_pages_lomem || p > heap_pages_himem) return FALSE; RB_DEBUG_COUNTER_INC(gc_isptr_range); - if (p % BASE_SLOT_SIZE != 0) return FALSE; + if (p % sizeof(VALUE) != 0) return FALSE; RB_DEBUG_COUNTER_INC(gc_isptr_align); page = heap_page_for_ptr(objspace, (uintptr_t)ptr); @@ -3495,7 +3557,7 @@ gc_sweep_plane(rb_objspace_t *objspace, rb_heap_t *heap, uintptr_t p, bits_t bit do { VALUE vp = (VALUE)p; - GC_ASSERT(vp % BASE_SLOT_SIZE == 0); + GC_ASSERT(vp % sizeof(VALUE) == 0); rb_asan_unpoison_object(vp, false); if (bitset & 1) { @@ -5594,7 +5656,7 @@ gc_compact_plane(rb_objspace_t *objspace, rb_heap_t *heap, uintptr_t p, bits_t b do { VALUE vp = (VALUE)p; - GC_ASSERT(vp % BASE_SLOT_SIZE == 0); + GC_ASSERT(vp % sizeof(VALUE) == 0); if (bitset & 1) { objspace->rcompactor.considered_count_table[BUILTIN_TYPE(vp)]++; @@ -9515,11 +9577,15 @@ rb_gc_impl_objspace_init(void *objspace_ptr) rb_bug("Could not preregister postponed job for GC"); } + /* A standard RVALUE (RBasic + embedded VALUEs + debug overhead) must fit + * in at least one pool. In debug builds RVALUE_OVERHEAD can push this + * beyond the 48-byte pool into the 64-byte pool, which is fine. */ + GC_ASSERT(rb_gc_impl_size_allocatable_p(sizeof(struct RBasic) + sizeof(VALUE[RBIMPL_RVALUE_EMBED_LEN_MAX]))); + for (int i = 0; i < HEAP_COUNT; i++) { rb_heap_t *heap = &heaps[i]; - heap->slot_size = (1 << i) * BASE_SLOT_SIZE; - slot_div_magics[i] = (uint32_t)((uint64_t)UINT32_MAX / heap->slot_size + 1); + heap->slot_size = heap_slot_size_table[i]; ccan_list_head_init(&heap->pages); } @@ -9541,8 +9607,10 @@ rb_gc_impl_objspace_init(void *objspace_ptr) #endif /* Set size pools allocatable pages. */ for (int i = 0; i < HEAP_COUNT; i++) { - /* Set the default value of heap_init_slots. */ - gc_params.heap_init_slots[i] = GC_HEAP_INIT_SLOTS; + /* Set the default value of heap_init_slots using a pre-computed + * table. Pool 1 (48B on 64-bit) gets the most slots since it holds + * the busiest objects; larger pools scale down proportionally. */ + gc_params.heap_init_slots[i] = heap_init_slots_table[i]; } init_mark_stack(&objspace->mark_stack); @@ -9557,6 +9625,13 @@ rb_gc_impl_init(void) VALUE gc_constants = rb_hash_new(); rb_hash_aset(gc_constants, ID2SYM(rb_intern("DEBUG")), GC_DEBUG ? Qtrue : Qfalse); rb_hash_aset(gc_constants, ID2SYM(rb_intern("BASE_SLOT_SIZE")), SIZET2NUM(BASE_SLOT_SIZE - RVALUE_OVERHEAD)); + /* Minimum slot size for a standard RVALUE (RBasic + embedded VALUEs) */ + size_t rvalue_min = sizeof(struct RBasic) + sizeof(VALUE[RBIMPL_RVALUE_EMBED_LEN_MAX]) + RVALUE_OVERHEAD; + size_t rvalue_slot = heap_slot_size_table[0]; + for (int i = 1; i < HEAP_COUNT && rvalue_slot < rvalue_min; i++) { + rvalue_slot = heap_slot_size_table[i]; + } + rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_SIZE")), SIZET2NUM(rvalue_slot - RVALUE_OVERHEAD)); rb_hash_aset(gc_constants, ID2SYM(rb_intern("RBASIC_SIZE")), SIZET2NUM(sizeof(struct RBasic))); rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_OVERHEAD")), SIZET2NUM(RVALUE_OVERHEAD)); rb_hash_aset(gc_constants, ID2SYM(rb_intern("HEAP_PAGE_BITMAP_SIZE")), SIZET2NUM(HEAP_PAGE_BITMAP_SIZE)); diff --git a/gc/mmtk/mmtk.c b/gc/mmtk/mmtk.c index 4832916ce6ea2f..3f6b2223447c65 100644 --- a/gc/mmtk/mmtk.c +++ b/gc/mmtk/mmtk.c @@ -16,6 +16,37 @@ #include #endif +#ifndef VM_CHECK_MODE +# define VM_CHECK_MODE RUBY_DEBUG +#endif + +#ifndef RACTOR_CHECK_MODE +# define RACTOR_CHECK_MODE (VM_CHECK_MODE || RUBY_DEBUG) && (SIZEOF_UINT64_T == SIZEOF_VALUE) +#endif + +#if RACTOR_CHECK_MODE || GC_DEBUG +struct rvalue_overhead { +# if RACTOR_CHECK_MODE + uint32_t _ractor_belonging_id; +# endif +# if GC_DEBUG + const char *file; + int line; +# endif +}; + +# define RVALUE_OVERHEAD (sizeof(struct { \ + union { \ + struct rvalue_overhead overhead; \ + VALUE value; \ + }; \ +})) +#else +# define RVALUE_OVERHEAD 0 +#endif + +static inline size_t mmtk_obj_full_slot_size(VALUE obj); + struct objspace { bool measure_gc_time; bool gc_stress; @@ -334,7 +365,7 @@ rb_mmtk_call_obj_free(MMTk_ObjectReference object) rb_gc_obj_free(objspace, obj); #ifdef MMTK_DEBUG - memset((void *)obj, 0, rb_gc_impl_obj_slot_size(obj)); + memset((void *)obj, 0, mmtk_obj_full_slot_size(obj)); #endif } @@ -618,22 +649,28 @@ void rb_gc_impl_set_params(void *objspace_ptr) { } static VALUE gc_verify_internal_consistency(VALUE self) { return Qnil; } #define MMTK_HEAP_COUNT 6 -#define MMTK_MAX_OBJ_SIZE 640 - +#if SIZEOF_VALUE >= 8 +#define MMTK_MAX_OBJ_SIZE 1024 +static size_t heap_sizes[MMTK_HEAP_COUNT + 1] = { + 32, 64, 128, 256, 512, MMTK_MAX_OBJ_SIZE, 0 +}; +#else +#define MMTK_MAX_OBJ_SIZE 512 static size_t heap_sizes[MMTK_HEAP_COUNT + 1] = { - 32, 40, 80, 160, 320, MMTK_MAX_OBJ_SIZE, 0 + 16, 32, 64, 128, 256, MMTK_MAX_OBJ_SIZE, 0 }; +#endif void rb_gc_impl_init(void) { VALUE gc_constants = rb_hash_new(); - rb_hash_aset(gc_constants, ID2SYM(rb_intern("BASE_SLOT_SIZE")), SIZET2NUM(sizeof(VALUE) * 5)); + rb_hash_aset(gc_constants, ID2SYM(rb_intern("BASE_SLOT_SIZE")), SIZET2NUM(SIZEOF_VALUE >= 8 ? 32 : 16)); + rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_SIZE")), SIZET2NUM((SIZEOF_VALUE >= 8 ? 64 : 32) - RVALUE_OVERHEAD)); rb_hash_aset(gc_constants, ID2SYM(rb_intern("RBASIC_SIZE")), SIZET2NUM(sizeof(struct RBasic))); - rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_OVERHEAD")), INT2NUM(0)); - rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVARGC_MAX_ALLOCATE_SIZE")), LONG2FIX(MMTK_MAX_OBJ_SIZE)); - // Pretend we have 5 size pools - rb_hash_aset(gc_constants, ID2SYM(rb_intern("SIZE_POOL_COUNT")), LONG2FIX(MMTK_HEAP_COUNT)); + rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_OVERHEAD")), SIZET2NUM(RVALUE_OVERHEAD)); + rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVARGC_MAX_ALLOCATE_SIZE")), LONG2FIX(MMTK_MAX_OBJ_SIZE - RVALUE_OVERHEAD)); + rb_hash_aset(gc_constants, ID2SYM(rb_intern("HEAP_COUNT")), LONG2FIX(MMTK_HEAP_COUNT)); // TODO: correctly set RVALUE_OLD_AGE when we have generational GC support rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_OLD_AGE")), INT2FIX(0)); OBJ_FREEZE(gc_constants); @@ -649,10 +686,18 @@ rb_gc_impl_init(void) rb_define_singleton_method(rb_mGC, "verify_compaction_references", rb_f_notimplement, -1); } +static size_t heap_usable_sizes[MMTK_HEAP_COUNT + 1] = { 0 }; + size_t * rb_gc_impl_heap_sizes(void *objspace_ptr) { - return heap_sizes; + if (heap_usable_sizes[0] == 0) { + for (int i = 0; i < MMTK_HEAP_COUNT; i++) { + heap_usable_sizes[i] = heap_sizes[i] - RVALUE_OVERHEAD; + } + } + + return heap_usable_sizes; } int @@ -808,10 +853,11 @@ rb_gc_impl_new_obj(void *objspace_ptr, void *cache_ptr, VALUE klass, VALUE flags struct objspace *objspace = objspace_ptr; struct MMTk_ractor_cache *ractor_cache = cache_ptr; + alloc_size += RVALUE_OVERHEAD; + if (alloc_size > MMTK_MAX_OBJ_SIZE) rb_bug("too big"); for (int i = 0; i < MMTK_HEAP_COUNT; i++) { - if (alloc_size == heap_sizes[i]) break; - if (alloc_size < heap_sizes[i]) { + if (alloc_size <= heap_sizes[i]) { alloc_size = heap_sizes[i]; break; } @@ -839,23 +885,35 @@ rb_gc_impl_new_obj(void *objspace_ptr, void *cache_ptr, VALUE klass, VALUE flags // TODO: only add when object needs obj_free to be called mmtk_add_obj_free_candidate(alloc_obj, obj_can_parallel_free_p((VALUE)alloc_obj)); +#if RACTOR_CHECK_MODE + void rb_ractor_setup_belonging(VALUE obj); + rb_ractor_setup_belonging((VALUE)alloc_obj); +#endif + objspace->total_allocated_objects++; return (VALUE)alloc_obj; } +static inline size_t +mmtk_obj_full_slot_size(VALUE obj) +{ + return ((VALUE *)obj)[-1]; +} + size_t rb_gc_impl_obj_slot_size(VALUE obj) { - return ((VALUE *)obj)[-1]; + return mmtk_obj_full_slot_size(obj) - RVALUE_OVERHEAD; } size_t rb_gc_impl_heap_id_for_size(void *objspace_ptr, size_t size) { + size += RVALUE_OVERHEAD; + for (int i = 0; i < MMTK_HEAP_COUNT; i++) { - if (size == heap_sizes[i]) return i; - if (size < heap_sizes[i]) return i; + if (size <= heap_sizes[i]) return i; } rb_bug("size too big"); @@ -864,7 +922,7 @@ rb_gc_impl_heap_id_for_size(void *objspace_ptr, size_t size) bool rb_gc_impl_size_allocatable_p(size_t size) { - return size <= MMTK_MAX_OBJ_SIZE; + return size <= MMTK_MAX_OBJ_SIZE - RVALUE_OVERHEAD; } // Malloc @@ -1067,7 +1125,7 @@ rb_gc_impl_each_objects_i(VALUE obj, void *d) { struct rb_gc_impl_each_objects_data *data = d; - size_t slot_size = rb_gc_impl_obj_slot_size(obj); + size_t slot_size = mmtk_obj_full_slot_size(obj); return data->func((void *)obj, (void *)(obj + slot_size), slot_size, data->data); } diff --git a/internal/class.h b/internal/class.h index ea68b07fc20968..5dabee3150a50f 100644 --- a/internal/class.h +++ b/internal/class.h @@ -149,9 +149,9 @@ struct RClass_and_rb_classext_t { }; #if SIZEOF_VALUE >= SIZEOF_LONG_LONG -// Assert that classes can be embedded in heaps[2] (which has 160B slot size) +// Assert that classes can be embedded in heaps[3] (256B slot size on 64-bit). // On 32bit platforms there is no variable width allocation so it doesn't matter. -STATIC_ASSERT(sizeof_rb_classext_t, sizeof(struct RClass_and_rb_classext_t) <= 4 * RVALUE_SIZE); +STATIC_ASSERT(sizeof_rb_classext_t, sizeof(struct RClass_and_rb_classext_t) <= 256); #endif struct RClass_boxable { diff --git a/object.c b/object.c index 4dcd5d615f85a9..d3036c52f0a7c0 100644 --- a/object.c +++ b/object.c @@ -93,6 +93,7 @@ static ID id_instance_variables_to_inspect; size_t rb_obj_embedded_size(uint32_t fields_count) { + if (fields_count < 1) fields_count = 1; return offsetof(struct RObject, as.ary) + (sizeof(VALUE) * fields_count); } diff --git a/shape.c b/shape.c index 90036722f10026..93ccd3eb595fdd 100644 --- a/shape.c +++ b/shape.c @@ -477,14 +477,14 @@ static attr_index_t shape_grow_capa(attr_index_t current_capa) { const attr_index_t *capacities = rb_shape_tree.capacities; + size_t heaps_count = rb_shape_tree.heaps_count; // First try to use the next size that will be embeddable in a larger object slot. - attr_index_t capa; - while ((capa = *capacities)) { + for (size_t i = 0; i < heaps_count; i++) { + attr_index_t capa = capacities[i]; if (capa > current_capa) { return capa; } - capacities++; } return (attr_index_t)rb_malloc_grow_capa(current_capa, sizeof(VALUE)); @@ -1543,8 +1543,14 @@ Init_default_shapes(void) capacities[heaps_count] = 0; size_t index; for (index = 0; index < heaps_count; index++) { - capacities[index] = (heap_sizes[index] - sizeof(struct RBasic)) / sizeof(VALUE); + if (heap_sizes[index] > sizeof(struct RBasic)) { + capacities[index] = (heap_sizes[index] - sizeof(struct RBasic)) / sizeof(VALUE); + } + else { + capacities[index] = 0; + } } + rb_shape_tree.heaps_count = heaps_count; rb_shape_tree.capacities = capacities; #ifdef HAVE_MMAP diff --git a/shape.h b/shape.h index 96c78f2bc1a356..1296e62a1a4d7f 100644 --- a/shape.h +++ b/shape.h @@ -115,6 +115,7 @@ typedef struct { rb_shape_t *shape_list; rb_shape_t *root_shape; const attr_index_t *capacities; + size_t heaps_count; rb_atomic_t next_shape_id; redblack_node_t *shape_cache; diff --git a/test/-ext-/string/test_capacity.rb b/test/-ext-/string/test_capacity.rb index df000f7cdb8103..80a2d5db91cd45 100644 --- a/test/-ext-/string/test_capacity.rb +++ b/test/-ext-/string/test_capacity.rb @@ -5,13 +5,13 @@ class Test_StringCapacity < Test::Unit::TestCase def test_capacity_embedded - assert_equal GC::INTERNAL_CONSTANTS[:BASE_SLOT_SIZE] - embed_header_size - 1, capa('foo') + assert_equal GC::INTERNAL_CONSTANTS[:RVALUE_SIZE] - embed_header_size - 1, capa('foo') assert_equal max_embed_len, capa('1' * max_embed_len) assert_equal max_embed_len, capa('1' * (max_embed_len - 1)) end def test_capacity_shared - sym = ("a" * GC::INTERNAL_CONSTANTS[:BASE_SLOT_SIZE]).to_sym + sym = ("a" * GC::INTERNAL_CONSTANTS[:RVALUE_SIZE]).to_sym assert_equal 0, capa(sym.to_s) end diff --git a/test/-ext-/string/test_set_len.rb b/test/-ext-/string/test_set_len.rb index 1531d76167c35c..a18bbbc70cd237 100644 --- a/test/-ext-/string/test_set_len.rb +++ b/test/-ext-/string/test_set_len.rb @@ -5,7 +5,7 @@ class Test_StrSetLen < Test::Unit::TestCase def setup # Make string long enough so that it is not embedded - @range_end = ("0".ord + GC::INTERNAL_CONSTANTS[:BASE_SLOT_SIZE]).chr + @range_end = ("0".ord + GC::INTERNAL_CONSTANTS[:RVALUE_SIZE]).chr @s0 = [*"0"..@range_end].join("").freeze @s1 = Bug::String.new(@s0) end diff --git a/test/.excludes-mmtk/TestObjSpace.rb b/test/.excludes-mmtk/TestObjSpace.rb index 703efd79bb8c69..94eb2c436d4435 100644 --- a/test/.excludes-mmtk/TestObjSpace.rb +++ b/test/.excludes-mmtk/TestObjSpace.rb @@ -1,5 +1,4 @@ exclude(:test_dump_all_full, "testing behaviour specific to default GC") exclude(:test_dump_flag_age, "testing behaviour specific to default GC") exclude(:test_dump_flags, "testing behaviour specific to default GC") -exclude(:test_dump_includes_slot_size, "can be removed when BASE_SLOT_SIZE is 32 bytes") exclude(:test_dump_objects_dumps_page_slot_sizes, "testing behaviour specific to default GC") diff --git a/test/objspace/test_objspace.rb b/test/objspace/test_objspace.rb index d631f97d1bcad8..39cb6d6734e501 100644 --- a/test/objspace/test_objspace.rb +++ b/test/objspace/test_objspace.rb @@ -33,7 +33,7 @@ def test_memsize_of_root_shared_string b = a.dup c = nil ObjectSpace.each_object(String) {|x| break c = x if a == x and x.frozen?} - rv_size = GC::INTERNAL_CONSTANTS[:BASE_SLOT_SIZE] + rv_size = GC::INTERNAL_CONSTANTS[:RVALUE_SIZE] assert_equal([rv_size, rv_size, a.length + 1 + rv_size], [a, b, c].map {|x| ObjectSpace.memsize_of(x)}) end @@ -473,12 +473,12 @@ def test_dump_object assert_include(info, '"embedded":true') assert_include(info, '"ivars":0') - # Non-embed object + # Non-embed object (needs > 6 ivars to exceed pool 0 embed capacity) obj = klass.new - 5.times { |i| obj.instance_variable_set("@ivar#{i}", 0) } + 7.times { |i| obj.instance_variable_set("@ivar#{i}", 0) } info = ObjectSpace.dump(obj) assert_not_include(info, '"embedded":true') - assert_include(info, '"ivars":5') + assert_include(info, '"ivars":7') end def test_dump_control_char @@ -648,7 +648,8 @@ def dump_my_heap_please next if obj["type"] == "SHAPE" assert_not_nil obj["slot_size"] - assert_equal 0, obj["slot_size"] % (GC::INTERNAL_CONSTANTS[:BASE_SLOT_SIZE] + GC::INTERNAL_CONSTANTS[:RVALUE_OVERHEAD]) + slot_sizes = GC::INTERNAL_CONSTANTS[:HEAP_COUNT].times.map { |i| GC.stat_heap(i, :slot_size) } + assert_include slot_sizes, obj["slot_size"] } end end diff --git a/test/ruby/test_file_exhaustive.rb b/test/ruby/test_file_exhaustive.rb index be9e6bd44e702d..3f9a71253331e7 100644 --- a/test/ruby/test_file_exhaustive.rb +++ b/test/ruby/test_file_exhaustive.rb @@ -897,10 +897,10 @@ def test_expand_path_memsize bug9934 = '[ruby-core:63114] [Bug #9934]' require "objspace" path = File.expand_path("/foo") - assert_operator(ObjectSpace.memsize_of(path), :<=, path.bytesize + GC::INTERNAL_CONSTANTS[:BASE_SLOT_SIZE], bug9934) + assert_operator(ObjectSpace.memsize_of(path), :<=, path.bytesize + GC::INTERNAL_CONSTANTS[:RVALUE_SIZE], bug9934) path = File.expand_path("/a"*25) assert_operator(ObjectSpace.memsize_of(path), :<=, - (path.bytesize + 1) * 2 + GC::INTERNAL_CONSTANTS[:BASE_SLOT_SIZE], bug9934) + (path.bytesize + 1) * 2 + GC::INTERNAL_CONSTANTS[:RVALUE_SIZE], bug9934) end def test_expand_path_encoding diff --git a/test/ruby/test_gc.rb b/test/ruby/test_gc.rb index 60f04f8e10cf11..8ed4c300e46848 100644 --- a/test/ruby/test_gc.rb +++ b/test/ruby/test_gc.rb @@ -230,7 +230,7 @@ def test_stat_heap GC.stat(stat) end - assert_equal (GC::INTERNAL_CONSTANTS[:BASE_SLOT_SIZE] + GC::INTERNAL_CONSTANTS[:RVALUE_OVERHEAD]) * (2**i), stat_heap[:slot_size] + assert_equal GC.stat_heap(i, :slot_size), stat_heap[:slot_size] assert_operator stat_heap[:heap_live_slots], :<=, stat[:heap_live_slots] assert_operator stat_heap[:heap_free_slots], :<=, stat[:heap_free_slots] assert_operator stat_heap[:heap_final_slots], :<=, stat[:heap_final_slots] diff --git a/test/ruby/test_gc_compact.rb b/test/ruby/test_gc_compact.rb index f3da8e4e138432..8f22b49a20a019 100644 --- a/test/ruby/test_gc_compact.rb +++ b/test/ruby/test_gc_compact.rb @@ -207,7 +207,7 @@ def test_updating_references_for_heap_allocated_shared_arrays end def test_updating_references_for_embed_shared_arrays - omit if GC::INTERNAL_CONSTANTS[:SIZE_POOL_COUNT] == 1 + omit if GC::INTERNAL_CONSTANTS[:HEAP_COUNT] == 1 assert_separately(%w[-robjspace], "#{<<~"begin;"}\n#{<<~"end;"}", timeout: 10) begin; @@ -256,7 +256,7 @@ def test_updating_references_for_heap_allocated_frozen_shared_arrays end def test_updating_references_for_embed_frozen_shared_arrays - omit if GC::INTERNAL_CONSTANTS[:SIZE_POOL_COUNT] == 1 + omit if GC::INTERNAL_CONSTANTS[:HEAP_COUNT] == 1 assert_separately(%w[-robjspace], "#{<<~"begin;"}\n#{<<~"end;"}", timeout: 10) begin; @@ -284,7 +284,7 @@ def test_updating_references_for_embed_frozen_shared_arrays end def test_moving_arrays_down_heaps - omit if GC::INTERNAL_CONSTANTS[:SIZE_POOL_COUNT] == 1 + omit if GC::INTERNAL_CONSTANTS[:HEAP_COUNT] == 1 assert_separately(%w[-robjspace], "#{<<~"begin;"}\n#{<<~"end;"}", timeout: 10) begin; @@ -306,7 +306,7 @@ def test_moving_arrays_down_heaps end def test_moving_arrays_up_heaps - omit if GC::INTERNAL_CONSTANTS[:SIZE_POOL_COUNT] == 1 + omit if GC::INTERNAL_CONSTANTS[:HEAP_COUNT] == 1 assert_separately(%w[-robjspace], "#{<<~"begin;"}\n#{<<~"end;"}", timeout: 10) begin; @@ -315,7 +315,7 @@ def test_moving_arrays_up_heaps GC.verify_compaction_references(expand_heap: true, toward: :empty) Fiber.new { - ary = "hello".chars + ary = "hello world".chars # > 6 elements to exceed pool 0 embed capacity $arys = ARY_COUNT.times.map do x = [] ary.each { |e| x << e } @@ -330,7 +330,7 @@ def test_moving_arrays_up_heaps end def test_moving_objects_between_heaps - omit if GC::INTERNAL_CONSTANTS[:SIZE_POOL_COUNT] == 1 + omit if GC::INTERNAL_CONSTANTS[:HEAP_COUNT] == 1 assert_separately(%w[-robjspace], "#{<<~"begin;"}\n#{<<~"end;"}", timeout: 60) begin; @@ -362,7 +362,7 @@ def add_ivars end def test_compact_objects_of_varying_sizes - omit if GC::INTERNAL_CONSTANTS[:SIZE_POOL_COUNT] == 1 + omit if GC::INTERNAL_CONSTANTS[:HEAP_COUNT] == 1 assert_ruby_status([], "#{<<~"begin;"}\n#{<<~"end;"}", timeout: 10) begin; @@ -378,7 +378,7 @@ def test_compact_objects_of_varying_sizes end def test_moving_strings_up_heaps - omit if GC::INTERNAL_CONSTANTS[:SIZE_POOL_COUNT] == 1 + omit if GC::INTERNAL_CONSTANTS[:HEAP_COUNT] == 1 assert_separately(%w[-robjspace], "#{<<~"begin;"}\n#{<<~"end;"}", timeout: 30) begin; @@ -399,7 +399,7 @@ def test_moving_strings_up_heaps end def test_moving_strings_down_heaps - omit if GC::INTERNAL_CONSTANTS[:SIZE_POOL_COUNT] == 1 + omit if GC::INTERNAL_CONSTANTS[:HEAP_COUNT] == 1 assert_separately(%w[-robjspace], "#{<<~"begin;"}\n#{<<~"end;"}", timeout: 30) begin; @@ -419,7 +419,7 @@ def test_moving_strings_down_heaps end def test_moving_hashes_down_heaps - omit if GC::INTERNAL_CONSTANTS[:SIZE_POOL_COUNT] == 1 + omit if GC::INTERNAL_CONSTANTS[:HEAP_COUNT] == 1 # AR and ST hashes are in the same size pool on 32 bit omit unless RbConfig::SIZEOF["uint64_t"] <= RbConfig::SIZEOF["void*"] diff --git a/test/ruby/test_object.rb b/test/ruby/test_object.rb index 2f340788be43b2..135296447f6ce7 100644 --- a/test/ruby/test_object.rb +++ b/test/ruby/test_object.rb @@ -358,38 +358,40 @@ def test_remove_instance_variable def test_remove_instance_variable_re_embed assert_separately(%w[-robjspace], "#{<<~"begin;"}\n#{<<~'end;'}") begin; - c = Class.new do - attr_reader :a, :b, :c + # Determine the RVALUE pool's embed capacity from GC constants. + rvalue_size = GC::INTERNAL_CONSTANTS[:RVALUE_SIZE] + rbasic_size = GC::INTERNAL_CONSTANTS[:RBASIC_SIZE] + embed_cap = (rvalue_size - rbasic_size) / RbConfig::SIZEOF["void*"] - def initialize - @a = nil - @b = nil - @c = nil - end - end + # Build a class whose initialize sets embed_cap ivars so objects + # are allocated in the RVALUE pool with embedded storage. + init_body = embed_cap.times.map { |i| "@v#{i} = nil" }.join("; ") + c = Class.new { class_eval("def initialize; #{init_body}; end") } o1 = c.new o2 = c.new - o1.instance_variable_set(:@foo, 5) - o1.instance_variable_set(:@a, 0) - o1.instance_variable_set(:@b, 1) - o1.instance_variable_set(:@c, 2) + # All embed_cap ivars fit - should be embedded + embed_cap.times { |i| o1.instance_variable_set(:"@v#{i}", i) } + assert_includes ObjectSpace.dump(o1), '"embedded":true' + + # One more ivar overflows embed capacity + o1.instance_variable_set(:@overflow, 99) refute_includes ObjectSpace.dump(o1), '"embedded":true' - o1.remove_instance_variable(:@foo) + + # Remove the overflow ivar - should re-embed + o1.remove_instance_variable(:@overflow) assert_includes ObjectSpace.dump(o1), '"embedded":true' - o2.instance_variable_set(:@a, 0) - o2.instance_variable_set(:@b, 1) - o2.instance_variable_set(:@c, 2) + # An object that never overflowed is also embedded + embed_cap.times { |i| o2.instance_variable_set(:"@v#{i}", i) } assert_includes ObjectSpace.dump(o2), '"embedded":true' - assert_equal(0, o1.a) - assert_equal(1, o1.b) - assert_equal(2, o1.c) - assert_equal(0, o2.a) - assert_equal(1, o2.b) - assert_equal(2, o2.c) + # Verify values survived re-embedding + embed_cap.times do |i| + assert_equal(i, o1.instance_variable_get(:"@v#{i}")) + assert_equal(i, o2.instance_variable_get(:"@v#{i}")) + end end; end diff --git a/test/ruby/test_time.rb b/test/ruby/test_time.rb index 333edb80218a64..277f30243b47a1 100644 --- a/test/ruby/test_time.rb +++ b/test/ruby/test_time.rb @@ -1421,7 +1421,7 @@ def test_memsize # Time objects are common in some code, try to keep them small omit "Time object size test" if /^(?:i.?86|x86_64)-linux/ !~ RUBY_PLATFORM omit "GC is in debug" if GC::INTERNAL_CONSTANTS[:RVALUE_OVERHEAD] > 0 - omit "memsize is not accurate due to using malloc_usable_size" if GC::INTERNAL_CONSTANTS[:SIZE_POOL_COUNT] == 1 + omit "memsize is not accurate due to using malloc_usable_size" if GC::INTERNAL_CONSTANTS[:HEAP_COUNT] == 1 omit "Only run this test on 64-bit" if RbConfig::SIZEOF["void*"] != 8 require 'objspace' @@ -1433,7 +1433,10 @@ def test_memsize RbConfig::SIZEOF["void*"] # Same size as VALUE end sizeof_vtm = RbConfig::SIZEOF["void*"] * 4 + 8 - expect = GC::INTERNAL_CONSTANTS[:BASE_SLOT_SIZE] + sizeof_timew + sizeof_vtm + data_size = GC::INTERNAL_CONSTANTS[:BASE_SLOT_SIZE] + sizeof_timew + sizeof_vtm + # Round up to the smallest slot size that fits + expect = GC::INTERNAL_CONSTANTS[:BASE_SLOT_SIZE] + expect <<= 1 while expect < data_size assert_operator ObjectSpace.memsize_of(t), :<=, expect rescue LoadError => e omit "failed to load objspace: #{e.message}" diff --git a/zjit/src/hir/opt_tests.rs b/zjit/src/hir/opt_tests.rs index e58e61f3c4e182..c6953b5cd2eec8 100644 --- a/zjit/src/hir/opt_tests.rs +++ b/zjit/src/hir/opt_tests.rs @@ -5205,12 +5205,8 @@ mod hir_opt_tests { v14:HeapBasicObject = RefineType v6, HeapBasicObject v17:Fixnum[2] = Const Value(2) PatchPoint SingleRactorMode - v36:CShape = LoadField v14, :_shape_id@0x1000 - v37:CShape[0x1003] = GuardBitEquals v36, CShape(0x1003) - StoreField v14, :@bar@0x1004, v17 - WriteBarrier v14, v17 - v40:CShape[0x1005] = Const CShape(0x1005) - StoreField v14, :_shape_id@0x1000, v40 + IncrCounter setivar_fallback_new_shape_needs_extension + SetIvar v14, :@bar, v17 CheckInterrupts Return v17 "); @@ -13368,21 +13364,18 @@ mod hir_opt_tests { v14:HeapBasicObject = RefineType v6, HeapBasicObject v17:Fixnum[2] = Const Value(2) PatchPoint SingleRactorMode - v50:CShape = LoadField v14, :_shape_id@0x1000 - v51:CShape[0x1003] = GuardBitEquals v50, CShape(0x1003) - StoreField v14, :@b@0x1004, v17 - WriteBarrier v14, v17 - v54:CShape[0x1005] = Const CShape(0x1005) - StoreField v14, :_shape_id@0x1000, v54 + IncrCounter setivar_fallback_new_shape_needs_extension + SetIvar v14, :@b, v17 v21:HeapBasicObject = RefineType v14, HeapBasicObject v24:Fixnum[3] = Const Value(3) PatchPoint SingleRactorMode - v57:CShape = LoadField v21, :_shape_id@0x1000 - v58:CShape[0x1005] = GuardBitEquals v57, CShape(0x1005) - StoreField v21, :@c@0x1006, v24 + v51:CShape = LoadField v21, :_shape_id@0x1000 + v52:CShape[0x1004] = GuardBitEquals v51, CShape(0x1004) + v53:CPtr = LoadField v21, :_as_heap@0x1002 + StoreField v53, :@c@0x1002, v24 WriteBarrier v21, v24 - v61:CShape[0x1007] = Const CShape(0x1007) - StoreField v21, :_shape_id@0x1000, v61 + v56:CShape[0x1005] = Const CShape(0x1005) + StoreField v21, :_shape_id@0x1000, v56 v28:HeapBasicObject = RefineType v21, HeapBasicObject v31:Fixnum[4] = Const Value(4) PatchPoint SingleRactorMode