diff --git a/Cargo.lock b/Cargo.lock index ceffbbe3e462fc..2108276e7213b3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -186,9 +186,9 @@ dependencies = [ [[package]] name = "insta" -version = "1.47.1" +version = "1.47.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99322078b2c076829a1db959d49da554fabc4342257fc0ba5a070a1eb3a01cd8" +checksum = "7b4a6248eb93a4401ed2f37dfe8ea592d3cf05b7cf4f8efa867b6895af7e094e" dependencies = [ "console", "once_cell", diff --git a/class.c b/class.c index 9cf5e4374216ac..70563c3094a00e 100644 --- a/class.c +++ b/class.c @@ -868,10 +868,7 @@ rb_class_new(VALUE super) rb_check_inheritable(super); VALUE klass = rb_class_boot(super); - if (super != rb_cObject && super != rb_cBasicObject) { - RCLASS_SET_MAX_IV_COUNT(klass, RCLASS_MAX_IV_COUNT(super)); - } - + RCLASS_SET_MAX_IV_COUNT(klass, RCLASS_MAX_IV_COUNT(super)); RUBY_ASSERT(getenv("RUBY_BOX") || RCLASS_PRIME_CLASSEXT_WRITABLE_P(klass)); return klass; @@ -1418,8 +1415,10 @@ Init_class_hierarchy(void) rb_cBasicObject = boot_defclass("BasicObject", 0); RCLASS_SET_ALLOCATOR(rb_cBasicObject, rb_class_allocate_instance); FL_SET_RAW(rb_cBasicObject, RCLASS_ALLOCATOR_DEFINED); + RCLASS_SET_EXPECT_NO_IVAR(rb_cBasicObject); + rb_cObject = boot_defclass("Object", rb_cBasicObject); - rb_vm_register_global_object(rb_cObject); + RCLASS_SET_EXPECT_NO_IVAR(rb_cObject); /* resolve class name ASAP for order-independence */ rb_set_class_path_string(rb_cObject, rb_cObject, rb_fstring_lit("Object")); diff --git a/doc/_regexp.rdoc b/doc/_regexp.rdoc index 83b80366bfa987..4ad6118ddda057 100644 --- a/doc/_regexp.rdoc +++ b/doc/_regexp.rdoc @@ -1128,6 +1128,13 @@ Regexp in extended mode: re = /#{pattern}/x re.match('MCMXLIII') # => # +Comments in regexp literals cannot include unescaped terminator +characters: + + / + foo # the following slash \/ must be escaped + /x + === Interpolation Mode Modifier +o+ means that the first time a literal regexp with interpolations diff --git a/gc.c b/gc.c index 0784010d6834dc..2d7ceba9b98796 100644 --- a/gc.c +++ b/gc.c @@ -3480,17 +3480,6 @@ rb_gc_mark_children(void *objspace, VALUE obj) gc_mark_internal(ptr[i]); } } - - attr_index_t fields_count = (attr_index_t)len; - if (fields_count) { - VALUE klass = RBASIC_CLASS(obj); - - // Increment max_iv_count if applicable, used to determine size pool allocation - if (RCLASS_MAX_IV_COUNT(klass) < fields_count) { - RCLASS_SET_MAX_IV_COUNT(klass, fields_count); - } - } - break; } diff --git a/imemo.c b/imemo.c index 5895045a8a3137..50d96595adb81d 100644 --- a/imemo.c +++ b/imemo.c @@ -123,44 +123,34 @@ rb_imemo_memo_new_value(VALUE a, VALUE b, VALUE c) return memo; } -static VALUE -imemo_fields_new(VALUE owner, size_t capa, bool shareable) +VALUE +rb_imemo_fields_new(VALUE owner, shape_id_t shape_id, bool shareable) { + size_t capa = RSHAPE_CAPACITY(shape_id); size_t embedded_size = offsetof(struct rb_fields, as.embed) + capa * sizeof(VALUE); + VALUE fields; if (rb_gc_size_allocatable_p(embedded_size)) { - VALUE fields = rb_imemo_new(imemo_fields, owner, embedded_size, shareable); - RUBY_ASSERT(IMEMO_TYPE_P(fields, imemo_fields)); - return fields; + fields = rb_imemo_new(imemo_fields, owner, embedded_size, shareable); } else { - VALUE fields = rb_imemo_new(imemo_fields, owner, sizeof(struct rb_fields), shareable); + fields = rb_imemo_new(imemo_fields, owner, sizeof(struct rb_fields), shareable); IMEMO_OBJ_FIELDS(fields)->as.external.ptr = ALLOC_N(VALUE, capa); FL_SET_RAW(fields, OBJ_FIELD_HEAP); - return fields; } + RBASIC_SET_SHAPE_ID(fields, shape_id); + return fields; } VALUE -rb_imemo_fields_new(VALUE owner, size_t capa, bool shareable) -{ - return imemo_fields_new(owner, capa, shareable); -} - -static VALUE -imemo_fields_new_complex(VALUE owner, size_t capa, bool shareable) +rb_imemo_fields_new_complex(VALUE owner, shape_id_t shape_id, size_t capa, bool shareable) { VALUE fields = rb_imemo_new(imemo_fields, owner, sizeof(struct rb_fields), shareable); IMEMO_OBJ_FIELDS(fields)->as.complex.table = st_init_numtable_with_size(capa); FL_SET_RAW(fields, OBJ_FIELD_HEAP); + RBASIC_SET_SHAPE_ID(fields, shape_id); return fields; } -VALUE -rb_imemo_fields_new_complex(VALUE owner, size_t capa, bool shareable) -{ - return imemo_fields_new_complex(owner, capa, shareable); -} - static int imemo_fields_trigger_wb_i(st_data_t key, st_data_t value, st_data_t arg) { @@ -177,11 +167,12 @@ imemo_fields_complex_wb_i(st_data_t key, st_data_t value, st_data_t arg) } VALUE -rb_imemo_fields_new_complex_tbl(VALUE owner, st_table *tbl, bool shareable) +rb_imemo_fields_new_complex_tbl(VALUE owner, shape_id_t shape_id, st_table *tbl, bool shareable) { VALUE fields = rb_imemo_new(imemo_fields, owner, sizeof(struct rb_fields), shareable); IMEMO_OBJ_FIELDS(fields)->as.complex.table = tbl; FL_SET_RAW(fields, OBJ_FIELD_HEAP); + RBASIC_SET_SHAPE_ID(fields, shape_id); st_foreach(tbl, imemo_fields_trigger_wb_i, (st_data_t)fields); return fields; } @@ -196,16 +187,14 @@ rb_imemo_fields_clone(VALUE fields_obj) st_table *src_table = rb_imemo_fields_complex_tbl(fields_obj); st_table *dest_table = xcalloc(1, sizeof(st_table)); - clone = rb_imemo_fields_new_complex_tbl(rb_imemo_fields_owner(fields_obj), dest_table, false /* TODO: check */); + clone = rb_imemo_fields_new_complex_tbl(rb_imemo_fields_owner(fields_obj), shape_id, dest_table, false /* TODO: check */); st_replace(dest_table, src_table); - RBASIC_SET_SHAPE_ID(clone, shape_id); st_foreach(dest_table, imemo_fields_complex_wb_i, (st_data_t)clone); } else { - clone = imemo_fields_new(rb_imemo_fields_owner(fields_obj), RSHAPE_CAPACITY(shape_id), false /* TODO: check */); - RBASIC_SET_SHAPE_ID(clone, shape_id); + clone = rb_imemo_fields_new(rb_imemo_fields_owner(fields_obj), shape_id, false /* TODO: check */); VALUE *fields = rb_imemo_fields_ptr(clone); attr_index_t fields_count = RSHAPE_LEN(shape_id); MEMCPY(fields, rb_imemo_fields_ptr(fields_obj), VALUE, fields_count); diff --git a/internal/class.h b/internal/class.h index 0ffb6b5354f09b..584c30aafbf259 100644 --- a/internal/class.h +++ b/internal/class.h @@ -91,6 +91,7 @@ struct rb_classext_struct { bool iclass_is_origin : 1; bool iclass_origin_shared_mtbl : 1; bool superclasses_with_self : 1; + bool expect_no_ivar : 1; VALUE classpath; }; typedef struct rb_classext_struct rb_classext_t; @@ -514,7 +515,7 @@ RCLASS_WRITABLE_ENSURE_FIELDS_OBJ(VALUE obj) RUBY_ASSERT(RB_TYPE_P(obj, RUBY_T_CLASS) || RB_TYPE_P(obj, RUBY_T_MODULE)); rb_classext_t *ext = RCLASS_EXT_WRITABLE(obj); if (!ext->fields_obj) { - RB_OBJ_WRITE(obj, &ext->fields_obj, rb_imemo_fields_new(obj, 1, true)); + RB_OBJ_WRITE(obj, &ext->fields_obj, rb_imemo_fields_new(obj, ROOT_SHAPE_ID, true)); } return ext->fields_obj; } @@ -731,9 +732,24 @@ RCLASS_SET_ATTACHED_OBJECT(VALUE klass, VALUE attached_object) static inline void RCLASS_SET_MAX_IV_COUNT(VALUE klass, attr_index_t count) { + RUBY_ASSERT(klass != rb_cObject); + RUBY_ASSERT(klass != rb_cBasicObject); + RCLASS_MAX_IV_COUNT(klass) = count; } +static inline void +RCLASS_SET_EXPECT_NO_IVAR(VALUE klass) +{ + RCLASS_EXT_PRIME(klass)->expect_no_ivar = true; +} + +static inline bool +RCLASS_EXPECT_NO_IVAR(VALUE klass) +{ + return RCLASS_EXT_PRIME(klass)->expect_no_ivar; +} + static inline void RCLASS_SET_CLONED(VALUE klass, bool cloned) { diff --git a/internal/imemo.h b/internal/imemo.h index 0c31d7c563b625..23e76875c7d889 100644 --- a/internal/imemo.h +++ b/internal/imemo.h @@ -249,9 +249,9 @@ STATIC_ASSERT(imemo_fields_complex_offset, offsetof(struct RObject, as.heap.fiel #define IMEMO_OBJ_FIELDS(fields) ((struct rb_fields *)fields) -VALUE rb_imemo_fields_new(VALUE owner, size_t capa, bool shareable); -VALUE rb_imemo_fields_new_complex(VALUE owner, size_t capa, bool shareable); -VALUE rb_imemo_fields_new_complex_tbl(VALUE owner, st_table *tbl, bool shareable); +VALUE rb_imemo_fields_new(VALUE owner, /* shape_id_t */ uint32_t shape_id, bool shareable); +VALUE rb_imemo_fields_new_complex(VALUE owner, /* shape_id_t */ uint32_t shape_id, size_t capa, bool shareable); +VALUE rb_imemo_fields_new_complex_tbl(VALUE owner, /* shape_id_t */ uint32_t shape_id, st_table *tbl, bool shareable); VALUE rb_imemo_fields_clone(VALUE fields_obj); void rb_imemo_fields_clear(VALUE fields_obj); diff --git a/object.c b/object.c index eaec5e89d28f97..23a83c572ede79 100644 --- a/object.c +++ b/object.c @@ -2224,6 +2224,7 @@ rb_class_initialize(int argc, VALUE *argv, VALUE klass) } } rb_class_set_super(klass, super); + RCLASS_SET_MAX_IV_COUNT(klass, RCLASS_MAX_IV_COUNT(super)); RCLASS_SET_ALLOCATOR(klass, RCLASS_ALLOCATOR(super)); rb_make_metaclass(klass, RBASIC(super)->klass); rb_class_inherited(super, klass); diff --git a/shape.c b/shape.c index c9169313cffe96..70999f985e3238 100644 --- a/shape.c +++ b/shape.c @@ -35,10 +35,29 @@ static ID id_object_id; +// Should be on its own cache line +static RUBY_ALIGNAS(128) rb_atomic_t redblack_cache_size; + +struct redblack_node { + ID key; + rb_shape_t *value; + redblack_id_t l; + redblack_id_t r; +}; +typedef struct redblack_node redblack_node_t; + +static redblack_node_t *redblack_cache; + #define LEAF 0 #define BLACK 0x0 #define RED 0x1 +static inline redblack_node_t * +redblack_node(redblack_id_t id) +{ + return id ? &redblack_cache[id - 1] : LEAF; +} + static redblack_node_t * redblack_left(redblack_node_t *node) { @@ -46,8 +65,8 @@ redblack_left(redblack_node_t *node) return LEAF; } else { - RUBY_ASSERT(node->l < rb_shape_tree.cache_size); - redblack_node_t *left = &rb_shape_tree.shape_cache[node->l - 1]; + RUBY_ASSERT(node->l < redblack_cache_size); + redblack_node_t *left = redblack_node(node->l); return left; } } @@ -59,14 +78,14 @@ redblack_right(redblack_node_t *node) return LEAF; } else { - RUBY_ASSERT(node->r < rb_shape_tree.cache_size); - redblack_node_t *right = &rb_shape_tree.shape_cache[node->r - 1]; + RUBY_ASSERT(node->r < redblack_cache_size); + redblack_node_t *right = redblack_node(node->r); return right; } } static redblack_node_t * -redblack_find(redblack_node_t *tree, ID key) +redblack_find0(redblack_node_t *tree, ID key) { if (tree == LEAF) { return LEAF; @@ -80,15 +99,21 @@ redblack_find(redblack_node_t *tree, ID key) } else { if (key < tree->key) { - return redblack_find(redblack_left(tree), key); + return redblack_find0(redblack_left(tree), key); } else { - return redblack_find(redblack_right(tree), key); + return redblack_find0(redblack_right(tree), key); } } } } +static redblack_node_t * +redblack_find(redblack_id_t tree_id, ID key) +{ + return redblack_find0(redblack_node(tree_id), key); +} + static inline rb_shape_t * redblack_value(redblack_node_t *node) { @@ -118,7 +143,7 @@ redblack_id_for(redblack_node_t *node) return 0; } else { - redblack_node_t *redblack_nodes = rb_shape_tree.shape_cache; + redblack_node_t *redblack_nodes = redblack_cache; redblack_id_t id = (redblack_id_t)(node - redblack_nodes); return id + 1; } @@ -127,7 +152,7 @@ redblack_id_for(redblack_node_t *node) static redblack_node_t * redblack_new(char color, ID key, rb_shape_t *value, redblack_node_t *left, redblack_node_t *right) { - if (rb_shape_tree.cache_size + 1 >= REDBLACK_CACHE_SIZE) { + if (redblack_cache_size + 1 >= REDBLACK_CACHE_SIZE) { // We're out of cache, just quit return LEAF; } @@ -135,8 +160,8 @@ redblack_new(char color, ID key, rb_shape_t *value, redblack_node_t *left, redbl RUBY_ASSERT(left == LEAF || left->key < key); RUBY_ASSERT(right == LEAF || right->key > key); - redblack_node_t *redblack_nodes = rb_shape_tree.shape_cache; - redblack_node_t *node = &redblack_nodes[(rb_shape_tree.cache_size)++]; + redblack_node_t *redblack_nodes = redblack_cache; + redblack_node_t *node = &redblack_nodes[RUBY_ATOMIC_FETCH_ADD(redblack_cache_size, 1)]; node->key = key; node->value = (rb_shape_t *)((uintptr_t)value | color); node->l = redblack_id_for(left); @@ -272,34 +297,37 @@ redblack_force_black(redblack_node_t *node) return node; } -static redblack_node_t * +static redblack_id_t redblack_insert(redblack_node_t *tree, ID key, rb_shape_t *value) { redblack_node_t *root = redblack_insert_aux(tree, key, value); if (redblack_red_p(root)) { - return redblack_force_black(root); + return redblack_id_for(redblack_force_black(root)); } else { - return root; + return redblack_id_for(root); } } #endif -rb_shape_tree_t rb_shape_tree = { 0 }; static VALUE shape_tree_obj = Qfalse; +rb_shape_tree_t rb_shape_tree = { 0 }; + +// Should be on its own cache line +static RUBY_ALIGNAS(128) rb_atomic_t shape_next_id; rb_shape_t * rb_shape_get_root_shape(void) { - return rb_shape_tree.root_shape; + return rb_shape_tree.shape_list; } static void shape_tree_mark_and_move(void *data) { rb_shape_t *cursor = rb_shape_get_root_shape(); - rb_shape_t *end = RSHAPE(rb_shape_tree.next_shape_id - 1); + rb_shape_t *end = RSHAPE(shape_next_id - 1); while (cursor <= end) { if (cursor->edges && !SINGLE_CHILD_P(cursor->edges)) { rb_gc_mark_and_move(&cursor->edges); @@ -308,10 +336,25 @@ shape_tree_mark_and_move(void *data) } } +size_t +rb_shapes_cache_size(void) +{ + return redblack_cache ? redblack_cache_size : 0; +} + +size_t +rb_shapes_count(void) +{ + return (size_t)RUBY_ATOMIC_LOAD(shape_next_id); +} + static size_t shape_tree_memsize(const void *data) { - return rb_shape_tree.cache_size * sizeof(redblack_node_t); + if (redblack_cache) { + return redblack_cache_size * sizeof(redblack_node_t); + } + return 0; } static const rb_data_type_t shape_tree_type = { @@ -358,7 +401,7 @@ rb_shape_each_shape_id(each_shape_callback callback, void *data) { rb_shape_t *start = rb_shape_get_root_shape(); rb_shape_t *cursor = start; - rb_shape_t *end = RSHAPE(rb_shapes_count()); + rb_shape_t *end = RSHAPE(RUBY_ATOMIC_LOAD(shape_next_id)); while (cursor < end) { callback((shape_id_t)(cursor - start), data); cursor += 1; @@ -402,12 +445,12 @@ shape_alloc(void) shape_id_t current, new_id; do { - current = RUBY_ATOMIC_LOAD(rb_shape_tree.next_shape_id); + current = RUBY_ATOMIC_LOAD(shape_next_id); if (current > MAX_SHAPE_ID) { return NULL; // Out of shapes } new_id = current + 1; - } while (current != RUBY_ATOMIC_CAS(rb_shape_tree.next_shape_id, current, new_id)); + } while (current != RUBY_ATOMIC_CAS(shape_next_id, current, new_id)); return &rb_shape_tree.shape_list[current]; } @@ -443,12 +486,10 @@ static redblack_node_t * redblack_cache_ancestors(rb_shape_t *shape) { if (!(shape->ancestor_index || shape->parent_id == INVALID_SHAPE_ID)) { - redblack_node_t *parent_index; - - parent_index = redblack_cache_ancestors(RSHAPE(shape->parent_id)); + redblack_node_t *parent_index_node = redblack_cache_ancestors(RSHAPE(shape->parent_id)); if (shape->type == SHAPE_IVAR) { - shape->ancestor_index = redblack_insert(parent_index, shape->edge_name, shape); + shape->ancestor_index = redblack_insert(parent_index_node, shape->edge_name, shape); #if RUBY_DEBUG if (shape->ancestor_index) { @@ -459,11 +500,11 @@ redblack_cache_ancestors(rb_shape_t *shape) #endif } else { - shape->ancestor_index = parent_index; + shape->ancestor_index = redblack_id_for(parent_index_node); } } - return shape->ancestor_index; + return redblack_node(shape->ancestor_index); } #else static redblack_node_t * @@ -822,7 +863,7 @@ shape_get_next(rb_shape_t *shape, enum shape_type shape_type, VALUE klass, ID id } // Check if we should update max_iv_count on the object's class - if (new_shape->next_field_index > RCLASS_MAX_IV_COUNT(klass)) { + if (new_shape->next_field_index > RCLASS_MAX_IV_COUNT(klass) && !RCLASS_EXPECT_NO_IVAR(klass)) { RCLASS_SET_MAX_IV_COUNT(klass, new_shape->next_field_index); } @@ -1456,7 +1497,7 @@ rb_shape_root_shape(VALUE self) static VALUE rb_shape_shapes_available(VALUE self) { - return INT2NUM(MAX_SHAPE_ID - (rb_shapes_count() - 1)); + return ULL2NUM(MAX_SHAPE_ID - (rb_shapes_count() - 1)); } static VALUE @@ -1464,10 +1505,16 @@ rb_shape_exhaust(int argc, VALUE *argv, VALUE self) { rb_check_arity(argc, 0, 1); int offset = argc == 1 ? NUM2INT(argv[0]) : 0; - RUBY_ATOMIC_SET(rb_shape_tree.next_shape_id, MAX_SHAPE_ID - offset + 1); + RUBY_ATOMIC_SET(shape_next_id, MAX_SHAPE_ID - offset + 1); return Qnil; } +static VALUE +rb_shape_class_max_iv_count(VALUE self, VALUE klass) +{ + return INT2NUM(RCLASS_MAX_IV_COUNT(klass)); +} + static VALUE shape_to_h(rb_shape_t *shape); static enum rb_id_table_iterator_result collect_keys_and_values(ID key, VALUE value, void *ref) @@ -1539,18 +1586,21 @@ Init_default_shapes(void) while (heap_sizes[heaps_count]) { heaps_count++; } - attr_index_t *capacities = ALLOC_N(attr_index_t, heaps_count); + + if (heaps_count > SHAPE_ID_HEAP_INDEX_MAX) { + rb_bug("Init_default_shapes initialized with %lu heaps, only up to %u are supported", heaps_count, SHAPE_ID_HEAP_INDEX_MAX); + } + size_t index; for (index = 0; index < heaps_count; index++) { if (heap_sizes[index] > sizeof(struct RBasic)) { - capacities[index] = (heap_sizes[index] - sizeof(struct RBasic)) / sizeof(VALUE); + rb_shape_tree.capacities[index] = (heap_sizes[index] - sizeof(struct RBasic)) / sizeof(VALUE); } else { - capacities[index] = 0; + rb_shape_tree.capacities[index] = 0; } } rb_shape_tree.heaps_count = heaps_count; - rb_shape_tree.capacities = capacities; #ifdef HAVE_MMAP size_t shape_list_mmap_size = rb_size_mul_or_raise(SHAPE_BUFFER_SIZE, sizeof(rb_shape_t), rb_eRuntimeError); @@ -1574,19 +1624,19 @@ Init_default_shapes(void) #ifdef HAVE_MMAP size_t shape_cache_mmap_size = rb_size_mul_or_raise(REDBLACK_CACHE_SIZE, sizeof(redblack_node_t), rb_eRuntimeError); - rb_shape_tree.shape_cache = (redblack_node_t *)mmap(NULL, shape_cache_mmap_size, + redblack_cache = (redblack_node_t *)mmap(NULL, shape_cache_mmap_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - rb_shape_tree.cache_size = 0; + redblack_cache_size = 0; // If mmap fails, then give up on the redblack tree cache. // We set the cache size such that the redblack node allocators think // the cache is full. - if (rb_shape_tree.shape_cache == MAP_FAILED) { - rb_shape_tree.shape_cache = 0; - rb_shape_tree.cache_size = REDBLACK_CACHE_SIZE; + if (redblack_cache == MAP_FAILED) { + redblack_cache = NULL; + redblack_cache_size = REDBLACK_CACHE_SIZE; } else { - ruby_annotate_mmap(rb_shape_tree.shape_cache, shape_cache_mmap_size, "Ruby:Init_default_shapes:shape_cache"); + ruby_annotate_mmap(redblack_cache, shape_cache_mmap_size, "Ruby:Init_default_shapes:shape_cache"); } #endif @@ -1597,9 +1647,8 @@ Init_default_shapes(void) rb_shape_t *root = rb_shape_alloc_with_parent_id(0, INVALID_SHAPE_ID); root->capacity = 0; root->type = SHAPE_ROOT; - rb_shape_tree.root_shape = root; - RUBY_ASSERT(raw_shape_id(rb_shape_tree.root_shape) == ROOT_SHAPE_ID); - RUBY_ASSERT(!(raw_shape_id(rb_shape_tree.root_shape) & SHAPE_ID_HAS_IVAR_MASK)); + RUBY_ASSERT(raw_shape_id(root) == ROOT_SHAPE_ID); + RUBY_ASSERT(!(raw_shape_id(root) & SHAPE_ID_HAS_IVAR_MASK)); bool dontcare; rb_shape_t *root_with_obj_id = get_next_shape_internal(root, id_object_id, SHAPE_OBJ_ID, &dontcare, true); @@ -1612,12 +1661,6 @@ Init_default_shapes(void) (void)root_with_obj_id; } -void -rb_shape_free_all(void) -{ - xfree((void *)rb_shape_tree.capacities); -} - void Init_shape(void) { @@ -1658,5 +1701,6 @@ Init_shape(void) rb_define_singleton_method(rb_cShape, "root_shape", rb_shape_root_shape, 0); rb_define_singleton_method(rb_cShape, "shapes_available", rb_shape_shapes_available, 0); rb_define_singleton_method(rb_cShape, "exhaust_shapes", rb_shape_exhaust, -1); + rb_define_singleton_method(rb_cShape, "class_max_iv_count", rb_shape_class_max_iv_count, 1); #endif } diff --git a/shape.h b/shape.h index 09487006bc912b..aeb783b8d8e8c7 100644 --- a/shape.h +++ b/shape.h @@ -14,8 +14,8 @@ STATIC_ASSERT(shape_id_num_bits, SHAPE_ID_NUM_BITS == sizeof(shape_id_t) * CHAR_ #define SHAPE_BUFFER_SIZE (1 << SHAPE_ID_OFFSET_NUM_BITS) #define SHAPE_ID_OFFSET_MASK (SHAPE_BUFFER_SIZE - 1) -#define SHAPE_ID_HEAP_INDEX_BITS 5 -#define SHAPE_ID_HEAP_INDEX_MAX ((1 << SHAPE_ID_HEAP_INDEX_BITS) - 1) +#define SHAPE_ID_HEAP_INDEX_BITS 4 +#define SHAPE_ID_HEAP_INDEX_MAX (((attr_index_t)1 << SHAPE_ID_HEAP_INDEX_BITS) - 1) #define SHAPE_ID_HEAP_INDEX_OFFSET SHAPE_ID_OFFSET_NUM_BITS #define SHAPE_ID_FL_USHIFT (SHAPE_ID_OFFSET_NUM_BITS + SHAPE_ID_HEAP_INDEX_BITS) @@ -23,14 +23,14 @@ STATIC_ASSERT(shape_id_num_bits, SHAPE_ID_NUM_BITS == sizeof(shape_id_t) * CHAR_ // shape_id_t bits: // 0-18 SHAPE_ID_OFFSET_MASK // index in rb_shape_tree.shape_list. Allow to access `rb_shape_t *`. -// 19-23 SHAPE_ID_HEAP_INDEX_MASK +// 19-22 SHAPE_ID_HEAP_INDEX_MASK // index in rb_shape_tree.capacities. Allow to access slot size. // Always 0 except for T_OBJECT. -// 24 SHAPE_ID_FL_FROZEN +// 23 SHAPE_ID_FL_FROZEN // Whether the object is frozen or not. -// 25 SHAPE_ID_FL_HAS_OBJECT_ID +// 24 SHAPE_ID_FL_HAS_OBJECT_ID // Whether the object has an `SHAPE_OBJ_ID` transition. -// 26 SHAPE_ID_FL_TOO_COMPLEX +// 25 SHAPE_ID_FL_TOO_COMPLEX // The object is backed by a `st_table`. enum shape_id_fl_type { @@ -75,33 +75,24 @@ typedef uint32_t redblack_id_t; #define ROOT_TOO_COMPLEX_SHAPE_ID (ROOT_SHAPE_ID | SHAPE_ID_FL_TOO_COMPLEX) #define ROOT_TOO_COMPLEX_WITH_OBJ_ID (ROOT_SHAPE_WITH_OBJ_ID | SHAPE_ID_FL_TOO_COMPLEX | SHAPE_ID_FL_HAS_OBJECT_ID) -typedef struct redblack_node redblack_node_t; +enum shape_type { + SHAPE_ROOT, + SHAPE_IVAR, + SHAPE_OBJ_ID, +}; struct rb_shape { VALUE edges; // id_table from ID (ivar) to next shape ID edge_name; // ID (ivar) for transition from parent to rb_shape - redblack_node_t *ancestor_index; + redblack_id_t ancestor_index; shape_id_t parent_id; attr_index_t next_field_index; // Fields are either ivars or internal properties like `object_id` attr_index_t capacity; // Total capacity of the object with this shape - uint8_t type; + enum shape_type type : 8; }; typedef struct rb_shape rb_shape_t; -struct redblack_node { - ID key; - rb_shape_t *value; - redblack_id_t l; - redblack_id_t r; -}; - -enum shape_type { - SHAPE_ROOT, - SHAPE_IVAR, - SHAPE_OBJ_ID, -}; - enum shape_flags { SHAPE_FL_FROZEN = 1 << 0, SHAPE_FL_HAS_OBJECT_ID = 1 << 1, @@ -111,26 +102,17 @@ enum shape_flags { }; typedef struct { - /* object shapes */ rb_shape_t *shape_list; - rb_shape_t *root_shape; - const attr_index_t *capacities; - size_t heaps_count; - rb_atomic_t next_shape_id; - - redblack_node_t *shape_cache; - unsigned int cache_size; + attr_index_t heaps_count; + attr_index_t capacities[SHAPE_ID_HEAP_INDEX_MAX]; } rb_shape_tree_t; RUBY_SYMBOL_EXPORT_BEGIN RUBY_EXTERN rb_shape_tree_t rb_shape_tree; RUBY_SYMBOL_EXPORT_END -static inline shape_id_t -rb_shapes_count(void) -{ - return (shape_id_t)RUBY_ATOMIC_LOAD(rb_shape_tree.next_shape_id); -} +size_t rb_shapes_cache_size(void); +size_t rb_shapes_count(void); union rb_attr_index_cache { uint64_t pack; @@ -232,8 +214,6 @@ shape_id_t rb_shape_transition_object_id(VALUE obj); shape_id_t rb_shape_transition_heap(VALUE obj, size_t heap_index); shape_id_t rb_shape_object_id(shape_id_t original_shape_id); -void rb_shape_free_all(void); - shape_id_t rb_shape_rebuild(shape_id_t initial_shape_id, shape_id_t dest_shape_id); void rb_shape_copy_fields(VALUE dest, VALUE *dest_buf, shape_id_t dest_shape_id, VALUE *src_buf, shape_id_t src_shape_id); void rb_shape_copy_complex_ivars(VALUE dest, VALUE obj, shape_id_t src_shape_id, st_table *fields_table); diff --git a/test/ruby/test_shapes.rb b/test/ruby/test_shapes.rb index 67e2c543a3f1a3..b3333b166095fe 100644 --- a/test/ruby/test_shapes.rb +++ b/test/ruby/test_shapes.rb @@ -131,6 +131,48 @@ def test_ordered_alloc_is_not_complex assert_operator obj["variation_count"], :<, RubyVM::Shape::SHAPE_MAX_VARIATIONS end + def test_max_iv_count + klass = Class.new + object = klass.new + + assert_equal 0, RubyVM::Shape.class_max_iv_count(klass) + 8.times do |i| + object.instance_variable_set("@ivar_#{i}", i) + end + assert_equal 8, RubyVM::Shape.class_max_iv_count(klass) + + subklass = Class.new(klass) + assert_equal 8, RubyVM::Shape.class_max_iv_count(subklass) + end + + def test_max_iv_count_on_Object + object = Object.new + + assert_equal 0, RubyVM::Shape.class_max_iv_count(Object) + 8.times do |i| + object.instance_variable_set("@ivar_#{i}", i) + end + assert_equal 0, RubyVM::Shape.class_max_iv_count(Object) + end + + def test_max_iv_count_on_BasicObject + object = BasicObject.new + + assert_equal 0, RubyVM::Shape.class_max_iv_count(BasicObject) + 8.times do |i| + Object.instance_method(:instance_variable_set).bind_call(object, "@ivar_#{i}", i) + end + assert_equal 0, RubyVM::Shape.class_max_iv_count(BasicObject) + + subklass = Class.new(BasicObject) + object = subklass.new + assert_equal 0, RubyVM::Shape.class_max_iv_count(subklass) + 8.times do |i| + Object.instance_method(:instance_variable_set).bind_call(object, "@ivar_#{i}", i) + end + assert_equal 8, RubyVM::Shape.class_max_iv_count(subklass) + end + def test_too_many_ivs_on_obj assert_separately([], "#{<<~"begin;"}\n#{<<~'end;'}") begin; diff --git a/variable.c b/variable.c index cfbacbc19ad3ac..884d4106e5f684 100644 --- a/variable.c +++ b/variable.c @@ -1610,8 +1610,7 @@ obj_transition_too_complex(VALUE obj, st_table *table) break; default: { - VALUE fields_obj = rb_imemo_fields_new_complex_tbl(obj, table, RB_OBJ_SHAREABLE_P(obj)); - RBASIC_SET_SHAPE_ID(fields_obj, shape_id); + VALUE fields_obj = rb_imemo_fields_new_complex_tbl(obj, shape_id, table, RB_OBJ_SHAREABLE_P(obj)); rb_obj_replace_fields(obj, fields_obj); } } @@ -1794,27 +1793,31 @@ static VALUE imemo_fields_complex_from_obj(VALUE owner, VALUE source_fields_obj, shape_id_t shape_id) { attr_index_t len = source_fields_obj ? RSHAPE_LEN(RBASIC_SHAPE_ID(source_fields_obj)) : 0; - VALUE fields_obj = rb_imemo_fields_new_complex(owner, len + 1, RB_OBJ_SHAREABLE_P(owner)); + VALUE fields_obj = rb_imemo_fields_new_complex(owner, shape_id, len + 1, RB_OBJ_SHAREABLE_P(owner)); rb_field_foreach(source_fields_obj, imemo_fields_complex_from_obj_i, (st_data_t)fields_obj, false); - RBASIC_SET_SHAPE_ID(fields_obj, shape_id); return fields_obj; } static VALUE -imemo_fields_copy_capa(VALUE owner, VALUE source_fields_obj, attr_index_t new_size) +imemo_fields_copy_append(VALUE owner, VALUE source_fields_obj, shape_id_t current_shape_id, shape_id_t target_shape_id, VALUE val) { - VALUE fields_obj = rb_imemo_fields_new(owner, new_size, RB_OBJ_SHAREABLE_P(owner)); + attr_index_t fields_count = RSHAPE_LEN(current_shape_id); + + VALUE fields_obj = rb_imemo_fields_new(owner, target_shape_id, RB_OBJ_SHAREABLE_P(owner)); + + VALUE *fields = rb_imemo_fields_ptr(fields_obj); + if (source_fields_obj) { - attr_index_t fields_count = RSHAPE_LEN(RBASIC_SHAPE_ID(source_fields_obj)); - VALUE *fields = rb_imemo_fields_ptr(fields_obj); MEMCPY(fields, rb_imemo_fields_ptr(source_fields_obj), VALUE, fields_count); - RBASIC_SET_SHAPE_ID(fields_obj, RBASIC_SHAPE_ID(source_fields_obj)); for (attr_index_t i = 0; i < fields_count; i++) { RB_OBJ_WRITTEN(fields_obj, Qundef, fields[i]); } } + + RB_OBJ_WRITE(fields_obj, &fields[fields_count], val); + return fields_obj; } @@ -1848,13 +1851,13 @@ imemo_fields_set(VALUE owner, VALUE fields_obj, shape_id_t target_shape_id, ID f else { attr_index_t index = RSHAPE_INDEX(target_shape_id); if (concurrent || index >= RSHAPE_CAPACITY(current_shape_id)) { - fields_obj = imemo_fields_copy_capa(owner, original_fields_obj, RSHAPE_CAPACITY(target_shape_id)); + return imemo_fields_copy_append(owner, original_fields_obj, current_shape_id, target_shape_id, val); } VALUE *table = rb_imemo_fields_ptr(fields_obj); RB_OBJ_WRITE(fields_obj, &table[index], val); - if (RSHAPE_LEN(target_shape_id) > RSHAPE_LEN(current_shape_id)) { + if (index >= RSHAPE_LEN(current_shape_id)) { RBASIC_SET_SHAPE_ID(fields_obj, target_shape_id); } } @@ -2299,11 +2302,10 @@ rb_copy_generic_ivar(VALUE dest, VALUE obj) return; } - new_fields_obj = rb_imemo_fields_new(dest, RSHAPE_CAPACITY(dest_shape_id), RB_OBJ_SHAREABLE_P(dest)); + new_fields_obj = rb_imemo_fields_new(dest, dest_shape_id, RB_OBJ_SHAREABLE_P(dest)); VALUE *src_buf = rb_imemo_fields_ptr(fields_obj); VALUE *dest_buf = rb_imemo_fields_ptr(new_fields_obj); rb_shape_copy_fields(new_fields_obj, dest_buf, dest_shape_id, src_buf, src_shape_id); - RBASIC_SET_SHAPE_ID(new_fields_obj, dest_shape_id); rb_obj_replace_fields(dest, new_fields_obj); } @@ -4591,7 +4593,7 @@ static attr_index_t class_fields_ivar_set(VALUE klass, VALUE fields_obj, ID id, VALUE val, bool concurrent, VALUE *new_fields_obj, bool *new_ivar_out) { const VALUE original_fields_obj = fields_obj; - fields_obj = original_fields_obj ? original_fields_obj : rb_imemo_fields_new(klass, 1, true); + fields_obj = original_fields_obj ? original_fields_obj : rb_imemo_fields_new(klass, ROOT_SHAPE_ID, true); shape_id_t current_shape_id = RBASIC_SHAPE_ID(fields_obj); shape_id_t next_shape_id = current_shape_id; // for too_complex @@ -4608,30 +4610,29 @@ class_fields_ivar_set(VALUE klass, VALUE fields_obj, ID id, VALUE val, bool conc } attr_index_t index = RSHAPE_INDEX(next_shape_id); - if (new_ivar) { - if (index >= RSHAPE_CAPACITY(current_shape_id)) { - // We allocate a new fields_obj even when concurrency isn't a concern - // so that we're embedded as long as possible. - fields_obj = imemo_fields_copy_capa(klass, fields_obj, RSHAPE_CAPACITY(next_shape_id)); - } - } - - VALUE *fields = rb_imemo_fields_ptr(fields_obj); - - if (concurrent && original_fields_obj == fields_obj) { - // In the concurrent case, if we're mutating the existing - // fields_obj, we must use an atomic write, because if we're - // adding a new field, the shape_id must be written after the field - // and if we're updating an existing field, we at least need a relaxed - // write to avoid reaping. - RB_OBJ_ATOMIC_WRITE(fields_obj, &fields[index], val); + if (new_ivar && index >= RSHAPE_CAPACITY(current_shape_id)) { + // We allocate a new fields_obj even when concurrency isn't a concern + // so that we're embedded as long as possible. + fields_obj = imemo_fields_copy_append(klass, fields_obj, current_shape_id, next_shape_id, val); } else { - RB_OBJ_WRITE(fields_obj, &fields[index], val); - } + VALUE *fields = rb_imemo_fields_ptr(fields_obj); - if (new_ivar) { - RBASIC_SET_SHAPE_ID(fields_obj, next_shape_id); + if (concurrent && original_fields_obj == fields_obj) { + // In the concurrent case, if we're mutating the existing + // fields_obj, we must use an atomic write, because if we're + // adding a new field, the shape_id must be written after the field + // and if we're updating an existing field, we at least need a relaxed + // write to avoid reaping. + RB_OBJ_ATOMIC_WRITE(fields_obj, &fields[index], val); + } + else { + RB_OBJ_WRITE(fields_obj, &fields[index], val); + } + + if (new_ivar) { + RBASIC_SET_SHAPE_ID(fields_obj, next_shape_id); + } } *new_fields_obj = fields_obj; diff --git a/vm.c b/vm.c index 856c3c7431a390..71d5a84034ecd1 100644 --- a/vm.c +++ b/vm.c @@ -867,7 +867,7 @@ vm_stat(int argc, VALUE *argv, VALUE self) SET(constant_cache_misses, ruby_vm_constant_cache_misses); SET(global_cvar_state, ruby_vm_global_cvar_state); SET(next_shape_id, (rb_serial_t)rb_shapes_count()); - SET(shape_cache_size, (rb_serial_t)rb_shape_tree.cache_size); + SET(shape_cache_size, (rb_serial_t)rb_shapes_cache_size()); #undef SET #if USE_DEBUG_COUNTER @@ -3477,7 +3477,6 @@ ruby_vm_destruct(rb_vm_t *vm) ruby_current_vm_ptr = NULL; if (rb_free_at_exit) { - rb_shape_free_all(); #if USE_YJIT rb_yjit_free_at_exit(); #endif diff --git a/vm_insnhelper.c b/vm_insnhelper.c index dcd4f1cda43f07..eeb8888f18c04a 100644 --- a/vm_insnhelper.c +++ b/vm_insnhelper.c @@ -6045,8 +6045,8 @@ vm_define_method(const rb_execution_context_t *ec, VALUE obj, ID id, VALUE iseqv rb_add_method_iseq(klass, id, (const rb_iseq_t *)iseqval, cref, visi); // Set max_iv_count on klasses based on number of ivar sets that are in the initialize method - if (id == idInitialize && klass != rb_cObject && RB_TYPE_P(klass, T_CLASS) && - !RCLASS_SINGLETON_P(klass) && + if (id == idInitialize && RB_TYPE_P(klass, T_CLASS) && + !RCLASS_SINGLETON_P(klass) && !RCLASS_EXPECT_NO_IVAR(klass) && (rb_get_alloc_func(klass) == rb_class_allocate_instance)) { RCLASS_SET_MAX_IV_COUNT(klass, rb_estimate_iv_count(klass, (const rb_iseq_t *)iseqval)); } diff --git a/yjit/src/codegen.rs b/yjit/src/codegen.rs index 8d22d30c4de517..16d0d18abdc6d5 100644 --- a/yjit/src/codegen.rs +++ b/yjit/src/codegen.rs @@ -2975,7 +2975,7 @@ fn gen_get_ivar( let ivar_index = unsafe { let shape_id = comptime_receiver.shape_id_of(); - let mut ivar_index: u16 = 0; + let mut ivar_index: attr_index_t = 0; if rb_shape_get_iv_index(shape_id, ivar_name, &mut ivar_index) { Some(ivar_index as usize) } else { @@ -3171,7 +3171,7 @@ fn gen_set_ivar( let shape_too_complex = comptime_receiver.shape_too_complex(); let ivar_index = if !comptime_receiver.special_const_p() && !shape_too_complex { let shape_id = comptime_receiver.shape_id_of(); - let mut ivar_index: u16 = 0; + let mut ivar_index: attr_index_t = 0; if unsafe { rb_shape_get_iv_index(shape_id, ivar_name, &mut ivar_index) } { Some(ivar_index as usize) } else { @@ -3461,7 +3461,7 @@ fn gen_definedivar( let shape_id = comptime_receiver.shape_id_of(); let ivar_exists = unsafe { - let mut ivar_index: u16 = 0; + let mut ivar_index: attr_index_t = 0; rb_shape_get_iv_index(shape_id, ivar_name, &mut ivar_index) }; diff --git a/yjit/src/cruby_bindings.inc.rs b/yjit/src/cruby_bindings.inc.rs index 7c750e2092d5e2..bbe024df941718 100644 --- a/yjit/src/cruby_bindings.inc.rs +++ b/yjit/src/cruby_bindings.inc.rs @@ -636,7 +636,7 @@ pub const VM_ENV_FLAG_ISOLATED: vm_frame_env_flags = 16; pub type vm_frame_env_flags = u32; pub type attr_index_t = u16; pub type shape_id_t = u32; -pub const SHAPE_ID_HAS_IVAR_MASK: shape_id_mask = 67633150; +pub const SHAPE_ID_HAS_IVAR_MASK: shape_id_mask = 34078718; pub type shape_id_mask = u32; #[repr(C)] pub struct rb_cvar_class_tbl_entry { diff --git a/zjit/src/codegen.rs b/zjit/src/codegen.rs index a98a85f41f64af..b9b8b6509abfb2 100644 --- a/zjit/src/codegen.rs +++ b/zjit/src/codegen.rs @@ -614,6 +614,7 @@ fn gen_insn(cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, functio &Insn::Const { val: Const::CUInt16(val) } => gen_const_uint16(val), &Insn::Const { val: Const::CUInt32(val) } => gen_const_uint32(val), &Insn::Const { val: Const::CUInt64(val) } => Opnd::UImm(val), + &Insn::Const { val: Const::CAttrIndex(val) } => gen_const_attr_index_t(val), &Insn::Const { val: Const::CShape(val) } => { assert_eq!(SHAPE_ID_NUM_BITS, 32); gen_const_uint32(val.0) @@ -1455,6 +1456,10 @@ fn gen_const_uint32(val: u32) -> lir::Opnd { Opnd::UImm(val as u64) } +fn gen_const_attr_index_t(val: attr_index_t) -> lir::Opnd { + Opnd::UImm(val as u64) +} + /// Compile a basic block argument fn gen_param(asm: &mut Assembler, _idx: usize) -> lir::Opnd { let vreg = asm.new_block_param(VALUE_BITS); diff --git a/zjit/src/cruby.rs b/zjit/src/cruby.rs index 83f4b13f5fa119..dfaede11bd5c68 100644 --- a/zjit/src/cruby.rs +++ b/zjit/src/cruby.rs @@ -1518,6 +1518,12 @@ mod class_name_tests { } pub fn class_has_leaf_allocator(class: VALUE) -> bool { + // We need to check if the class is initialized and not a singleton before + // trying to read the allocator, otherwise it will raise. + // Because of this they should be considered non-leaf anyways. + if !unsafe { rb_zjit_class_initialized_p(class) } { return false; } + if unsafe { rb_zjit_singleton_class_p(class) } { return false; } + // empty_hash_alloc if class == unsafe { rb_cHash } { return true; } // empty_ary_alloc diff --git a/zjit/src/cruby_bindings.inc.rs b/zjit/src/cruby_bindings.inc.rs index fe0a393e9ccf28..a31f00fb1267ef 100644 --- a/zjit/src/cruby_bindings.inc.rs +++ b/zjit/src/cruby_bindings.inc.rs @@ -1484,12 +1484,12 @@ pub const VM_ENV_FLAG_ISOLATED: vm_frame_env_flags = 16; pub type vm_frame_env_flags = u32; pub type attr_index_t = u16; pub type shape_id_t = u32; -pub const SHAPE_ID_HEAP_INDEX_MASK: shape_id_fl_type = 16252928; -pub const SHAPE_ID_FL_FROZEN: shape_id_fl_type = 16777216; -pub const SHAPE_ID_FL_HAS_OBJECT_ID: shape_id_fl_type = 33554432; -pub const SHAPE_ID_FL_TOO_COMPLEX: shape_id_fl_type = 67108864; -pub const SHAPE_ID_FL_NON_CANONICAL_MASK: shape_id_fl_type = 50331648; -pub const SHAPE_ID_FLAGS_MASK: shape_id_fl_type = 133693440; +pub const SHAPE_ID_HEAP_INDEX_MASK: shape_id_fl_type = 7864320; +pub const SHAPE_ID_FL_FROZEN: shape_id_fl_type = 8388608; +pub const SHAPE_ID_FL_HAS_OBJECT_ID: shape_id_fl_type = 16777216; +pub const SHAPE_ID_FL_TOO_COMPLEX: shape_id_fl_type = 33554432; +pub const SHAPE_ID_FL_NON_CANONICAL_MASK: shape_id_fl_type = 25165824; +pub const SHAPE_ID_FLAGS_MASK: shape_id_fl_type = 66584576; pub type shape_id_fl_type = u32; pub const CONST_DEPRECATED: rb_const_flag_t = 256; pub const CONST_VISIBILITY_MASK: rb_const_flag_t = 255; diff --git a/zjit/src/cruby_methods.rs b/zjit/src/cruby_methods.rs index 93ba7da6403321..f2694c03a105de 100644 --- a/zjit/src/cruby_methods.rs +++ b/zjit/src/cruby_methods.rs @@ -247,6 +247,7 @@ pub fn init() -> Annotations { annotate!(rb_cBasicObject, "!", inline_basic_object_not, types::BoolExact, no_gc, leaf, elidable); annotate!(rb_cBasicObject, "!=", inline_basic_object_neq, types::BoolExact); annotate!(rb_cBasicObject, "initialize", inline_basic_object_initialize); + annotate!(rb_cClass, "allocate", inline_class_allocate); annotate!(rb_cInteger, "succ", inline_integer_succ); annotate!(rb_cInteger, "^", inline_integer_xor); annotate!(rb_cInteger, "==", inline_integer_eq); @@ -850,6 +851,13 @@ fn inline_basic_object_neq(fun: &mut hir::Function, block: hir::BlockId, recv: h Some(result) } +fn inline_class_allocate(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option { + if !args.is_empty() { return None; } + + // Inline only in the case we have a leaf allocator + fun.try_inline_object_alloc(block, recv, state) +} + fn inline_basic_object_initialize(fun: &mut hir::Function, block: hir::BlockId, _recv: hir::InsnId, args: &[hir::InsnId], _state: hir::InsnId) -> Option { if !args.is_empty() { return None; } let result = fun.push_insn(block, hir::Insn::Const { val: hir::Const::Value(Qnil) }); diff --git a/zjit/src/hir.rs b/zjit/src/hir.rs index 4f060fcedecf8a..812f793489b908 100644 --- a/zjit/src/hir.rs +++ b/zjit/src/hir.rs @@ -311,6 +311,7 @@ pub enum Const { CUInt8(u8), CUInt16(u16), CUInt32(u32), + CAttrIndex(attr_index_t), CShape(ShapeId), CUInt64(u64), CPtr(*const u8), @@ -3079,6 +3080,7 @@ impl Function { Insn::Const { val: Const::CUInt8(val) } => Type::from_cint(types::CUInt8, *val as i64), Insn::Const { val: Const::CUInt16(val) } => Type::from_cint(types::CUInt16, *val as i64), Insn::Const { val: Const::CUInt32(val) } => Type::from_cint(types::CUInt32, *val as i64), + Insn::Const { val: Const::CAttrIndex(val) } => Type::from_cint(types::CAttrIndex, *val as i64), Insn::Const { val: Const::CShape(val) } => Type::from_cint(types::CShape, val.0 as i64), Insn::Const { val: Const::CUInt64(val) } => Type::from_cint(types::CUInt64, *val as i64), Insn::Const { val: Const::CPtr(val) } => Type::from_cptr(*val), @@ -3617,6 +3619,21 @@ impl Function { self.push_insn_id(block, orig_insn_id); } + pub fn try_inline_object_alloc(&mut self, block: BlockId, recv: InsnId, state: InsnId) -> Option { + let recv_type = self.type_of(recv); + if recv_type.is_subtype(types::Class) { + if let Some(class) = recv_type.ruby_object() { + // See class_get_alloc_func in object.c; if the class isn't initialized, is + // a singleton class, or has a custom allocator, ObjectAlloc might raise an + // exception or run arbitrary code. + if class_has_leaf_allocator(class) { + return Some(self.push_insn(block, Insn::ObjectAllocClass { class, state })); + } + } + } + None + } + fn try_rewrite_freeze(&mut self, block: BlockId, orig_insn_id: InsnId, self_val: InsnId, state: InsnId) { if self.is_a(self_val, types::StringExact) { self.rewrite_if_frozen(block, orig_insn_id, self_val, STRING_REDEFINED_OP_FLAG, BOP_FREEZE, state); @@ -4050,32 +4067,12 @@ impl Function { self.make_equal_to(insn_id, replacement); } Insn::ObjectAlloc { val, state } => { - let val_type = self.type_of(val); - if !val_type.is_subtype(types::Class) { - self.push_insn_id(block, insn_id); continue; - } - let Some(class) = val_type.ruby_object() else { - self.push_insn_id(block, insn_id); continue; - }; - // See class_get_alloc_func in object.c; if the class isn't initialized, is - // a singleton class, or has a custom allocator, ObjectAlloc might raise an - // exception or run arbitrary code. - // - // We also need to check if the class is initialized or a singleton before - // trying to read the allocator, otherwise it might raise. - if !unsafe { rb_zjit_class_initialized_p(class) } { - self.push_insn_id(block, insn_id); continue; - } - if unsafe { rb_zjit_singleton_class_p(class) } { - self.push_insn_id(block, insn_id); continue; - } - if !class_has_leaf_allocator(class) { - // Custom, known unsafe, or NULL allocator; could run arbitrary code. - self.push_insn_id(block, insn_id); continue; + if let Some(replacement) = self.try_inline_object_alloc(block, val, state) { + self.insn_types[replacement.0] = self.infer_type(replacement); + self.make_equal_to(insn_id, replacement); + } else { + self.push_insn_id(block, insn_id); } - let replacement = self.push_insn(block, Insn::ObjectAllocClass { class, state }); - self.insn_types[replacement.0] = self.infer_type(replacement); - self.make_equal_to(insn_id, replacement); } Insn::NewRange { low, high, flag, state } => { let low_is_fix = self.is_a(low, types::Fixnum); @@ -4445,10 +4442,10 @@ impl Function { }) } - fn load_ivar_c_call(&mut self, block: BlockId, recv: InsnId, ivar_index: u16) -> InsnId { + fn load_ivar_c_call(&mut self, block: BlockId, recv: InsnId, ivar_index: attr_index_t) -> InsnId { // NOTE: it's fine to use rb_ivar_get_at_no_ractor_check because // getinstancevariable does assume_single_ractor_mode() - let ivar_index_insn = self.push_insn(block, Insn::Const { val: Const::CUInt16(ivar_index) }); + let ivar_index_insn = self.push_insn(block, Insn::Const { val: Const::CAttrIndex(ivar_index) }); self.push_insn(block, Insn::CCall { cfunc: rb_ivar_get_at_no_ractor_check as *const u8, recv, @@ -4459,7 +4456,7 @@ impl Function { elidable: true }) } - fn load_ivar_heap(&mut self, block: BlockId, recv: InsnId, id: ID, ivar_index: u16) -> InsnId { + fn load_ivar_heap(&mut self, block: BlockId, recv: InsnId, id: ID, ivar_index: attr_index_t) -> InsnId { // See ROBJECT_FIELDS() from include/ruby/internal/core/robject.h let ptr = self.push_insn(block, Insn::LoadField { recv, id: ID!(_as_heap), @@ -4473,7 +4470,7 @@ impl Function { }) } - fn load_ivar_embedded(&mut self, block: BlockId, recv: InsnId, id: ID, ivar_index: u16) -> InsnId { + fn load_ivar_embedded(&mut self, block: BlockId, recv: InsnId, id: ID, ivar_index: attr_index_t) -> InsnId { // See ROBJECT_FIELDS() from include/ruby/internal/core/robject.h let offset = ROBJECT_OFFSET_AS_ARY as i32 + (SIZEOF_VALUE * ivar_index.to_usize()) as i32; @@ -4483,7 +4480,7 @@ impl Function { }) } - fn load_ivar_from_fields(&mut self, block: BlockId, recv: InsnId, is_embedded: bool, id: ID, ivar_index: u16) -> InsnId { + fn load_ivar_from_fields(&mut self, block: BlockId, recv: InsnId, is_embedded: bool, id: ID, ivar_index: attr_index_t) -> InsnId { if is_embedded { return self.load_ivar_embedded(block, recv, id, ivar_index); } else { @@ -4512,7 +4509,7 @@ impl Function { // Too-complex shapes use hash tables; rb_shape_get_iv_index doesn't support them. // Callers must filter these out before calling load_ivar. assert!(!recv_type.shape().is_too_complex(), "load_ivar called with too-complex shape"); - let mut ivar_index: u16 = 0; + let mut ivar_index: attr_index_t = 0; if ! unsafe { rb_shape_get_iv_index(recv_type.shape().0, id, &mut ivar_index) } { // If there is no IVAR index, then the ivar was undefined when we // entered the compiler. That means we can just return nil for this @@ -4616,7 +4613,7 @@ impl Function { let self_val = self.load_ivar_guard_type(block, self_val, recv_type, state); let shape = self.load_shape(block, self_val); self.guard_shape(block, shape, recv_type.shape(), state, None); - let mut ivar_index: u16 = 0; + let mut ivar_index: attr_index_t = 0; let replacement = if unsafe { rb_shape_get_iv_index(recv_type.shape().0, id, &mut ivar_index) } { self.push_insn(block, Insn::Const { val: Const::Value(pushval) }) } else { @@ -4655,7 +4652,7 @@ impl Function { self.count(block, Counter::setivar_fallback_frozen); self.push_insn_id(block, insn_id); continue; } - let mut ivar_index: u16 = 0; + let mut ivar_index: attr_index_t = 0; let mut next_shape_id = recv_type.shape(); if !unsafe { rb_shape_get_iv_index(recv_type.shape().0, id, &mut ivar_index) } { // Current shape does not contain this ivar; do a shape transition. @@ -6198,8 +6195,7 @@ impl Function { Insn::HashDup { val, .. } => self.assert_subtype(insn_id, val, types::HashExact), // Other Insn::ObjectAllocClass { class, .. } => { - let has_leaf_allocator = unsafe { rb_zjit_class_has_default_allocator(class) } || class_has_leaf_allocator(class); - if !has_leaf_allocator { + if !class_has_leaf_allocator(class) { return Err(ValidationError::MiscValidationError(insn_id, "ObjectAllocClass must have leaf allocator".to_string())); } Ok(()) @@ -6300,6 +6296,7 @@ impl Function { Const::CUInt8(_) => self.assert_subtype(insn_id, val, types::CUInt8), Const::CUInt16(_) => self.assert_subtype(insn_id, val, types::CUInt16), Const::CUInt32(_) => self.assert_subtype(insn_id, val, types::CUInt32), + Const::CAttrIndex(_) => self.assert_subtype(insn_id, val, types::CAttrIndex), Const::CShape(_) => self.assert_subtype(insn_id, val, types::CShape), Const::CUInt64(_) => self.assert_subtype(insn_id, val, types::CUInt64), Const::CBool(_) => self.assert_subtype(insn_id, val, types::CBool), diff --git a/zjit/src/hir/opt_tests.rs b/zjit/src/hir/opt_tests.rs index 3492f7c18713f3..143201134cba3c 100644 --- a/zjit/src/hir/opt_tests.rs +++ b/zjit/src/hir/opt_tests.rs @@ -4717,6 +4717,94 @@ mod hir_opt_tests { "); } + #[test] + fn test_inline_class_allocate() { + eval(" + class C; end + def test = C.allocate + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, C) + v20:Class[C@0x1008] = Const Value(VALUE(0x1008)) + PatchPoint MethodRedefined(Class@0x1010, allocate@0x1018, cme:0x1020) + v23:ObjectSubclass[class_exact:C] = ObjectAllocClass C:VALUE(0x1008) + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_dont_inline_class_allocate_with_args() { + eval(" + class C; end + def test = C.allocate(1) + test rescue 0 + test rescue 0 + "); + // Not specialized + assert_snapshot!(hir_string("test"), @" + fn test@:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, C) + v22:Class[C@0x1008] = Const Value(VALUE(0x1008)) + v12:Fixnum[1] = Const Value(1) + v14:BasicObject = Send v22, :allocate, v12 # SendFallbackReason: SendWithoutBlock: unsupported method type Cfunc + CheckInterrupts + Return v14 + "); + } + + #[test] + fn test_dont_inline_class_allocate_with_singleton_class() { + eval(" + class C; end + SC = C.singleton_class + def test = SC.allocate + test rescue 0 + "); + // Not specialized: singleton classes are not leaf allocators + assert_snapshot!(hir_string("test"), @" + fn test@:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, SC) + v20:Class[Class@0x1008] = Const Value(VALUE(0x1008)) + PatchPoint MethodRedefined(Class@0x1010, allocate@0x1018, cme:0x1020) + v23:BasicObject = CCallWithFrame v20, :Class.allocate@0x1048 + CheckInterrupts + Return v23 + "); + } + #[test] fn test_opt_length() { eval(" @@ -7685,7 +7773,7 @@ mod hir_opt_tests { v17:HeapBasicObject = GuardType v6, HeapBasicObject v18:CShape = LoadField v17, :_shape_id@0x1000 v19:CShape[0x1001] = GuardBitEquals v18, CShape(0x1001) - v20:CUInt16[0] = Const CUInt16(0) + v20:CAttrIndex[0] = Const CAttrIndex(0) v21:BasicObject = CCall v17, :rb_ivar_get_at_no_ractor_check@0x1008, v20 CheckInterrupts Return v21 diff --git a/zjit/src/hir_type/gen_hir_type.rb b/zjit/src/hir_type/gen_hir_type.rb index aca8e574d61457..e9d925ba0f5b13 100644 --- a/zjit/src/hir_type/gen_hir_type.rb +++ b/zjit/src/hir_type/gen_hir_type.rb @@ -155,6 +155,7 @@ def final_type name, base: $object, c_name: nil unsigned.subtype "CUInt#{width}" } unsigned.subtype "CShape" +unsigned.subtype "CAttrIndex" # Assign individual bits to type leaves and union bit patterns to nodes with subtypes num_bits = 0 diff --git a/zjit/src/hir_type/hir_type.inc.rs b/zjit/src/hir_type/hir_type.inc.rs index 46c45f17f39e07..8b95914ddf180a 100644 --- a/zjit/src/hir_type/hir_type.inc.rs +++ b/zjit/src/hir_type/hir_type.inc.rs @@ -10,72 +10,73 @@ mod bits { pub const Bignum: u64 = 1u64 << 4; pub const BoolExact: u64 = FalseClass | TrueClass; pub const BuiltinExact: u64 = ArrayExact | BasicObjectExact | FalseClass | Float | HashExact | Integer | ModuleExact | NilClass | NumericExact | ObjectExact | RangeExact | RegexpExact | SetExact | StringExact | Symbol | TrueClass; - pub const CBool: u64 = 1u64 << 5; - pub const CDouble: u64 = 1u64 << 6; + pub const CAttrIndex: u64 = 1u64 << 5; + pub const CBool: u64 = 1u64 << 6; + pub const CDouble: u64 = 1u64 << 7; pub const CInt: u64 = CSigned | CUnsigned; - pub const CInt16: u64 = 1u64 << 7; - pub const CInt32: u64 = 1u64 << 8; - pub const CInt64: u64 = 1u64 << 9; - pub const CInt8: u64 = 1u64 << 10; - pub const CNull: u64 = 1u64 << 11; - pub const CPtr: u64 = 1u64 << 12; - pub const CShape: u64 = 1u64 << 13; + pub const CInt16: u64 = 1u64 << 8; + pub const CInt32: u64 = 1u64 << 9; + pub const CInt64: u64 = 1u64 << 10; + pub const CInt8: u64 = 1u64 << 11; + pub const CNull: u64 = 1u64 << 12; + pub const CPtr: u64 = 1u64 << 13; + pub const CShape: u64 = 1u64 << 14; pub const CSigned: u64 = CInt16 | CInt32 | CInt64 | CInt8; - pub const CUInt16: u64 = 1u64 << 14; - pub const CUInt32: u64 = 1u64 << 15; - pub const CUInt64: u64 = 1u64 << 16; - pub const CUInt8: u64 = 1u64 << 17; - pub const CUnsigned: u64 = CShape | CUInt16 | CUInt32 | CUInt64 | CUInt8; + pub const CUInt16: u64 = 1u64 << 15; + pub const CUInt32: u64 = 1u64 << 16; + pub const CUInt64: u64 = 1u64 << 17; + pub const CUInt8: u64 = 1u64 << 18; + pub const CUnsigned: u64 = CAttrIndex | CShape | CUInt16 | CUInt32 | CUInt64 | CUInt8; pub const CValue: u64 = CBool | CDouble | CInt | CNull | CPtr; - pub const CallableMethodEntry: u64 = 1u64 << 18; - pub const Class: u64 = 1u64 << 19; - pub const DynamicSymbol: u64 = 1u64 << 20; + pub const CallableMethodEntry: u64 = 1u64 << 19; + pub const Class: u64 = 1u64 << 20; + pub const DynamicSymbol: u64 = 1u64 << 21; pub const Empty: u64 = 0u64; - pub const FalseClass: u64 = 1u64 << 21; + pub const FalseClass: u64 = 1u64 << 22; pub const Falsy: u64 = FalseClass | NilClass; - pub const Fixnum: u64 = 1u64 << 22; + pub const Fixnum: u64 = 1u64 << 23; pub const Float: u64 = Flonum | HeapFloat; - pub const Flonum: u64 = 1u64 << 23; + pub const Flonum: u64 = 1u64 << 24; pub const Hash: u64 = HashExact | HashSubclass; - pub const HashExact: u64 = 1u64 << 24; - pub const HashSubclass: u64 = 1u64 << 25; + pub const HashExact: u64 = 1u64 << 25; + pub const HashSubclass: u64 = 1u64 << 26; pub const HeapBasicObject: u64 = BasicObject & !Immediate; - pub const HeapFloat: u64 = 1u64 << 26; + pub const HeapFloat: u64 = 1u64 << 27; pub const HeapObject: u64 = Object & !Immediate; pub const Immediate: u64 = FalseClass | Fixnum | Flonum | NilClass | StaticSymbol | TrueClass | Undef; pub const Integer: u64 = Bignum | Fixnum; pub const Module: u64 = Class | ModuleExact | ModuleSubclass; - pub const ModuleExact: u64 = 1u64 << 27; - pub const ModuleSubclass: u64 = 1u64 << 28; - pub const NilClass: u64 = 1u64 << 29; + pub const ModuleExact: u64 = 1u64 << 28; + pub const ModuleSubclass: u64 = 1u64 << 29; + pub const NilClass: u64 = 1u64 << 30; pub const NotNil: u64 = BasicObject & !NilClass; pub const Numeric: u64 = Float | Integer | NumericExact | NumericSubclass; - pub const NumericExact: u64 = 1u64 << 30; - pub const NumericSubclass: u64 = 1u64 << 31; + pub const NumericExact: u64 = 1u64 << 31; + pub const NumericSubclass: u64 = 1u64 << 32; pub const Object: u64 = Array | FalseClass | Hash | Module | NilClass | Numeric | ObjectExact | ObjectSubclass | Range | Regexp | Set | String | Symbol | TrueClass; - pub const ObjectExact: u64 = 1u64 << 32; - pub const ObjectSubclass: u64 = 1u64 << 33; + pub const ObjectExact: u64 = 1u64 << 33; + pub const ObjectSubclass: u64 = 1u64 << 34; pub const Range: u64 = RangeExact | RangeSubclass; - pub const RangeExact: u64 = 1u64 << 34; - pub const RangeSubclass: u64 = 1u64 << 35; + pub const RangeExact: u64 = 1u64 << 35; + pub const RangeSubclass: u64 = 1u64 << 36; pub const Regexp: u64 = RegexpExact | RegexpSubclass; - pub const RegexpExact: u64 = 1u64 << 36; - pub const RegexpSubclass: u64 = 1u64 << 37; + pub const RegexpExact: u64 = 1u64 << 37; + pub const RegexpSubclass: u64 = 1u64 << 38; pub const RubyValue: u64 = BasicObject | CallableMethodEntry | Undef; pub const Set: u64 = SetExact | SetSubclass; - pub const SetExact: u64 = 1u64 << 38; - pub const SetSubclass: u64 = 1u64 << 39; - pub const StaticSymbol: u64 = 1u64 << 40; + pub const SetExact: u64 = 1u64 << 39; + pub const SetSubclass: u64 = 1u64 << 40; + pub const StaticSymbol: u64 = 1u64 << 41; pub const String: u64 = StringExact | StringSubclass; - pub const StringExact: u64 = 1u64 << 41; - pub const StringSubclass: u64 = 1u64 << 42; + pub const StringExact: u64 = 1u64 << 42; + pub const StringSubclass: u64 = 1u64 << 43; pub const Subclass: u64 = ArraySubclass | BasicObjectSubclass | HashSubclass | ModuleSubclass | NumericSubclass | ObjectSubclass | RangeSubclass | RegexpSubclass | SetSubclass | StringSubclass; pub const Symbol: u64 = DynamicSymbol | StaticSymbol; - pub const TrueClass: u64 = 1u64 << 43; + pub const TrueClass: u64 = 1u64 << 44; pub const Truthy: u64 = BasicObject & !Falsy; - pub const TypedTData: u64 = 1u64 << 44; - pub const Undef: u64 = 1u64 << 45; - pub const AllBitPatterns: [(&str, u64); 75] = [ + pub const TypedTData: u64 = 1u64 << 45; + pub const Undef: u64 = 1u64 << 46; + pub const AllBitPatterns: [(&str, u64); 76] = [ ("Any", Any), ("RubyValue", RubyValue), ("Immediate", Immediate), @@ -144,6 +145,7 @@ mod bits { ("CInt16", CInt16), ("CDouble", CDouble), ("CBool", CBool), + ("CAttrIndex", CAttrIndex), ("Bignum", Bignum), ("BasicObjectSubclass", BasicObjectSubclass), ("BasicObjectExact", BasicObjectExact), @@ -152,7 +154,7 @@ mod bits { ("ArrayExact", ArrayExact), ("Empty", Empty), ]; - pub const NumTypeBits: u64 = 46; + pub const NumTypeBits: u64 = 47; } pub mod types { use super::*; @@ -166,6 +168,7 @@ pub mod types { pub const Bignum: Type = Type::from_bits(bits::Bignum); pub const BoolExact: Type = Type::from_bits(bits::BoolExact); pub const BuiltinExact: Type = Type::from_bits(bits::BuiltinExact); + pub const CAttrIndex: Type = Type::from_bits(bits::CAttrIndex); pub const CBool: Type = Type::from_bits(bits::CBool); pub const CDouble: Type = Type::from_bits(bits::CDouble); pub const CInt: Type = Type::from_bits(bits::CInt); diff --git a/zjit/src/hir_type/mod.rs b/zjit/src/hir_type/mod.rs index ff59e3016ac3a9..206b74e7d38de3 100644 --- a/zjit/src/hir_type/mod.rs +++ b/zjit/src/hir_type/mod.rs @@ -269,6 +269,7 @@ impl Type { Const::CUInt8(v) => Self::from_cint(types::CUInt8, v as i64), Const::CUInt16(v) => Self::from_cint(types::CUInt16, v as i64), Const::CUInt32(v) => Self::from_cint(types::CUInt32, v as i64), + Const::CAttrIndex(v) => Self::from_cint(types::CAttrIndex, v as i64), Const::CShape(v) => Self::from_cint(types::CShape, v.0 as i64), Const::CUInt64(v) => Self::from_cint(types::CUInt64, v as i64), Const::CPtr(v) => Self::from_cptr(v),