add array element stride for frozen heap arrays

Author: AlexaCampusano

array.c

@@ -3676,6 +3676,7 @@ iseq_peephole_optimize(rb_iseq_t *iseq, LINK_ELEMENT *list, const int do_tailcal
             if (vm_ci_simple(ci) && vm_ci_argc(ci) == 0 && blockiseq == NULL && vm_ci_mid(ci) == idFreeze) {
                 VALUE ary = iobj->operands[0];
                 rb_obj_reveal(ary, rb_cArray);
+                rb_ary_narrow(ary);
 
                 insn_replace_with_operands(iseq, iobj, BIN(opt_ary_freeze), 2, ary, (VALUE)ci);
                 ELEM_REMOVE(next);

compile.c

@@ -3677,6 +3677,11 @@ rb_gc_mark_children(void *objspace, VALUE obj)
                 gc_mark_and_pin_internal(root);
             }
         }
+        else if (rb_array_stride(obj) != RARRAY_STRIDE_VALUE) {
+            /* narrow storage holds unboxed values, never object references, so
+             * there is nothing to trace; widening here is impossible anyway
+             * because it allocates */
+        }
         else {
             long len = RARRAY_LEN(obj);
             const VALUE *ptr = RARRAY_CONST_PTR(obj);
@@ -4620,7 +4625,8 @@ gc_ref_update_array(void *objspace, VALUE v)
     else {
         long len = RARRAY_LEN(v);
 
-        if (len > 0) {
+        /* narrow storage holds no object references: nothing to re-point */
+        if (len > 0 && rb_array_stride(v) == RARRAY_STRIDE_VALUE) {
             VALUE *ptr = (VALUE *)RARRAY_CONST_PTR(v);
             for (long i = 0; i < len; i++) {
                 UPDATE_IF_MOVED(objspace, ptr[i]);

gc.c

@@ -321,6 +321,21 @@ rb_objid_hash(st_index_t index)
     return (long)st_index_hash(index);
 }
 
+/* The value NUM2LONG(rb_hash(fixnum)) would produce, without the type switch
+ * and the Fixnum round trip.  For code that hashes many Fixnums in a row. */
+long
+rb_fixnum_hash(VALUE fixnum)
+{
+    RUBY_ASSERT(FIXNUM_P(fixnum));
+    /* same clamping as the tail of any_hash() */
+    st_index_t hnum = rb_objid_hash((st_index_t)fixnum);
+    if ((SIGNED_VALUE)hnum > 0)
+        hnum &= FIXNUM_MAX;
+    else
+        hnum |= FIXNUM_MIN;
+    return (long)hnum;
+}
+
 static st_index_t
 objid_hash(VALUE obj)
 {

hash.c

@@ -111,7 +111,31 @@ enum ruby_rarray_flags {
      * store array elements.  It was a bad idea to expose this to them.
      */
     RARRAY_EMBED_LEN_MASK  = RUBY_FL_USER9 | RUBY_FL_USER8 | RUBY_FL_USER7 | RUBY_FL_USER6 |
-                                 RUBY_FL_USER5 | RUBY_FL_USER4 | RUBY_FL_USER3
+                                 RUBY_FL_USER5 | RUBY_FL_USER4 | RUBY_FL_USER3,
+
+    /**
+     * Width of one element in the backend storage.  Zero means the storage is
+     * an array of ::VALUE, which is the only representation 3rd parties can
+     * see.  A non-zero value means the elements are stored unboxed, narrower
+     * than a ::VALUE, and the array must be widened before its buffer is
+     * handed out.  See ::rb_ary_widen.
+     *
+     * @internal
+     *
+     * 3rd parties must not be aware that there even is more than one way to
+     * store array elements.  It was a bad idea to expose this to them.
+     */
+    RARRAY_STRIDE_MASK     = RUBY_FL_USER11 | RUBY_FL_USER10,
+
+    /**
+     * Set when a narrowed array's unboxed elements are to be read as unsigned.
+     * Doubles the non-negative range a given width can hold.
+     *
+     * @internal
+     *
+     * 3rd parties must not be aware of this.
+     */
+    RARRAY_STRIDE_UNSIGNED = RUBY_FL_USER2
 };
 
 /**
@@ -120,9 +144,43 @@ enum ruby_rarray_flags {
  */
 enum ruby_rarray_consts {
     /** Where ::RARRAY_EMBED_LEN_MASK resides. */
-    RARRAY_EMBED_LEN_SHIFT = RUBY_FL_USHIFT + 3
+    RARRAY_EMBED_LEN_SHIFT = RUBY_FL_USHIFT + 3,
+
+    /** Where ::RARRAY_STRIDE_MASK resides. */
+    RARRAY_STRIDE_SHIFT    = RUBY_FL_USHIFT + 10
+};
+
+/**
+ * Element widths an array can use, in bits.  ::RARRAY_STRIDE_VALUE is the
+ * default: elements are ::VALUE.
+ */
+enum ruby_rarray_stride {
+    RARRAY_STRIDE_VALUE = 0,
+    RARRAY_STRIDE_W8    = 1,
+    RARRAY_STRIDE_W16   = 2,
+    RARRAY_STRIDE_W32   = 3
 };
 
+/**
+ * @private
+ *
+ * True when a narrowed array's elements are unsigned.
+ */
+static inline int
+rb_array_stride_unsigned_p(VALUE a)
+{
+    return RB_FL_ANY_RAW(a, RARRAY_STRIDE_UNSIGNED) ? 1 : 0;
+}
+
+/**
+ * Converts a narrowed array back to ::VALUE storage, in place.  A no-op for an
+ * array that already uses ::RARRAY_STRIDE_VALUE.  Allocates, so it must not be
+ * called while allocation is forbidden, for instance during GC marking.
+ *
+ * @param[out]  ary  An object of ::RArray.
+ */
+void rb_ary_widen(VALUE ary);
+
 /** Ruby's array. */
 struct RArray {
 
@@ -282,13 +340,30 @@ RARRAY_LENINT(VALUE ary)
     return rb_long2int(RARRAY_LEN(ary));
 }
 
-RBIMPL_ATTR_PURE_UNLESS_DEBUG()
+/**
+ * @private
+ *
+ * The element width currently used by the backend storage.
+ *
+ * @param[in]  a  An object of ::RArray.
+ * @return     One of ::ruby_rarray_stride.
+ */
+static inline int
+rb_array_stride(VALUE a)
+{
+    RBIMPL_ASSERT_TYPE(a, RUBY_T_ARRAY);
+    return (int)((RBASIC(a)->flags & RARRAY_STRIDE_MASK) >> RARRAY_STRIDE_SHIFT);
+}
+
 /**
  * @private
  *
  * This is  an implementation  detail of  RARRAY_PTR().  People  do not  use it
  * directly.
  *
+ * This is deliberately not pure: a narrowed array is widened in place before
+ * its buffer is returned, because every caller expects ::VALUE elements.
+ *
  * @param[in]  a  An object of ::RArray.
  * @return     Its backend storage.
  */
@@ -297,6 +372,10 @@ rb_array_const_ptr(VALUE a)
 {
     RBIMPL_ASSERT_TYPE(a, RUBY_T_ARRAY);
 
+    if (RB_UNLIKELY(rb_array_stride(a) != RARRAY_STRIDE_VALUE)) {
+        rb_ary_widen(a);
+    }
+
     if (RB_FL_ANY_RAW(a, RARRAY_EMBED_FLAG)) {
         return FIX_CONST_VALUE_PTR(RARRAY(a)->as.ary);
     }

include/ruby/internal/core/rarray.h

@@ -23,6 +23,79 @@
 #define RARRAY_PTR_IN_USE_FLAG  FL_USER14
 #define RARRAY_FAKEARY          FL_USER19
 
+/* Address of a narrowed array's packed buffer.  Only the narrow decode path
+ * may use this; everything else goes through RARRAY_CONST_PTR, which widens.
+ * Narrow arrays always own a heap buffer: embedded arrays are never narrowed. */
+static inline const void *
+rarray_raw_ptr(VALUE ary)
+{
+    RUBY_ASSERT(rb_array_stride(ary) != RARRAY_STRIDE_VALUE);
+    RUBY_ASSERT(!FL_TEST_RAW(ary, RARRAY_EMBED_FLAG));
+    return (const void *)RARRAY(ary)->as.heap.ptr;
+}
+
+/* Load element i of a narrowed array as a C integer.  This is the single
+ * place that knows the width/sign encoding.  It serves any strategy whose
+ * unboxed payload is an integer; a strategy with a different payload type
+ * would add a sibling loader. */
+static inline long
+rarray_narrow_load_int(VALUE ary, long i)
+{
+    const void *p = rarray_raw_ptr(ary);
+    if (rb_array_stride_unsigned_p(ary)) {
+        switch (rb_array_stride(ary)) {
+          case RARRAY_STRIDE_W8:  return (long)((const uint8_t  *)p)[i];
+          case RARRAY_STRIDE_W16: return (long)((const uint16_t *)p)[i];
+          default:                return (long)((const uint32_t *)p)[i];
+        }
+    }
+    switch (rb_array_stride(ary)) {
+      case RARRAY_STRIDE_W8:  return ((const int8_t  *)p)[i];
+      case RARRAY_STRIDE_W16: return ((const int16_t *)p)[i];
+      default:                return ((const int32_t *)p)[i];
+    }
+}
+
+/* Decode element i of a narrowed array into a ::VALUE.  This is where the
+ * per-strategy knowledge lives: how to rebuild a VALUE from the payload. */
+static inline VALUE
+rarray_narrow_aref(VALUE ary, long i)
+{
+    return LONG2FIX(rarray_narrow_load_int(ary, i));
+}
+
+/* Core's RARRAY_AREF replaces the public macro (which must stay an lvalue-
+ * capable macro for extensions).  It adds a bounds assertion, and it is
+ * narrow-aware: an element of a narrowed array is decoded in place rather
+ * than widening the whole array, so read-only iteration (each, map, sum,
+ * index, ==, ...) leaves a narrowed array narrow.  Never allocates, so it is
+ * also safe where RARRAY_CONST_PTR is not (opt_aref, GC callbacks).
+ *
+ * Not PURE: it is still a pure function of (ary, i) in the narrow case, but
+ * the wide case reads through RARRAY_CONST_PTR, which is not pure either. */
+#undef RARRAY_AREF
+RBIMPL_ATTR_ARTIFICIAL()
+static inline VALUE
+RARRAY_AREF(VALUE ary, long i)
+{
+    VALUE val;
+    RBIMPL_ASSERT_TYPE(ary, RUBY_T_ARRAY);
+
+    RUBY_ASSERT(i < RARRAY_LEN(ary));
+
+    if (RB_UNLIKELY(rb_array_stride(ary) != RARRAY_STRIDE_VALUE)) {
+        return rarray_narrow_aref(ary, i);
+    }
+
+    RBIMPL_WARNING_PUSH();
+#if defined(__GNUC__) && !defined(__clang__) && __GNUC__ == 13
+    RBIMPL_WARNING_IGNORED(-Warray-bounds);
+#endif
+    val = RARRAY_CONST_PTR(ary)[i];
+    RBIMPL_WARNING_POP();
+    return val;
+}
+
 /* array.c */
 VALUE rb_ary_hash_values(long len, const VALUE *elements);
 VALUE rb_ary_last(int, const VALUE *, VALUE);
@@ -34,6 +107,11 @@ size_t rb_ary_memsize(VALUE);
 VALUE rb_to_array_type(VALUE obj);
 VALUE rb_to_array(VALUE obj);
 void rb_ary_cancel_sharing(VALUE ary);
+/* Pack a heap array whose elements can all be stored unboxed to the narrowest
+ * element width that holds every value.  Returns the resulting stride;
+ * RARRAY_STRIDE_VALUE means the array was left as-is.  Used by the freeze
+ * trigger and by the compiler for frozen array literals. */
+int rb_ary_narrow(VALUE ary);
 size_t rb_ary_size_as_embedded(VALUE ary);
 void rb_ary_make_embedded(VALUE ary);
 bool rb_ary_embeddable_p(VALUE ary);
@@ -59,7 +137,6 @@ static inline VALUE
 rb_ary_entry_internal(VALUE ary, long offset)
 {
     long len = RARRAY_LEN(ary);
-    const VALUE *ptr = RARRAY_CONST_PTR(ary);
     if (len == 0) return Qnil;
     if (offset < 0) {
         offset += len;
@@ -68,7 +145,7 @@ rb_ary_entry_internal(VALUE ary, long offset)
     else if (len <= offset) {
         return Qnil;
     }
-    return ptr[offset];
+    return RARRAY_AREF(ary, offset); /* narrow-aware; does not allocate */
 }
 
 static inline bool
@@ -135,24 +212,6 @@ ARY_SHARED_ROOT_REFCNT(VALUE ary)
     })
 #endif
 
-#undef RARRAY_AREF
-RBIMPL_ATTR_PURE_UNLESS_DEBUG()
-RBIMPL_ATTR_ARTIFICIAL()
-static inline VALUE
-RARRAY_AREF(VALUE ary, long i)
-{
-    VALUE val;
-    RBIMPL_ASSERT_TYPE(ary, RUBY_T_ARRAY);
-
-    RUBY_ASSERT(i < RARRAY_LEN(ary));
 
-    RBIMPL_WARNING_PUSH();
-#if defined(__GNUC__) && !defined(__clang__) && __GNUC__ == 13
-    RBIMPL_WARNING_IGNORED(-Warray-bounds);
-#endif
-    val = RARRAY_CONST_PTR(ary)[i];
-    RBIMPL_WARNING_POP();
-    return val;
-}
 
 #endif /* INTERNAL_ARRAY_H */

internal/array.h

@@ -75,6 +75,7 @@ VALUE rb_hash_default_value(VALUE hash, VALUE key);
 VALUE rb_hash_set_default(VALUE hash, VALUE ifnone);
 VALUE rb_hash_set_default_proc(VALUE hash, VALUE proc);
 long rb_dbl_long_hash(double d);
+long rb_fixnum_hash(VALUE fixnum);
 st_table *rb_init_identtable(void);
 void rb_init_existing_identtable_with_size(st_table *tbl, st_index_t size);
 st_index_t rb_any_hash(VALUE a);

internal/hash.h

@@ -1721,6 +1721,11 @@ obj_traverse_i(VALUE obj, struct obj_traverse_data *data)
         {
             rb_ary_cancel_sharing(obj);
 
+            /* Narrow storage holds unboxed values, never object references:
+             * every element would hit the RB_SPECIAL_CONST_P early return in
+             * obj_traverse_i, so there is nothing to visit.  Skip the loop. */
+            if (rb_array_stride(obj) != RARRAY_STRIDE_VALUE) break;
+
             for (int i = 0; i < RARRAY_LENINT(obj); i++) {
                 VALUE e = RARRAY_AREF(obj, i);
                 if (obj_traverse_i(e, data)) return 1;