Backport 4.0 [Bug #22387] Fix termlen in str_shared_replace

The buffer is copied over based on the new string's termlen
so we can just set the new encoding raw
instead of recalculating based on the old encoding's termlen.

Author: rwstauner

string.c

@@ -1808,8 +1808,6 @@ str_shared_replace(VALUE str, VALUE str2)
     if (str_embed_capa(str) >= RSTRING_LEN(str2) + termlen) {
         STR_SET_EMBED(str);
         memcpy(RSTRING_PTR(str), RSTRING_PTR(str2), (size_t)RSTRING_LEN(str2) + termlen);
-        rb_enc_associate(str, enc);
-        ENC_CODERANGE_SET(str, cr);
     }
     else {
         if (STR_EMBED_P(str2)) {
@@ -1841,9 +1839,12 @@ str_shared_replace(VALUE str, VALUE str2)
         STR_SET_EMBED(str2);
         RSTRING_PTR(str2)[0] = 0;
         STR_SET_LEN(str2, 0);
-        rb_enc_associate(str, enc);
-        ENC_CODERANGE_SET(str, cr);
     }
+
+    // We used str2's termlen above so we set enc raw
+    // to avoid adjusting it based on str1's old enc.
+    rb_enc_raw_set(str, enc);
+    ENC_CODERANGE_SET(str, cr);
 }
 
 VALUE

test/-ext-/string/test_capacity.rb

@@ -2,6 +2,7 @@
 require 'test/unit'
 require '-test-/string'
 require 'rbconfig/sizeof'
+require 'objspace'
 
 class Test_StringCapacity < Test::Unit::TestCase
   def test_capacity_embedded
@@ -74,12 +75,34 @@ def test_frozen_root_capacity_with_multibyte_terminator
     assert_equal(capacity, capa(s))
   end
 
+  def test_encode_bang_capacity_with_multibyte_terminator
+    s = smallest_slot_utf16le_string
+    assert(Bug::String.cstr_embedded?(s))
+
+    s.encode!("UTF-8")
+
+    # The UTF-8 form no longer fits in the slot so the buffer is on heap.
+    refute(Bug::String.cstr_embedded?(s))
+
+    # and sized precisely
+    assert_equal(s.bytesize, capa(s))
+  end
+
   private
 
   def capa(str)
     Bug::String.capacity(str)
   end
 
+  # A UTF-16LE string that fills the smallest slot, so that its UTF-8 form
+  # (1.5x as long) cannot stay embedded.
+  def smallest_slot_utf16le_string
+    embed_capa = pool_slot_size(0) - embed_header_size
+    str = ("\u{30AF}" * ((embed_capa - 2) / 2)).encode("UTF-16LE").b
+    str.force_encoding("UTF-16LE")
+    str
+  end
+
   def multibyte_terminator_string
     ("\u{30AF}\u{30FC}\u{30DD}\u{30F3}\u{1F381}" * 100).encode("UTF-16LE")
   end
@@ -88,6 +111,10 @@ def embed_header_size
     GC::INTERNAL_CONSTANTS[:RBASIC_SIZE] + RbConfig::SIZEOF['void*']
   end
 
+  def pool_slot_size(_idx = 0)
+    Integer(ObjectSpace.dump("")[/"slot_size":(\d+)/, 1])
+  end
+
   def max_embed_len
     GC::INTERNAL_CONSTANTS[:RVARGC_MAX_ALLOCATE_SIZE] - embed_header_size - 1
   end