Read escaped surrogate pairs in JSON strings

src/serde/json/reader/JsonReaderHelper.Unescaping.cs

@@ -545,10 +545,12 @@ out int written
                             "invalid escape sequences must have already been caught by Utf8JsonReader.Read()"
                         );
 
-                        if (!DecodeUnicodeEscape(source, destination, ref idx, ref written))
+                        int digitIdx = idx + 1;
+                        if (!DecodeUnicodeEscape(source, destination, ref digitIdx, ref written))
                         {
                             goto DestinationTooShort;
                         }
+                        idx = digitIdx - 1;
                         break;
                 }
 
@@ -659,15 +661,15 @@ ref int written
                 );
 
                 // We must have a low surrogate following a high surrogate.
-                if (source.Length < idx + 7 || source[idx + 1] != '\\' || source[idx + 2] != 'u')
+                if (source.Length < idx + 6 || source[idx] != '\\' || source[idx + 1] != 'u')
                 {
                     ThrowHelper.ThrowInvalidOperationException_ReadIncompleteUTF16();
                 }
 
                 // The source is known to be valid JSON, and hence if we see a \u, it is guaranteed to have 4 hex digits following it
                 // Otherwise, the Utf8JsonReader would have already thrown an exception.
                 result = Utf8Parser.TryParse(
-                    source.Slice(idx + 3, 4),
+                    source.Slice(idx + 2, 4),
                     out int lowSurrogate,
                     out bytesConsumed,
                     'x'

test/Serde.Test/JsonDeserializeTests.cs

@@ -59,6 +59,20 @@ public void DeserializeObject()
             );
         }
 
+        [Fact]
+        public void DeserializeEscapedSurrogatePairs()
+        {
+            var value = new JsonValue.String("a🤖🌍z");
+            Assert.Equal(
+                value,
+                JsonSerializer.DeserializeJsonValue("\"a\\uD83E\\uDD16\\uD83C\\uDF0Dz\"")
+            );
+            Assert.Equal(
+                value,
+                JsonSerializer.DeserializeJsonValue(JsonSerializer.Serialize<JsonValue>(value))
+            );
+        }
+
         [Fact]
         public void DeserializeInt()
         {

test/Serde.Test/RoundtripTests.cs

@@ -31,6 +31,23 @@ public void TestArray()
         );
     }
 
+    [GenerateSerde]
+    private partial record UnicodeNote
+    {
+        public required string Text { get; init; }
+    }
+
+    [Fact]
+    public void EscapedSurrogatePairsRoundtrip()
+    {
+        var note = new UnicodeNote { Text = "a🤖🌍z" };
+        Assert.Equal(note, JsonSerializer.Deserialize<UnicodeNote>(JsonSerializer.Serialize(note)));
+        Assert.Equal(
+            note,
+            JsonSerializer.Deserialize<UnicodeNote>("""{"text":"a\uD83E\uDD16\uD83C\uDF0Dz"}""")
+        );
+    }
+
     [GenerateSerde]
     [SerdeTypeOptions(MemberFormat = MemberFormat.None)]
     public partial record Point

test/Serde.Test/generated/SerdeGenerator/Serde.SerdeImplRoslynGenerator/Serde.Test.RoundtripTests.UnicodeNote.ISerde.g.cs

@@ -0,0 +1,67 @@
+
+#nullable enable
+
+using System;
+using Serde;
+
+namespace Serde.Test;
+
+partial class RoundtripTests
+{
+    partial record UnicodeNote
+    {
+        sealed partial class _SerdeObj : global::Serde.ISerde<Serde.Test.RoundtripTests.UnicodeNote>
+        {
+            global::Serde.ISerdeInfo global::Serde.ISerdeInfoProvider.SerdeInfo => Serde.Test.RoundtripTests.UnicodeNote.s_serdeInfo;
+
+            void global::Serde.ISerialize<Serde.Test.RoundtripTests.UnicodeNote>.Serialize(Serde.Test.RoundtripTests.UnicodeNote value, global::Serde.ISerializer serializer)
+            {
+                var _l_info = global::Serde.SerdeInfoProvider.GetInfo(this);
+                var _l_type = serializer.WriteType(_l_info, 1);
+                _l_type.WriteString(_l_info, 0, value.Text);
+                _l_type.End(_l_info);
+            }
+            Serde.Test.RoundtripTests.UnicodeNote Serde.IDeserialize<Serde.Test.RoundtripTests.UnicodeNote>.Deserialize(IDeserializer deserializer)
+            {
+                string _l_text = default!;
+
+                byte _r_assignedValid = 0;
+
+                var _l_serdeInfo = global::Serde.SerdeInfoProvider.GetInfo(this);
+                var typeDeserialize = deserializer.ReadType(_l_serdeInfo);
+                while (true)
+                {
+                    var (_l_index_, _) = typeDeserialize.TryReadIndexWithName(_l_serdeInfo);
+                    if (_l_index_ == Serde.ITypeDeserializer.EndOfType)
+                    {
+                        break;
+                    }
+
+                    switch (_l_index_)
+                    {
+                        case 0:
+                            Serde.DeserializeException.ThrowIfDuplicate(_r_assignedValid, 0, _l_serdeInfo);
+                            _l_text = typeDeserialize.ReadString(_l_serdeInfo, _l_index_);
+                            _r_assignedValid |= ((byte)1) << 0;
+                            break;
+                        case Serde.ITypeDeserializer.IndexNotFound:
+                            typeDeserialize.SkipValue(_l_serdeInfo, _l_index_);
+                            break;
+                        default:
+                            throw new InvalidOperationException("Unexpected index: " + _l_index_);
+                    }
+                }
+                typeDeserialize.End(_l_serdeInfo);
+                if ((_r_assignedValid & 0b1) != 0b1)
+                {
+                    throw Serde.DeserializeException.UnassignedMember(_r_assignedValid, 0b1, _l_serdeInfo);
+                }
+                var newType = new Serde.Test.RoundtripTests.UnicodeNote() {
+                    Text = _l_text,
+                };
+
+                return newType;
+            }
+        }
+    }
+}

test/Serde.Test/generated/SerdeGenerator/Serde.SerdeImplRoslynGenerator/Serde.Test.RoundtripTests.UnicodeNote.ISerdeInfoProvider.g.cs

@@ -0,0 +1,18 @@
+
+#nullable enable
+
+namespace Serde.Test;
+
+partial class RoundtripTests
+{
+    partial record UnicodeNote
+    {
+        private static global::Serde.ISerdeInfo s_serdeInfo = Serde.SerdeInfo.MakeCustom(
+            "UnicodeNote",
+            typeof(Serde.Test.RoundtripTests.UnicodeNote).GetCustomAttributesData(),
+            new global::Serde.SerdeInfo.FieldInfo[] {
+                new("text", global::Serde.SerdeInfoProvider.GetSerializeInfo<string, global::Serde.StringProxy>())
+            }
+        );
+    }
+}

test/Serde.Test/generated/SerdeGenerator/Serde.SerdeImplRoslynGenerator/Serde.Test.RoundtripTests.UnicodeNote.ISerdeProvider.g.cs

@@ -0,0 +1,11 @@
+
+namespace Serde.Test;
+
+partial class RoundtripTests
+{
+    partial record UnicodeNote : Serde.ISerdeProvider<Serde.Test.RoundtripTests.UnicodeNote, Serde.Test.RoundtripTests.UnicodeNote._SerdeObj, Serde.Test.RoundtripTests.UnicodeNote>
+    {
+        static Serde.Test.RoundtripTests.UnicodeNote._SerdeObj global::Serde.ISerdeProvider<Serde.Test.RoundtripTests.UnicodeNote, Serde.Test.RoundtripTests.UnicodeNote._SerdeObj, Serde.Test.RoundtripTests.UnicodeNote>.Instance { get; }
+            = new Serde.Test.RoundtripTests.UnicodeNote._SerdeObj();
+    }
+}