Fix UTF-8 offset in the STL regex pre-tokenizer fallback

operators/tokenizer/bpe_utils.hpp

@@ -900,8 +900,24 @@ class PreTokenizerWithRegEx {
     auto& text = m_utf8_text;
     if (std::regex_search(text, match, *fallback_patterns_)) {
       ustring res(match[0]);
-      auto pos = m_text.find(res);
-      m_utf8_text = m_utf8_text.substr(pos + match[0].str().size());
+      // match.position() is a byte offset into the UTF-8 copy; m_text holds
+      // code points, so count the code points (non-continuation bytes) that
+      // precede the match. Using one offset for both copies desyncs them as
+      // soon as a multi-byte character precedes a match.
+      const size_t byte_pos = static_cast<size_t>(match.position(0));
+      const size_t byte_len = static_cast<size_t>(match.length(0));
+      // Callers treat an empty view as no match and then consume the next
+      // character themselves, so a zero-length match must not move either copy.
+      if (byte_len == 0) {
+        return {};
+      }
+      size_t pos = 0;
+      for (size_t b = 0; b < byte_pos; ++b) {
+        if ((static_cast<unsigned char>(text[b]) & 0xC0) != 0x80) {
+          ++pos;
+        }
+      }
+      m_utf8_text = m_utf8_text.substr(byte_pos + byte_len);
       auto res_view = m_text.substr(pos, res.size());
       m_text = m_text.substr(pos + res.size());
       return res_view;

test/pp_api_test/test_tokenizer_impl.cc

@@ -79,6 +79,59 @@ TEST(OrtxTokenizerTest, Qwen35RegexTest) {
   EXPECT_EQ(res, expected);
 }
 
+TEST(OrtxTokenizerTest, FallbackRegexAfterMultiByteChar) {
+  // This pattern has no hardcoded matcher, so it runs through std::regex on a
+  // UTF-8 copy of the text. Multi-byte characters before each match must not
+  // desync that copy from the code-point text.
+  ort_extensions::bpe::PreTokenizerWithRegEx splitter;
+  auto status = splitter.Compile(R"((?:\r?\n)+(?!\r?\n))");
+  ASSERT_TRUE(status.IsOk()) << status.ToString();
+
+  auto chunks = splitter.SplitIsolated(U"caf\u00e9\nnext\n\u4e2d\u6587\nend");
+  std::vector<std::u32string> expected = {U"caf\u00e9", U"\n", U"next", U"\n", U"\u4e2d\u6587", U"\n", U"end"};
+  EXPECT_EQ(chunks, expected);
+}
+
+TEST(OrtxTokenizerTest, FallbackRegexZeroLengthMatchSplit) {
+  // A zero-length fallback match after the start of the text must not consume
+  // the characters before it: they stay in the unmatched chunk.
+  ort_extensions::bpe::PreTokenizerWithRegEx end_splitter;
+  auto status = end_splitter.Compile("$");
+  ASSERT_TRUE(status.IsOk()) << status.ToString();
+  EXPECT_EQ(end_splitter.SplitIsolated(U"ab"), std::vector<std::u32string>{U"ab"});
+
+  ort_extensions::bpe::PreTokenizerWithRegEx lookahead_splitter;
+  status = lookahead_splitter.Compile(R"((?=\n))");
+  ASSERT_TRUE(status.IsOk()) << status.ToString();
+  EXPECT_EQ(lookahead_splitter.SplitIsolated(U"caf\u00e9\nend"), std::vector<std::u32string>{U"caf\u00e9\nend"});
+}
+
+TEST(OrtxTokenizerTest, FallbackRegexZeroLengthMatchNextToken) {
+  auto tokens_of = [](ort_extensions::bpe::PreTokenizerWithRegEx& splitter, std::u32string_view text) {
+    std::vector<std::u32string> tokens;
+    splitter.Set(text);
+    for (auto tok = splitter.GetNextToken(); !tok.empty(); tok = splitter.GetNextToken()) {
+      tokens.emplace_back(tok);
+    }
+    return tokens;
+  };
+
+  ort_extensions::bpe::PreTokenizerWithRegEx end_splitter;
+  auto status = end_splitter.Compile("$");
+  ASSERT_TRUE(status.IsOk()) << status.ToString();
+  EXPECT_EQ(tokens_of(end_splitter, U"ab"), (std::vector<std::u32string>{U"a", U"b"}));
+
+  ort_extensions::bpe::PreTokenizerWithRegEx lookahead_splitter;
+  status = lookahead_splitter.Compile(R"((?=\n))");
+  ASSERT_TRUE(status.IsOk()) << status.ToString();
+  std::u32string text = U"caf\u00e9\nend";
+  std::vector<std::u32string> expected;
+  for (char32_t ch : text) {
+    expected.emplace_back(1, ch);
+  }
+  EXPECT_EQ(tokens_of(lookahead_splitter, text), expected);
+}
+
 TEST(OrtxTokenizerTest, AddedTokensTest) {
   auto tokenizer = std::make_unique<ort_extensions::TokenizerImpl>();
   auto status = tokenizer->Load("data/added-tokens");