Skip to content

Commit 899d013

Browse files
schuayRenegade334
authored andcommitted
deps: V8: backport f607706154ea
Original commit message: [regexp] Handle supplementary case equivalents of BMP characters Unicode 18 adds U+1DF95, which simple-folds to U+00DF. This creates the first case-equivalence class spanning BMP and supplementary characters: {U+00DF, U+1E9E, U+1DF95}. Discard supplementary characters from the generator's non-unicode closures before calling Canonicalize, which requires UTF-16 code units and otherwise CHECK-fails with ICU 79. Apply the same filtering in runtime non-unicode closure. Generate SupplementaryEquivalentsSet and desugar its BMP characters into character classes under /iu and /iv, allowing literals to match surrogate pairs. Backreferences with differing UTF-16 character widths remain unsupported and are tracked in crbug.com/571793821. IgnoreSet is unchanged with ICU 78 and with ICU 78.2 patched to upstream Unicode 18 case data. With the patched data, the generator succeeds, SupplementaryEquivalentsSet is {U+00DF, U+1E9E}, and regexp tests pass in release and debug. Bug: 571232615, 571668686 Change-Id: I9eacb273a74c66e17f0cfb1b256e863f2718c130 Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/8548134 Reviewed-by: Patrick Thier <pthier@chromium.org> Commit-Queue: Jakob Linke <jgruber@chromium.org> Cr-Commit-Position: refs/heads/main@{#110522} Refs: v8/v8@f607706 Co-authored-by: Renegade334 <rene@renegade334.me.uk> PR-URL: #67527 Reviewed-By: Antoine du Hamel <duhamelantoine1995@gmail.com>
1 parent 7d0a65d commit 899d013

6 files changed

Lines changed: 144 additions & 14 deletions

File tree

‎common.gypi‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@
5050

5151
# Reset this number to 0 on major V8 upgrades.
5252
# Increment by one for each non-official patch applied to deps/v8.
53-
'v8_embedder_string': '-node.56',
53+
'v8_embedder_string': '-node.57',
5454

5555
##### V8 defaults for Node.js #####
5656

‎deps/v8/src/regexp/gen-regexp-special-case.cc‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
#include "src/base/strings.h"
1111
#include "src/regexp/special-case.h"
1212
#include "unicode/locid.h"
13+
#include "unicode/uchar.h"
1314
#include "unicode/unistr.h"
1415

1516
namespace v8 {
@@ -51,6 +52,19 @@ UChar32 Canonicalize(UChar32 ch) {
5152
return cu;
5253
}
5354

55+
// Removes supplementary code points from |set| and returns whether any were
56+
// present. Since Unicode 18, the case closure of a BMP character can contain
57+
// supplementary code points (U+1DF95 LATIN SMALL LIGATURE LONG S WITH
58+
// DESCENDER S case-folds to U+00DF LATIN SMALL LETTER SHARP S). Non-unicode
59+
// ignoreCase matching compares UTF-16 code units, so a supplementary code
60+
// point never matches a BMP character and does not take part in Canonicalize.
61+
bool RemoveSupplementary(icu::UnicodeSet& set) {
62+
constexpr UChar32 kFirstSupplementary = static_cast<UChar32>(kNonBmpStart);
63+
if (!set.containsSome(kFirstSupplementary, UCHAR_MAX_VALUE)) return false;
64+
set.remove(kFirstSupplementary, UCHAR_MAX_VALUE);
65+
return true;
66+
}
67+
5468
// The following code generates "src/regexp/special-case.cc".
5569
void PrintSet(std::ofstream& out, const char* name,
5670
const icu::UnicodeSet& set) {
@@ -85,6 +99,7 @@ void PrintSpecial(std::ofstream& out) {
8599
icu::UnicodeSet current;
86100
icu::UnicodeSet special_add;
87101
icu::UnicodeSet ignore;
102+
icu::UnicodeSet supplementary_equivalents;
88103
UErrorCode status = U_ZERO_ERROR;
89104
icu::UnicodeSet upper("[\\p{Lu}]", status);
90105
CHECK(U_SUCCESS(status));
@@ -97,6 +112,7 @@ void PrintSpecial(std::ofstream& out) {
97112
}
98113
current.set(i, i);
99114
current.closeOver(USET_SIMPLE_CASE_INSENSITIVE);
115+
if (RemoveSupplementary(current)) supplementary_equivalents.add(i);
100116

101117
// Check to see if all characters in the case-folding equivalence
102118
// class as defined by UnicodeSet::closeOver all map to the same
@@ -146,6 +162,7 @@ void PrintSpecial(std::ofstream& out) {
146162
UChar32 canonical = Canonicalize(c);
147163
current.set(c, c);
148164
current.closeOver(USET_SIMPLE_CASE_INSENSITIVE);
165+
RemoveSupplementary(current);
149166
current.removeAll(ignore);
150167
for (int32_t j = 0; j < current.getRangeCount(); j++) {
151168
for (UChar32 c2 = current.getRangeStart(j);
@@ -157,6 +174,7 @@ void PrintSpecial(std::ofstream& out) {
157174
}
158175

159176
PrintSet(out, "IgnoreSet", ignore);
177+
PrintSet(out, "SupplementaryEquivalentsSet", supplementary_equivalents);
160178
}
161179

162180
void WriteHeader(const char* header_filename) {

‎deps/v8/src/regexp/regexp-parser.cc‎

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
#include "src/regexp/regexp-ast.h"
1111
#include "src/regexp/regexp-macro-assembler.h"
1212
#include "src/regexp/regexp.h"
13+
#include "src/regexp/special-case.h"
1314
#include "src/strings/char-predicates-inl.h"
1415
#include "src/utils/ostreams.h"
1516
#include "src/utils/utils.h"
@@ -107,6 +108,10 @@ void RegExpTextBuilder::FlushText() {
107108
}
108109

109110
void RegExpTextBuilder::AddCharacter(base::uc16 c) {
111+
if (NeedsDesugaringForIgnoreCase(c)) {
112+
AddClassRangesForDesugaring(c);
113+
return;
114+
}
110115
if (characters_ == nullptr) {
111116
characters_ = zone()->New<ZoneList<base::uc16>>(4, zone());
112117
}
@@ -188,15 +193,18 @@ bool RegExpTextBuilder::NeedsDesugaringForUnicode(RegExpClassRanges* cc) {
188193
return false;
189194
}
190195

191-
// We only use this for characters made of surrogate pairs. All other
192-
// characters outside of character classes are made case independent in the
193-
// code generation.
196+
// Desugar supplementary characters with case equivalents and BMP characters
197+
// with supplementary equivalents. Other atoms are folded during code
198+
// generation.
194199
bool RegExpTextBuilder::NeedsDesugaringForIgnoreCase(base::uc32 c) {
195200
#ifdef V8_INTL_SUPPORT
196201
if (IsUnicodeMode() && ignore_case()) {
202+
if (c <= unibrow::Utf16::kMaxNonSurrogateCharCode) {
203+
return RegExpCaseFolding::HasSupplementaryEquivalents(
204+
static_cast<base::uc16>(c));
205+
}
197206
icu::UnicodeSet set(c, c);
198-
set.closeOver(USET_CASE_INSENSITIVE);
199-
set.removeAllStrings();
207+
RegExpCaseFolding::CloseOver(set, RegExpCaseFolding::Mode::kUnicode);
200208
return set.size() > 1;
201209
}
202210
// In the case where ICU is not included, we act as if the unicode flag is

‎deps/v8/src/regexp/special-case.h‎

Lines changed: 23 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,8 @@
77

88
#ifdef V8_INTL_SUPPORT
99
#include "src/base/logging.h"
10+
#include "src/base/strings.h"
1011
#include "src/common/globals.h"
11-
1212
#include "unicode/uchar.h"
1313
#include "unicode/uniset.h"
1414

@@ -52,6 +52,12 @@ namespace internal {
5252
// another character. Characters that match no other characters in
5353
// their equivalence class are added to IgnoreSet. The generator verifies
5454
// that each Unicode class has at most one non-trivial JS class.
55+
// Supplementary code points are ignored throughout: non-unicode matching
56+
// compares UTF-16 code units, so they can never match a BMP character.
57+
//
58+
// The generator additionally emits SupplementaryEquivalentsSet, the BMP
59+
// characters whose Unicode case closure contains supplementary code points.
60+
// This set is used for unicode-mode matching, see HasSupplementaryEquivalents.
5561

5662
class V8_EXPORT_PRIVATE RegExpCaseFolding final : public AllStatic {
5763
public:
@@ -74,9 +80,9 @@ class V8_EXPORT_PRIVATE RegExpCaseFolding final : public AllStatic {
7480
set.closeOver(USET_SIMPLE_CASE_INSENSITIVE);
7581
return;
7682
}
83+
DCHECK(!set.containsSome(0x10000, 0x10ffff));
7784
if (IgnoreSet().containsNone(set)) {
78-
set.closeOver(USET_SIMPLE_CASE_INSENSITIVE);
79-
set.removeAll(IgnoreSet());
85+
CloseOverNonUnicode(set);
8086
return;
8187
}
8288
if (IgnoreSet().containsAll(set)) return;
@@ -85,13 +91,25 @@ class V8_EXPORT_PRIVATE RegExpCaseFolding final : public AllStatic {
8591
icu::UnicodeSet ignored(set);
8692
ignored.retainAll(IgnoreSet());
8793
set.removeAll(IgnoreSet());
88-
set.closeOver(USET_SIMPLE_CASE_INSENSITIVE);
89-
set.removeAll(IgnoreSet());
94+
CloseOverNonUnicode(set);
9095
set.addAll(ignored);
9196
}
9297

98+
// Whether c has supplementary Unicode case equivalents.
99+
static bool HasSupplementaryEquivalents(base::uc16 c) {
100+
return SupplementaryEquivalentsSet().contains(c);
101+
}
102+
93103
private:
94104
static const icu::UnicodeSet& IgnoreSet();
105+
static const icu::UnicodeSet& SupplementaryEquivalentsSet();
106+
107+
// Requires a set containing no IgnoreSet characters.
108+
static void CloseOverNonUnicode(icu::UnicodeSet& set) {
109+
set.closeOver(USET_SIMPLE_CASE_INSENSITIVE);
110+
set.removeAll(IgnoreSet());
111+
set.remove(0x10000, 0x10ffff);
112+
}
95113
};
96114

97115
} // namespace internal

‎deps/v8/test/intl/regexp-case-equivalence.js‎

Lines changed: 60 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,14 @@
22
// Use of this source code is governed by a BSD-style license that can be
33
// found in the LICENSE file.
44

5-
// Exercise literals, ranges, and backreferences with the same case pairs.
5+
// Unicode 18 assigns U+1DF95 LATIN SMALL LIGATURE LONG S WITH DESCENDER S,
6+
// the first supplementary character that case-folds to a BMP character
7+
// (U+00DF). It is thus case-equivalent to U+00DF and U+1E9E under /iu and
8+
// /iv, but never under /i, which compares UTF-16 code units.
9+
const hasUnicode18 = '\u{1DF95}'.toUpperCase() === 'SS';
10+
11+
// Exercise literals, ranges, backreferences, lookbehinds, and class strings
12+
// with the same case pairs.
613
const pairs = [
714
['a', 'A', true, true],
815
['k', '\u212a', false, true],
@@ -12,6 +19,8 @@ const pairs = [
1219
['i', '\u0130', false, false],
1320
['I', '\u0131', false, false],
1421
['\u{10400}', '\u{10428}', false, true],
22+
['\u00df', '\u{1DF95}', false, hasUnicode18],
23+
['\u1e9e', '\u{1DF95}', false, hasUnicode18],
1524
['\ud800', '\ud800', true, true],
1625
['\ud800', '\ud801', false, false],
1726
];
@@ -29,16 +38,64 @@ for (const [a, b, legacy, unicode] of pairs) {
2938
? new RegExp('^' + prefix + '[' + c1 + '-' + c1 + ']$', flags)
3039
: null;
3140
assertEquals(expected, literal.test(prefix + c2), literal.toString());
32-
assertEquals(expected, backref.test(prefix + c1 + c2),
33-
backref.toString());
41+
// TODO(crbug.com/571793821): Case-insensitive backreferences do not
42+
// yet match captures of a different UTF-16 length (U+00DF vs U+1DF95).
43+
if (c1.length === c2.length) {
44+
assertEquals(expected, backref.test(prefix + c1 + c2),
45+
backref.toString());
46+
}
3447
if (range !== null) {
3548
assertEquals(expected, range.test(prefix + c2), range.toString());
3649
}
50+
const lookbehind = new RegExp('(?<=^' + prefix + c1 + ')x$', flags);
51+
assertEquals(expected, lookbehind.test(prefix + c2 + 'x'),
52+
lookbehind.toString());
53+
if (flags === 'iv') {
54+
const classString = new RegExp(
55+
'^' + prefix + '[\\q{' + c1 + 'x}]$', flags);
56+
assertEquals(expected, classString.test(prefix + c2 + 'x'),
57+
classString.toString());
58+
}
3759
}
3860
}
3961
}
4062
}
4163

64+
// Supplementary case equivalents of BMP characters must be matched in
65+
// unicode mode both as plain pattern characters and inside character
66+
// classes, but never in legacy mode.
67+
if (hasUnicode18) {
68+
const S = '\u{1DF95}';
69+
const cases = [
70+
// [pattern, subject, expected]
71+
['^\u00df$', S, true],
72+
['^\u1e9e$', S, true],
73+
['^' + S + '$', '\u00df', true],
74+
['^' + S + '$', '\u1e9e', true],
75+
['^\\u00df$', S, true],
76+
['^\\u{df}$', S, true],
77+
['^a\u00dfb$', 'A' + S + 'B', true],
78+
['^(?:\u00df)+$', S + '\u00df\u1e9e' + S, true],
79+
['^\u00df{2}$', S + '\u1e9e', true],
80+
['^\u00df{2}$', S, false],
81+
['^(?:x|\u00df)$', S, true],
82+
['^[\u00df]$', S, true],
83+
['^[^\u00df]$', S, false],
84+
['^(\u00df)$', S, true],
85+
];
86+
for (const [pattern, subject, expected] of cases) {
87+
for (const flags of ['iu', 'iv']) {
88+
const re = new RegExp(pattern, flags);
89+
assertEquals(expected, re.test(subject), re.toString() + ' ' + subject);
90+
}
91+
// In legacy mode, the astral character consists of two code units,
92+
// neither of which has case equivalents.
93+
const re = new RegExp(pattern, 'i');
94+
assertFalse(re.test(subject), re.toString() + ' ' + subject);
95+
}
96+
assertEquals([S, S], /(\u00df)/iu.exec(S));
97+
}
98+
4299
for (const flags of ['i', 'iu', 'iv']) {
43100
const re = new RegExp('^[a-c\u017f\u00df]$', flags);
44101
for (const c of ['a', 'B', 'c', '\u017f', '\u00df']) {

‎deps/v8/test/unittests/regexp/regexp-unittest.cc‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1392,6 +1392,31 @@ TEST_F(RegExpTest, NonUnicodeCaseEquivalence) {
13921392
ASSERT_TRUE(expected == actual) << c;
13931393
}
13941394
}
1395+
1396+
TEST_F(RegExpTest, SupplementaryCaseEquivalents) {
1397+
icu::UnicodeSet expected;
1398+
for (UChar32 c = 0; c <= 0xffff; ++c) {
1399+
icu::UnicodeSet closure(c, c);
1400+
closure.closeOver(USET_SIMPLE_CASE_INSENSITIVE);
1401+
const bool has_supplementary = closure.containsSome(0x10000, 0x10ffff);
1402+
if (has_supplementary) expected.add(c);
1403+
const bool actual = RegExpCaseFolding::HasSupplementaryEquivalents(
1404+
static_cast<base::uc16>(c));
1405+
ASSERT_EQ(has_supplementary, actual) << c;
1406+
1407+
// Non-unicode closure must never yield supplementary code points.
1408+
icu::UnicodeSet non_unicode(c, c);
1409+
RegExpCaseFolding::CloseOver(non_unicode,
1410+
RegExpCaseFolding::Mode::kNonUnicode);
1411+
ASSERT_FALSE(non_unicode.containsSome(0x10000, 0x10ffff)) << c;
1412+
}
1413+
// U+1DF95 LATIN SMALL LIGATURE LONG S WITH DESCENDER S (Unicode 18)
1414+
// case-folds to U+00DF LATIN SMALL LETTER SHARP S.
1415+
if (u_foldCase(0x1df95, U_FOLD_CASE_DEFAULT) == 0xdf) {
1416+
EXPECT_TRUE(expected.contains(0xdf));
1417+
EXPECT_TRUE(expected.contains(0x1e9e));
1418+
}
1419+
}
13951420
#endif // !DEBUG
13961421

13971422
TEST_F(RegExpTest, CaseClosureMixedSets) {
@@ -1403,6 +1428,10 @@ TEST_F(RegExpTest, CaseClosureMixedSets) {
14031428
expected.add('A', 'C').add('K');
14041429
if (mode == RegExpCaseFolding::Mode::kUnicode) {
14051430
expected.add('s').add('S').add(0x212a).add(0x1e9e);
1431+
// Since Unicode 18, U+1DF95 case-folds to U+00DF.
1432+
if (u_foldCase(0x1df95, U_FOLD_CASE_DEFAULT) == 0xdf) {
1433+
expected.add(0x1df95);
1434+
}
14061435
actual.add(0x10400);
14071436
expected.add(0x10400).add(0x10428);
14081437
}

0 commit comments

Comments
 (0)