Pin JS casing to generated Unicode data (#4321)

Author: jakebailey

internal/stringutil/_scripts/generate-unicode-data.mts

@@ -34,6 +34,7 @@ type SpecialCasingEntry = {
     codePoint: number;
     lower: number[];
     upper: number[];
+    conditionalLower: number[];
     condition: string;
 };
 
@@ -47,6 +48,11 @@ async function loadMapping(property: string): Promise<Map<number, number[]>> {
     return module.default as Map<number, number[]>;
 }
 
+async function loadSimpleMapping(property: string): Promise<Map<number, number>> {
+    const module = await import(`${PACKAGE}/${property}/code-points.js`);
+    return module.default as Map<number, number>;
+}
+
 // Group a sorted, de-duplicated run of code points into ranges sharing a
 // constant stride. The stride of each range is taken from the gap to the next
 // code point, so it never includes a code point that is not in the set; this
@@ -120,7 +126,7 @@ ${r32}
 `;
 }
 
-async function buildSpecialCasing(): Promise<SpecialCasingEntry[]> {
+async function buildSpecialCasing(simpleLowercase: Map<number, number>, simpleUppercase: Map<number, number>): Promise<SpecialCasingEntry[]> {
     // The unconditional, locale-insensitive multi-rune mappings. Each map keys a
     // code point to its full lower/upper expansion (identity when unchanged).
     const lowerMappings = await loadMapping("Special_Casing/Lowercase");
@@ -131,39 +137,52 @@ async function buildSpecialCasing(): Promise<SpecialCasingEntry[]> {
 
     const entries: SpecialCasingEntry[] = [];
 
-    const codePoints = new Set([...lowerMappings.keys(), ...upperMappings.keys()]);
+    const codePoints = new Set([...simpleLowercase.keys(), ...simpleUppercase.keys(), ...lowerMappings.keys(), ...upperMappings.keys()]);
     for (const codePoint of codePoints) {
         entries.push({
             codePoint,
-            lower: lowerMappings.get(codePoint) ?? [codePoint],
-            upper: upperMappings.get(codePoint) ?? [codePoint],
+            lower: lowerMappings.get(codePoint) ?? [simpleLowercase.get(codePoint) ?? codePoint],
+            upper: upperMappings.get(codePoint) ?? [simpleUppercase.get(codePoint) ?? codePoint],
+            conditionalLower: [codePoint],
             condition: "specialCasingConditionNone",
         });
     }
 
     for (const [codePoint, lower] of finalSigmaMappings) {
-        entries.push({
-            codePoint,
-            lower,
-            upper: upperMappings.get(codePoint) ?? [codePoint],
-            condition: "specialCasingConditionFinalSigma",
-        });
+        const entry = entries.find(entry => entry.codePoint === codePoint);
+        if (entry === undefined) {
+            entries.push({
+                codePoint,
+                lower: [simpleLowercase.get(codePoint) ?? codePoint],
+                upper: upperMappings.get(codePoint) ?? [simpleUppercase.get(codePoint) ?? codePoint],
+                conditionalLower: lower,
+                condition: "specialCasingConditionFinalSigma",
+            });
+        }
+        else {
+            entry.conditionalLower = lower;
+            entry.condition = "specialCasingConditionFinalSigma";
+        }
     }
 
     entries.sort((a, b) => a.codePoint - b.codePoint);
     return entries;
 }
 
 function renderCaseFile(entries: SpecialCasingEntry[], casedTable: RangeTable, caseIgnorableTable: RangeTable): string {
-    const mappings = entries.map(entry => `\t${goRuneLiteral(entry.codePoint)}: {lower: ${goStringLiteral(entry.lower)}, upper: ${goStringLiteral(entry.upper)}, condition: ${entry.condition}},`).join("\n");
+    const mappings = entries.map(entry => {
+        const conditionalLower = entry.condition === "specialCasingConditionFinalSigma" ? `, conditionalLower: ${goStringLiteral(entry.conditionalLower)}` : "";
+        return `\t${goRuneLiteral(entry.codePoint)}: {lower: ${goStringLiteral(entry.lower)}, upper: ${goStringLiteral(entry.upper)}${conditionalLower}, condition: ${entry.condition}},`;
+    }).join("\n");
 
     return `// Code generated by generate-unicode-data.mts. DO NOT EDIT.
 // Derived from the ${PACKAGE} package (Unicode ${UNICODE_VERSION}).
 // Includes only the locale-insensitive multi-rune mappings needed for ECMAScript
 // default casing, plus the Final_Sigma context mapping. String.prototype.toLowerCase
 // applies Final_Sigma, but Go's unicode package does not, so the caser applies it
-// from this data when in context. Go's unicode package handles the simple one-rune
-// mappings, so those are omitted here.
+// from this data when in context. Simple one-rune mappings are included here too
+// so casing stays pinned to this Unicode version, rather than the Go toolchain's
+// unicode tables.
 
 package stringutil
 
@@ -177,9 +196,10 @@ const (
 )
 
 type specialCasingMapping struct {
-\tlower     string
-\tupper     string
-\tcondition specialCasingCondition
+\tlower            string
+\tupper            string
+\tconditionalLower string
+\tcondition        specialCasingCondition
 }
 
 var specialCasingMappings = map[rune]specialCasingMapping{
@@ -209,7 +229,9 @@ ${renderRangeTable("unicodeESNextIdentifierPart", partTable)}
 }
 
 async function main() {
-    const entries = await buildSpecialCasing();
+    const simpleLowercase = await loadSimpleMapping("Simple_Case_Mapping/Lowercase");
+    const simpleUppercase = await loadSimpleMapping("Simple_Case_Mapping/Uppercase");
+    const entries = await buildSpecialCasing(simpleLowercase, simpleUppercase);
     const casedTable = toRangeTable(await loadCodePoints("Binary_Property/Cased"));
     const caseIgnorableTable = toRangeTable(await loadCodePoints("Binary_Property/Case_Ignorable"));
     fs.writeFileSync(CASE_OUTPUT_PATH, renderCaseFile(entries, casedTable, caseIgnorableTable));

internal/stringutil/js_case.go

@@ -26,13 +26,13 @@ func ToLowerJS(str string) string {
 			// bytes because WriteRune would re-encode the surrogate as U+FFFD.
 			builder.WriteString(EncodeJSStringRune(r))
 		} else if mapping, ok := specialCasingMappings[r]; ok {
-			if mapping.condition == specialCasingConditionFinalSigma && !isFinalSigmaContext(casedBefore, str, i) {
-				builder.WriteRune(unicode.ToLower(r))
+			if mapping.condition == specialCasingConditionFinalSigma && isFinalSigmaContext(casedBefore, str, i) {
+				builder.WriteString(mapping.conditionalLower)
 			} else {
 				builder.WriteString(mapping.lower)
 			}
 		} else {
-			builder.WriteRune(unicode.ToLower(r))
+			builder.WriteRune(r)
 		}
 		if !isUnicodeCaseIgnorable(r) {
 			casedBefore = isSigmaCased(r)
@@ -58,10 +58,11 @@ func ToUpperJS(str string) string {
 		} else if mapping, ok := specialCasingMappings[r]; ok {
 			builder.WriteString(mapping.upper)
 		} else {
-			builder.WriteRune(unicode.ToUpper(r))
+			builder.WriteRune(r)
 		}
 		i += size
 	}
+
 	return builder.String()
 }