fix: RegexOptions.NonBacktracking returns consistent matches with Backtracking

src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/Symbolic/SymbolicRegexNode.cs

@@ -466,6 +466,13 @@ internal static SymbolicRegexNode<TSet> CreateConcat(SymbolicRegexBuilder<TSet>
                 return CreateEffect(builder, CreateConcat(builder, left._left, right), left._right);
             }
 
+            // Maintain Concat in right associative normal form: left must never itself be a Concat.
+            if (left._kind == SymbolicRegexNodeKind.Concat)
+            {
+                Debug.Assert(left._left is not null && left._right is not null);
+                return CreateConcat(builder, left._left, CreateConcat(builder, left._right, right));
+            }
+
             return Create(builder, SymbolicRegexNodeKind.Concat, left, right, -1, -1, default, SymbolicRegexInfo.Concat(left._info, right._info));
         }
 
@@ -2305,9 +2312,17 @@ internal int CountSingletons()
                     return 1;
 
                 case SymbolicRegexNodeKind.Concat:
+                    Debug.Assert(_left is not null && _right is not null);
+                    // #(this) = #(_left) + #(_right)
+                    return Sum(_left.CountSingletons(), _right.CountSingletons());
+
                 case SymbolicRegexNodeKind.Alternate:
                     Debug.Assert(_left is not null && _right is not null);
                     // #(this) = #(_left) + #(_right)
+                    if (ContainsHighPriorityNullableUnboundedLoop(_left) || ContainsHighPriorityNullableUnboundedLoop(_right))
+                    {
+                        return Times(2, Sum(_left.CountSingletons(), _right.CountSingletons()));
+                    }
                     return Sum(_left.CountSingletons(), _right.CountSingletons());
 
                 case SymbolicRegexNodeKind.Loop:
@@ -2372,7 +2387,13 @@ internal int CountSingletons()
         /// Used by CountSingletons to detect nested unbounded loops even when captures or other
         /// structural nodes are interposed between the loops.
         /// </summary>
-        private static bool ContainsNestedUnboundedLoop(SymbolicRegexNode<TSet> node)
+        private static bool ContainsNestedUnboundedLoop(SymbolicRegexNode<TSet> node) =>
+            ContainsLoop(node, static current => current._upper == int.MaxValue);
+
+        private static bool ContainsHighPriorityNullableUnboundedLoop(SymbolicRegexNode<TSet> node) =>
+            ContainsLoop(node, static current => current._upper == int.MaxValue && current._lower == 0 && current._info.IsLazyLoop);
+
+        private static bool ContainsLoop(SymbolicRegexNode<TSet> node, Func<SymbolicRegexNode<TSet>, bool> isMatch)
         {
             Stack<SymbolicRegexNode<TSet>> stack = new();
             stack.Push(node);
@@ -2384,7 +2405,7 @@ private static bool ContainsNestedUnboundedLoop(SymbolicRegexNode<TSet> node)
                 switch (current._kind)
                 {
                     case SymbolicRegexNodeKind.Loop:
-                        if (current._upper == int.MaxValue)
+                        if (isMatch(current))
                         {
                             return true;
                         }

src/libraries/System.Text.RegularExpressions/tests/FunctionalTests/Regex.MultipleMatches.Tests.cs

@@ -75,6 +75,18 @@ public static IEnumerable<object[]> Matches_TestData()
                     }
                 };
 
+                // Regression test for NonBacktracking dropping a match adjacent to an optional element.
+                yield return new object[]
+                {
+                    engine,
+                    " ?, ?", " a,,", RegexOptions.None,
+                    new CaptureData[]
+                    {
+                        new CaptureData(",", 2, 1),
+                        new CaptureData(",", 3, 1),
+                    }
+                };
+
                 yield return new object[]
                 {
                     engine,

src/libraries/System.Text.RegularExpressions/tests/UnitTests/SymbolicRegexTests.cs

@@ -96,7 +96,8 @@ public static IEnumerable<object[]> SafeThresholdTests_MemberData()
             {
                 RegexNode tree = RegexParser.Parse(".*?" + Pattern, options | RegexOptions.ExplicitCapture, CultureInfo.CurrentCulture).Root;
                 SymbolicRegexNode<BDD> rootNode = converter.ConvertToSymbolicRegexNode(tree);
-                yield return new object[] { bddBuilder, rootNode, 1 + ExpectedSafeSize };
+                int expected = tree.Child(0).Kind == RegexNodeKind.Alternate ? 1 + (2 * ExpectedSafeSize) : 1 + ExpectedSafeSize;
+                yield return new object[] { bddBuilder, rootNode, expected };
             }
 
             // use of anchors increases the estimate by 5x in general but in reality much less, at most 3x