Skip to content

Commit

Permalink
Improve parsing performance
Browse files Browse the repository at this point in the history 
- In LinguiniParser, ZeroCopyReader, ZeroCopyUtil: change helpers from dealing in ReadOnlyMemory<char> to char. In inner loops this removes a lot of redundant checks for a span of length 1 and inlines more readily.
- In ZeroCopyReader add helpers for SeekEol, IndexOfAnyChar. These use existing helper methods of Span that are SIMD accelerated to search for indexes quickly. These provide significant speedups compared to incrementing the index in a loop.
- In FluentBundle.AddResourceOverriding, provide a specialized 'override' method to avoid two dictionary lookups and allocating an empty list.
  • Loading branch information
@RoosterDragon
RoosterDragon committed Jun 10, 2023
1 parent 97a2723 commit 5ccecf7
Show file tree
Hide file tree
Showing 7 changed files with 295 additions and 346 deletions.
12 changes: 9 additions & 3 deletions Linguini.Bundle/FluentBundle.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,15 +126,21 @@ public void AddResourceOverriding(Resource res)

if (entry is AstTerm or AstMessage)
{
AddEntry(new List<FluentError>(), entry, true);
AddEntryOverriding(entry);
}
}
}

private void AddEntry(List<FluentError> errors, IEntry term, bool overwrite = false)
private void AddEntryOverriding(IEntry term)
{
var id = (term.GetId(), term.ToKind());
if (_entries.ContainsKey(id) && !overwrite)
_entries[id] = term;
}

private void AddEntry(List<FluentError> errors, IEntry term)
{
var id = (term.GetId(), term.ToKind());
if (_entries.ContainsKey(id))
{
errors.Add(new OverrideFluentError(id.Item1, id.Item2));
}
Expand Down
134 changes: 19 additions & 115 deletions Linguini.Shared/Util/ZeroCopyUtil.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,123 +9,51 @@ namespace Linguini.Shared.Util
/// </summary>
public static class ZeroCopyUtil
{
private const int CharLength = 1;
private static readonly ReadOnlyMemory<char> Eof = ReadOnlyMemory<char>.Empty;

public static ReadOnlySpan<char> PeakCharAt(this ReadOnlyMemory<char> memory, int pos)
{
memory.TryReadCharSpan(pos, out var span);
return span;
}

public static bool TryReadCharSpan(this ReadOnlyMemory<char> memory, int pos, out ReadOnlySpan<char> span)
public static bool TryReadChar(this ReadOnlyMemory<char> memory, int pos, out char c)
{
span = Eof.Span;
if (pos + CharLength > memory.Length)
if (pos >= memory.Length)
{
c = default;
return false;
}

span = memory.Slice(pos, CharLength).Span;
c = memory.Span[pos];
return true;
}

public static bool EqualsSpans(this char chr, ReadOnlySpan<char> chrSpan)
{
if (chrSpan.Length > CharLength)
{
return false;
}

return chrSpan.Length != 0 && IsEqual(chrSpan, chr);
}

public static bool EqualsSpans(this char? lhs, ReadOnlySpan<char> chrSpan)
{
if (lhs == null)
{
return chrSpan == Eof.Span;
}

return EqualsSpans((char) lhs, chrSpan);
}

public static bool IsEqual(this ReadOnlySpan<char> charSpan, char c1)
{
if (charSpan.Length != CharLength)
{
return false;
}

return MemoryMarshal.GetReference(charSpan) == c1;
}

public static bool IsAsciiAlphabetic(this ReadOnlySpan<char> charSpan)
public static bool IsAsciiAlphabetic(this char c)
{
if (charSpan.Length != CharLength)
{
return false;
}

var c = MemoryMarshal.GetReference(charSpan);
return IsInside(c, 'a', 'z')
|| IsInside(c, 'A', 'Z');
}

public static bool IsAsciiHexdigit(this ReadOnlySpan<char> charSpan)
public static bool IsAsciiHexdigit(this char c)
{
if (charSpan.Length != CharLength)
{
return false;
}

var c = MemoryMarshal.GetReference(charSpan);
return IsInside(c, '0', '9')
|| IsInside(c, 'a', 'f')
|| IsInside(c, 'A', 'F');
}

public static bool IsAsciiUppercase(this ReadOnlySpan<char> charSpan)
public static bool IsAsciiUppercase(this char c)
{
if (charSpan.Length != CharLength)
{
return false;
}

var c = MemoryMarshal.GetReference(charSpan);
return IsInside(c, 'A', 'Z');
}

public static bool IsAsciiDigit(this ReadOnlySpan<char> charSpan)
public static bool IsAsciiDigit(this char c)
{
if (charSpan.Length != CharLength)
{
return false;
}

var c = MemoryMarshal.GetReference(charSpan);
return IsInside(c, '0', '9');
}



public static bool IsNumberStart(this ReadOnlySpan<char> charSpan)
public static bool IsNumberStart(this char c)
{
if (charSpan.Length != CharLength)
{
return false;
}

var c = MemoryMarshal.GetReference(charSpan);
return IsInside(c, '0', '9') || c == '-';
}

public static bool IsCallee(this ReadOnlyMemory<char> charSpan)
public static bool IsCallee(this ReadOnlySpan<char> charSpan)
{
bool isCallee = true;
for (int i = 0; i < charSpan.Length - 1; i++)
foreach (var c in charSpan)
{
var c = charSpan.Slice(i, 1).Span;
if (!(c.IsAsciiUppercase() || c.IsAsciiDigit() || c.IsOneOf('_', '-')))
{
isCallee = false;
Expand All @@ -136,53 +64,29 @@ public static bool IsCallee(this ReadOnlyMemory<char> charSpan)
return isCallee;
}


public static bool IsIdentifier(this ReadOnlySpan<char> charSpan)
public static bool IsIdentifier(this char c)
{
if (charSpan.Length != CharLength)
{
return false;
}

var c = MemoryMarshal.GetReference(charSpan);
return IsInside(c, 'a', 'z')
|| IsInside(c, 'A', 'Z')
|| IsInside(c, '0', '9')
|| c == '-' || c == '_';
}

public static bool IsOneOf(this ReadOnlySpan<char> charSpan, char c1, char c2)
public static bool IsOneOf(this char c, char c1, char c2)
{
if (charSpan.Length != CharLength)
{
return false;
}

var x = MemoryMarshal.GetReference(charSpan);
return x == c1 || x == c2;
return c == c1 || c == c2;
}

public static bool IsOneOf(this ReadOnlySpan<char> charSpan, char c1, char c2, char c3)
public static bool IsOneOf(this char c, char c1, char c2, char c3)
{
if (charSpan.Length != CharLength)
{
return false;
}

var x = MemoryMarshal.GetReference(charSpan);
return x == c1 || x == c2 || x == c3;
return c == c1 || c == c2 || c == c3;
}

public static bool IsOneOf(this ReadOnlySpan<char> charSpan, char c1, char c2, char c3, char c4)
public static bool IsOneOf(this char c, char c1, char c2, char c3, char c4)
{
if (charSpan.Length != CharLength)
{
return false;
}

var x = MemoryMarshal.GetReference(charSpan);
return x == c1 || x == c2 || x == c3 || x == c4;
return c == c1 || c == c2 || c == c3 || c == c4;
}

#if !NET5_0_OR_GREATER
// Polyfill for netstandard 2.1 until dotnet backports MemoryExtension
public static ReadOnlyMemory<char> TrimEndPolyFill(this ReadOnlyMemory<char> memory)
Expand Down
14 changes: 2 additions & 12 deletions Linguini.Syntax.Tests/IO/NonValidCharInputTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,20 +14,10 @@ public class NonValidCharInputTest
[TestCase("단편")]
[TestCase("かんじ")]
[TestCase("Северный поток")]
public void OperationOnNonCharSpanReturnFalse(string input)
public void OperationOnNonCalleeReturnFalse(string input)
{
var span = input.AsSpan();
Assert.False('c'.EqualsSpans(span));
Assert.False(span.IsEqual('c'));
Assert.False(span.IsAsciiAlphabetic());
Assert.False(span.IsAsciiDigit());
Assert.False(span.IsAsciiHexdigit());
Assert.False(span.IsAsciiUppercase());
Assert.False(span.IsOneOf('c', 's'));
Assert.False(span.IsOneOf('c', 's', 'l'));
Assert.False(span.IsOneOf('c', 's', 'a', 'l'));
Assert.False(span.IsNumberStart());
Assert.False(input.AsMemory().IsCallee());
Assert.False(span.IsCallee());
}
}
}
47 changes: 33 additions & 14 deletions Linguini.Syntax.Tests/IO/ZeroCopyReaderTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,9 @@ public class ZeroCopyReaderTest
public void TestPeekChar(string text, char expected)
{
ZeroCopyReader reader = new ZeroCopyReader(text);
Assert.That(expected.EqualsSpans(reader.PeekCharSpan()));
Assert.That(expected.EqualsSpans(reader.PeekCharSpan()));
Assert.That(expected.EqualsSpans(reader.PeekCharSpan()));
Assert.That(expected == reader.PeekChar());
Assert.That(expected == reader.PeekChar());
Assert.That(expected == reader.PeekChar());
}


Expand All @@ -43,7 +43,7 @@ public void TestExpectChar(string text, char expectedChr, bool expected, char? p
{
ZeroCopyReader reader = new ZeroCopyReader(text);
Assert.AreEqual(expected, reader.ReadCharIf(expectedChr));
Assert.True(peek.EqualsSpans(reader.PeekCharSpan()));
Assert.True(peek == reader.PeekChar());
}

[Test]
Expand All @@ -56,9 +56,9 @@ public void TestExpectChar(string text, char expectedChr, bool expected, char? p
public void TestPeekGetChar(string text, char expected1, char expected2)
{
ZeroCopyReader reader = new ZeroCopyReader(text);
Assert.That(expected1.EqualsSpans(reader.PeekCharSpan()));
Assert.That(expected1.EqualsSpans(reader.GetCharSpan()));
Assert.That(expected2.EqualsSpans(reader.PeekCharSpan()));
Assert.That(expected1 == reader.PeekChar());
reader.Position += 1;
Assert.That(expected2 == reader.PeekChar());
}

[Test]
Expand All @@ -71,8 +71,8 @@ public void TestPeekGetChar(string text, char expected1, char expected2)
public void TestPeekCharOffset(string text, char expected1, char expected2)
{
ZeroCopyReader reader = new ZeroCopyReader(text);
Assert.That(expected1.EqualsSpans(reader.PeekCharSpan()));
Assert.That(expected2.EqualsSpans(reader.PeekCharSpan(1)));
Assert.That(expected1 == reader.PeekChar());
Assert.That(expected2 == reader.PeekChar(1));
}

[Test]
Expand All @@ -86,19 +86,38 @@ public void TestSkipBlank(string text, char postSkipChar)
{
ZeroCopyReader reader = new ZeroCopyReader(text);
reader.SkipBlankBlock();
Assert.That(postSkipChar.EqualsSpans(reader.GetCharSpan()));
Assert.That(postSkipChar == reader.PeekChar());
}

[Test]
[Parallelizable]
[TestCase("", false, null)]
[TestCase(" \nb", true, 3, 2)]
[TestCase(" \r\nb2", true, 5, 2)]
[TestCase(" \n漢字", true, 5, 2)]
[TestCase(" \n단편", true, 6, 2)]
[TestCase(" \nか", true, 7, 2)]
[TestCase(" \rか", false, 8, 1)]
[TestCase("", false, 0, 1)]
[TestCase("bbbbb", false, 5, 1)]
public void TestSeekEol(string text, bool expectedEol, int expectedPosition, int expectedRow)
{
ZeroCopyReader reader = new ZeroCopyReader(text);
var foundEol = reader.SeekEol();
Assert.That(expectedEol, Is.EqualTo(foundEol));
Assert.That(expectedPosition, Is.EqualTo(reader.Position));
Assert.That(expectedRow, Is.EqualTo(reader.Row));
}

[Test]
[Parallelizable]
[TestCase("", false, default(char))]
[TestCase("a", true, 'a')]
public void TestTryReadCharSpan(string text, bool isChar, char? expected1)
public void TestTryReadChar(string text, bool isChar, char? expected1)
{
ReadOnlyMemory<char> mem = new ReadOnlyMemory<char>(text.ToCharArray());
bool isThereChar = mem.TryReadCharSpan(0, out var readChr);
bool isThereChar = mem.TryReadChar(0, out var readChr);
Assert.That(isThereChar, Is.EqualTo(isChar));
Assert.That(expected1.EqualsSpans(readChr));
Assert.That(expected1 == readChr);
}

[Test]
Expand Down
Loading

0 comments on commit 5ccecf7

Please sign in to comment.