Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Improve parsing performance #29

Merged
merged 1 commit into from
Jun 11, 2023
Merged

Improve parsing performance #29

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 9 additions & 3 deletions Linguini.Bundle/FluentBundle.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,15 +126,21 @@ public void AddResourceOverriding(Resource res)

if (entry is AstTerm or AstMessage)
{
AddEntry(new List<FluentError>(), entry, true);
AddEntryOverriding(entry);
}
}
}

private void AddEntry(List<FluentError> errors, IEntry term, bool overwrite = false)
private void AddEntryOverriding(IEntry term)
{
var id = (term.GetId(), term.ToKind());
if (_entries.ContainsKey(id) && !overwrite)
_entries[id] = term;
}

private void AddEntry(List<FluentError> errors, IEntry term)
{
var id = (term.GetId(), term.ToKind());
if (_entries.ContainsKey(id))
{
errors.Add(new OverrideFluentError(id.Item1, id.Item2));
}
Expand Down
134 changes: 19 additions & 115 deletions Linguini.Shared/Util/ZeroCopyUtil.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,123 +9,51 @@ namespace Linguini.Shared.Util
/// </summary>
public static class ZeroCopyUtil
{
private const int CharLength = 1;
private static readonly ReadOnlyMemory<char> Eof = ReadOnlyMemory<char>.Empty;

public static ReadOnlySpan<char> PeakCharAt(this ReadOnlyMemory<char> memory, int pos)
{
memory.TryReadCharSpan(pos, out var span);
return span;
}

public static bool TryReadCharSpan(this ReadOnlyMemory<char> memory, int pos, out ReadOnlySpan<char> span)
public static bool TryReadChar(this ReadOnlyMemory<char> memory, int pos, out char c)
{
span = Eof.Span;
if (pos + CharLength > memory.Length)
if (pos >= memory.Length)
{
c = default;
return false;
}

span = memory.Slice(pos, CharLength).Span;
c = memory.Span[pos];
return true;
}

public static bool EqualsSpans(this char chr, ReadOnlySpan<char> chrSpan)
{
if (chrSpan.Length > CharLength)
{
return false;
}

return chrSpan.Length != 0 && IsEqual(chrSpan, chr);
}

public static bool EqualsSpans(this char? lhs, ReadOnlySpan<char> chrSpan)
{
if (lhs == null)
{
return chrSpan == Eof.Span;
}

return EqualsSpans((char) lhs, chrSpan);
}

public static bool IsEqual(this ReadOnlySpan<char> charSpan, char c1)
{
if (charSpan.Length != CharLength)
{
return false;
}

return MemoryMarshal.GetReference(charSpan) == c1;
}

public static bool IsAsciiAlphabetic(this ReadOnlySpan<char> charSpan)
public static bool IsAsciiAlphabetic(this char c)
{
if (charSpan.Length != CharLength)
{
return false;
}

var c = MemoryMarshal.GetReference(charSpan);
return IsInside(c, 'a', 'z')
|| IsInside(c, 'A', 'Z');
}

public static bool IsAsciiHexdigit(this ReadOnlySpan<char> charSpan)
public static bool IsAsciiHexdigit(this char c)
{
if (charSpan.Length != CharLength)
{
return false;
}

var c = MemoryMarshal.GetReference(charSpan);
return IsInside(c, '0', '9')
|| IsInside(c, 'a', 'f')
|| IsInside(c, 'A', 'F');
}

public static bool IsAsciiUppercase(this ReadOnlySpan<char> charSpan)
public static bool IsAsciiUppercase(this char c)
{
if (charSpan.Length != CharLength)
{
return false;
}

var c = MemoryMarshal.GetReference(charSpan);
return IsInside(c, 'A', 'Z');
}

public static bool IsAsciiDigit(this ReadOnlySpan<char> charSpan)
public static bool IsAsciiDigit(this char c)
{
if (charSpan.Length != CharLength)
{
return false;
}

var c = MemoryMarshal.GetReference(charSpan);
return IsInside(c, '0', '9');
}



public static bool IsNumberStart(this ReadOnlySpan<char> charSpan)
public static bool IsNumberStart(this char c)
{
if (charSpan.Length != CharLength)
{
return false;
}

var c = MemoryMarshal.GetReference(charSpan);
return IsInside(c, '0', '9') || c == '-';
}

public static bool IsCallee(this ReadOnlyMemory<char> charSpan)
public static bool IsCallee(this ReadOnlySpan<char> charSpan)
{
bool isCallee = true;
for (int i = 0; i < charSpan.Length - 1; i++)
foreach (var c in charSpan)
{
var c = charSpan.Slice(i, 1).Span;
if (!(c.IsAsciiUppercase() || c.IsAsciiDigit() || c.IsOneOf('_', '-')))
{
isCallee = false;
Expand All @@ -136,53 +64,29 @@ public static bool IsCallee(this ReadOnlyMemory<char> charSpan)
return isCallee;
}


public static bool IsIdentifier(this ReadOnlySpan<char> charSpan)
public static bool IsIdentifier(this char c)
{
if (charSpan.Length != CharLength)
{
return false;
}

var c = MemoryMarshal.GetReference(charSpan);
return IsInside(c, 'a', 'z')
|| IsInside(c, 'A', 'Z')
|| IsInside(c, '0', '9')
|| c == '-' || c == '_';
}

public static bool IsOneOf(this ReadOnlySpan<char> charSpan, char c1, char c2)
public static bool IsOneOf(this char c, char c1, char c2)
{
if (charSpan.Length != CharLength)
{
return false;
}

var x = MemoryMarshal.GetReference(charSpan);
return x == c1 || x == c2;
return c == c1 || c == c2;
}

public static bool IsOneOf(this ReadOnlySpan<char> charSpan, char c1, char c2, char c3)
public static bool IsOneOf(this char c, char c1, char c2, char c3)
{
if (charSpan.Length != CharLength)
{
return false;
}

var x = MemoryMarshal.GetReference(charSpan);
return x == c1 || x == c2 || x == c3;
return c == c1 || c == c2 || c == c3;
}

public static bool IsOneOf(this ReadOnlySpan<char> charSpan, char c1, char c2, char c3, char c4)
public static bool IsOneOf(this char c, char c1, char c2, char c3, char c4)
{
if (charSpan.Length != CharLength)
{
return false;
}

var x = MemoryMarshal.GetReference(charSpan);
return x == c1 || x == c2 || x == c3 || x == c4;
return c == c1 || c == c2 || c == c3 || c == c4;
}

#if !NET5_0_OR_GREATER
// Polyfill for netstandard 2.1 until dotnet backports MemoryExtension
public static ReadOnlyMemory<char> TrimEndPolyFill(this ReadOnlyMemory<char> memory)
Expand Down
14 changes: 2 additions & 12 deletions Linguini.Syntax.Tests/IO/NonValidCharInputTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,20 +14,10 @@ public class NonValidCharInputTest
[TestCase("단편")]
[TestCase("かんじ")]
[TestCase("Северный поток")]
public void OperationOnNonCharSpanReturnFalse(string input)
public void OperationOnNonCalleeReturnFalse(string input)
{
var span = input.AsSpan();
Assert.False('c'.EqualsSpans(span));
Assert.False(span.IsEqual('c'));
Assert.False(span.IsAsciiAlphabetic());
Assert.False(span.IsAsciiDigit());
Assert.False(span.IsAsciiHexdigit());
Assert.False(span.IsAsciiUppercase());
Assert.False(span.IsOneOf('c', 's'));
Assert.False(span.IsOneOf('c', 's', 'l'));
Assert.False(span.IsOneOf('c', 's', 'a', 'l'));
Assert.False(span.IsNumberStart());
Assert.False(input.AsMemory().IsCallee());
Assert.False(span.IsCallee());
}
}
}
47 changes: 33 additions & 14 deletions Linguini.Syntax.Tests/IO/ZeroCopyReaderTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,9 @@ public class ZeroCopyReaderTest
public void TestPeekChar(string text, char expected)
{
ZeroCopyReader reader = new ZeroCopyReader(text);
Assert.That(expected.EqualsSpans(reader.PeekCharSpan()));
Assert.That(expected.EqualsSpans(reader.PeekCharSpan()));
Assert.That(expected.EqualsSpans(reader.PeekCharSpan()));
Assert.That(expected == reader.PeekChar());
Assert.That(expected == reader.PeekChar());
Assert.That(expected == reader.PeekChar());
}


Expand All @@ -43,7 +43,7 @@ public void TestExpectChar(string text, char expectedChr, bool expected, char? p
{
ZeroCopyReader reader = new ZeroCopyReader(text);
Assert.AreEqual(expected, reader.ReadCharIf(expectedChr));
Assert.True(peek.EqualsSpans(reader.PeekCharSpan()));
Assert.True(peek == reader.PeekChar());
}

[Test]
Expand All @@ -56,9 +56,9 @@ public void TestExpectChar(string text, char expectedChr, bool expected, char? p
public void TestPeekGetChar(string text, char expected1, char expected2)
{
ZeroCopyReader reader = new ZeroCopyReader(text);
Assert.That(expected1.EqualsSpans(reader.PeekCharSpan()));
Assert.That(expected1.EqualsSpans(reader.GetCharSpan()));
Assert.That(expected2.EqualsSpans(reader.PeekCharSpan()));
Assert.That(expected1 == reader.PeekChar());
reader.Position += 1;
Assert.That(expected2 == reader.PeekChar());
}

[Test]
Expand All @@ -71,8 +71,8 @@ public void TestPeekGetChar(string text, char expected1, char expected2)
public void TestPeekCharOffset(string text, char expected1, char expected2)
{
ZeroCopyReader reader = new ZeroCopyReader(text);
Assert.That(expected1.EqualsSpans(reader.PeekCharSpan()));
Assert.That(expected2.EqualsSpans(reader.PeekCharSpan(1)));
Assert.That(expected1 == reader.PeekChar());
Assert.That(expected2 == reader.PeekChar(1));
}

[Test]
Expand All @@ -86,19 +86,38 @@ public void TestSkipBlank(string text, char postSkipChar)
{
ZeroCopyReader reader = new ZeroCopyReader(text);
reader.SkipBlankBlock();
Assert.That(postSkipChar.EqualsSpans(reader.GetCharSpan()));
Assert.That(postSkipChar == reader.PeekChar());
}

[Test]
[Parallelizable]
[TestCase("", false, null)]
[TestCase(" \nb", true, 3, 2)]
[TestCase(" \r\nb2", true, 5, 2)]
[TestCase(" \n漢字", true, 5, 2)]
[TestCase(" \n단편", true, 6, 2)]
[TestCase(" \nか", true, 7, 2)]
[TestCase(" \rか", false, 8, 1)]
[TestCase("", false, 0, 1)]
[TestCase("bbbbb", false, 5, 1)]
public void TestSeekEol(string text, bool expectedEol, int expectedPosition, int expectedRow)
{
ZeroCopyReader reader = new ZeroCopyReader(text);
var foundEol = reader.SeekEol();
Assert.That(expectedEol, Is.EqualTo(foundEol));
Assert.That(expectedPosition, Is.EqualTo(reader.Position));
Assert.That(expectedRow, Is.EqualTo(reader.Row));
}

[Test]
[Parallelizable]
[TestCase("", false, default(char))]
[TestCase("a", true, 'a')]
public void TestTryReadCharSpan(string text, bool isChar, char? expected1)
public void TestTryReadChar(string text, bool isChar, char? expected1)
{
ReadOnlyMemory<char> mem = new ReadOnlyMemory<char>(text.ToCharArray());
bool isThereChar = mem.TryReadCharSpan(0, out var readChr);
bool isThereChar = mem.TryReadChar(0, out var readChr);
Assert.That(isThereChar, Is.EqualTo(isChar));
Assert.That(expected1.EqualsSpans(readChr));
Assert.That(expected1 == readChr);
}

[Test]
Expand Down
Loading
Loading