Skip to content

Commit

Permalink
Close #312 - Remove Lexer from public API
Browse files Browse the repository at this point in the history
  • Loading branch information
@sys27
sys27 committed Sep 13, 2020
1 parent 9f2178e commit 379ecc8
Show file tree
Hide file tree
Showing 53 changed files with 979 additions and 7,103 deletions.
7 changes: 0 additions & 7 deletions xFunc.Benchmark/Benchmarks/CtorBenchmark.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,18 +15,11 @@

using BenchmarkDotNet.Attributes;
using xFunc.Maths;
using xFunc.Maths.Tokenization;

namespace xFunc.Benchmark.Benchmarks
{
public class CtorBenchmark
{
[Benchmark]
public ILexer LexerCtor()
{
return new Lexer();
}

[Benchmark]
public IParser ParserCtor()
{
Expand Down
64 changes: 0 additions & 64 deletions xFunc.Benchmark/Benchmarks/LexerBenchmark.cs
View file

This file was deleted.

12 changes: 1 addition & 11 deletions xFunc.Benchmark/Benchmarks/ParserBenchmark.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,33 +14,23 @@
// limitations under the License.

using BenchmarkDotNet.Attributes;
using System.Collections.Generic;
using System.Linq;
using xFunc.Maths;
using xFunc.Maths.Expressions;
using xFunc.Maths.Tokenization;
using xFunc.Maths.Tokenization.Tokens;

namespace xFunc.Benchmark.Benchmarks
{
public class ParserBenchmark
{
private Parser parser;

private IList<IToken> tokens;

[GlobalSetup]
public void Setup()
{
parser = new Parser();

var lexer = new Lexer();

tokens = lexer.Tokenize("(100.1 + 2(3sin(4cos(5tan(6ctg(10x)))) * 3) / (func(a, b, c) ^ 2)) - (cos(y) - 111.3) & (true | false -> true <-> false eq true) + (det({{1, 2}, {3, 4}}) * 10log(2, 3)) + re(3 + 2i) - im(2 - 9i) + (9 + 2i)").ToList();
}

[Benchmark]
public IExpression Parse()
=> parser.Parse(tokens);
=> parser.Parse("(100.1 + 2(3sin(4cos(5tan(6ctg(10x)))) * 3) / (func(a, b, c) ^ 2)) - (cos(y) - 111.3) & (true | false -> true <-> false eq true) + (det({{1, 2}, {3, 4}}) * 10log(2, 3)) + re(3 + 2i) - im(2 - 9i) + (9 + 2i)");
}
}
39 changes: 0 additions & 39 deletions xFunc.DotnetTool/Options/TokenizeOptions.cs
View file

This file was deleted.

13 changes: 1 addition & 12 deletions xFunc.DotnetTool/Program.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,16 +28,6 @@ public class Program
{
private static readonly Processor processor = new Processor();

private static void Tokenize(TokenizeOptions options)
{
Run(options, () =>
{
var tokens = processor.Tokenize(options.StringExpression);
foreach (var token in tokens)
Console.WriteLine(token);
});
}

private static void Parse(ParseOptions options)
{
Run(options, () =>
Expand Down Expand Up @@ -161,8 +151,7 @@ private static void PrintError(DebugInfoOptions options, Exception e)
public static void Main(string[] args)
{
CommandLine.Parser.Default
.ParseArguments<TokenizeOptions, ParseOptions, SolveOptions, InteractiveOptions>(args)
.WithParsed<TokenizeOptions>(Tokenize)
.ParseArguments<ParseOptions, SolveOptions, InteractiveOptions>(args)
.WithParsed<ParseOptions>(Parse)
.WithParsed<SolveOptions>(Solve)
.WithParsed<InteractiveOptions>(Interactive);
Expand Down
37 changes: 19 additions & 18 deletions xFunc.Maths/Helpers.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,24 +52,25 @@ public static bool HasVariable(IExpression expression, Variable variable)
return expression is Variable && expression.Equals(variable);
}

/// <summary>
/// Gets parameters of expression.
/// </summary>
/// <param name="tokens">The list of tokens.</param>
/// <returns>A collection of parameters.</returns>
public static ParameterCollection GetParameters(IEnumerable<IToken> tokens)
{
if (tokens == null)
throw new ArgumentNullException(nameof(tokens));

var c = new SortedSet<Parameter>();

foreach (var token in tokens)
if (token is IdToken var)
c.Add(new Parameter(var.Id, false));

return new ParameterCollection(c, false);
}
// TODO: remove?
// /// <summary>
// /// Gets parameters of expression.
// /// </summary>
// /// <param name="tokens">The list of tokens.</param>
// /// <returns>A collection of parameters.</returns>
// public static ParameterCollection GetParameters(IEnumerable<IToken> tokens)
// {
// if (tokens == null)
// throw new ArgumentNullException(nameof(tokens));
//
// var c = new SortedSet<Parameter>();
//
// foreach (var token in tokens)
// if (token is IdToken var)
// c.Add(new Parameter(var.Id, false));
//
// return new ParameterCollection(c, false);
// }

/// <summary>
/// Converts the logic expression to collection.
Expand Down
6 changes: 2 additions & 4 deletions xFunc.Maths/IParser.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.

using System.Collections.Generic;
using xFunc.Maths.Expressions;
using xFunc.Maths.Tokenization.Tokens;

namespace xFunc.Maths
{
Expand All @@ -27,8 +25,8 @@ public interface IParser
/// <summary>
/// Parses the specified function.
/// </summary>
/// <param name="tokens">The list of tokens.</param>
/// <param name="function">The string that contains the functions and operators.</param>
/// <returns>The parsed expression.</returns>
IExpression Parse(IEnumerable<IToken> tokens);
IExpression Parse(string function);
}
}
17 changes: 10 additions & 7 deletions xFunc.Maths/Parser.TokenReader.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@

using System;
using System.Buffers;
using System.Collections.Generic;
using System.Diagnostics;
using xFunc.Maths.Tokenization;
using xFunc.Maths.Tokenization.Tokens;

namespace xFunc.Maths
Expand All @@ -30,7 +30,9 @@ public partial class Parser
{
private const int BufferSize = 64;

private readonly IEnumerator<IToken> enumerator;
// TODO: do not mark as 'readonly'
private Lexer lexer;

private bool enumerableEnded;

// points to read position (item != null)
Expand All @@ -43,9 +45,10 @@ public partial class Parser

private int scopeCount;

public TokenReader(IEnumerable<IToken> tokens)
public TokenReader(Lexer lexer)
{
enumerator = tokens.GetEnumerator();
this.lexer = lexer;

enumerableEnded = false;
readIndex = -1;
writeIndex = -1;
Expand All @@ -55,7 +58,6 @@ public TokenReader(IEnumerable<IToken> tokens)

public void Dispose()
{
enumerator.Dispose();
ArrayPool<IToken?>.Shared.Return(buffer);
}

Expand All @@ -64,6 +66,7 @@ private void EnsureEnoughSpace()
if (writeIndex < buffer.Length)
return;

// TODO: cover
var newBuffer = ArrayPool<IToken>.Shared.Rent(buffer.Length * 2);
Array.Copy(buffer, newBuffer, buffer.Length);
ArrayPool<IToken?>.Shared.Return(buffer);
Expand All @@ -78,7 +81,7 @@ private void EnsureEnoughSpace()
// read from enumerator and write to buffer
if (readIndex == writeIndex)
{
var result = enumerator.MoveNext();
var result = lexer.MoveNext();
if (!result)
{
enumerableEnded = true;
Expand All @@ -93,7 +96,7 @@ private void EnsureEnoughSpace()

EnsureEnoughSpace();

return (buffer[writeIndex] = enumerator.Current) as TToken;
return (buffer[writeIndex] = lexer.Current) as TToken;
}

// readIndex < writeIndex
Expand Down
12 changes: 7 additions & 5 deletions xFunc.Maths/Parser.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
using xFunc.Maths.Expressions.Angles;
using xFunc.Maths.Expressions.Matrices;
using xFunc.Maths.Resources;
using xFunc.Maths.Tokenization;
using xFunc.Maths.Tokenization.Tokens;

namespace xFunc.Maths
Expand Down Expand Up @@ -57,14 +58,15 @@ public Parser(IDifferentiator differentiator, ISimplifier simplifier)
/// <summary>
/// Parses the specified function.
/// </summary>
/// <param name="tokens">The list of tokens.</param>
/// <param name="function">The string that contains the functions and operators.</param>
/// <returns>The parsed expression.</returns>
public IExpression Parse(IEnumerable<IToken> tokens)
public IExpression Parse(string function)
{
if (tokens == null)
throw new ArgumentNullException(nameof(tokens));
// if (tokens == null)
// throw new ArgumentNullException(nameof(tokens));
var lexer = new Lexer(function);
var tokenReader = new TokenReader(lexer);

var tokenReader = new TokenReader(tokens);
try
{
var exp = Statement(ref tokenReader);
Expand Down
13 changes: 1 addition & 12 deletions xFunc.Maths/Processor.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,14 +35,12 @@ public class Processor
private readonly IDifferentiator differentiator;
private readonly ISimplifier simplifier;
private readonly IParser parser;
private readonly ILexer lexer;

/// <summary>
/// Initializes a new instance of the <see cref="Processor"/> class.
/// </summary>
public Processor()
{
lexer = new Lexer();
simplifier = new Simplifier();
differentiator = new Differentiator();
parser = new Parser(differentiator, simplifier);
Expand Down Expand Up @@ -81,7 +79,6 @@ public Processor()
ITypeAnalyzer typeAnalyzer,
ExpressionParameters parameters)
{
lexer = new Lexer();
parser = new Parser();

this.simplifier = simplifier ??
Expand Down Expand Up @@ -232,15 +229,7 @@ public IExpression Differentiate(IExpression expression, Variable variable)
/// <exception cref="ArgumentNullException"><paramref name="function"/> is null.</exception>
/// <exception cref="ParseException">Error while parsing.</exception>
public IExpression Parse(string function)
=> parser.Parse(lexer.Tokenize(function));

/// <summary>
/// Converts the string into a sequence of tokens.
/// </summary>
/// <param name="function">The string that contains the functions and operators.</param>
/// <returns>The sequence of tokens.</returns>
public IEnumerable<IToken> Tokenize(string function)
=> lexer.Tokenize(function);
=> parser.Parse(function);

/// <summary>
/// Gets expression parameters object.
Expand Down

0 comments on commit 379ecc8

Please sign in to comment.