From ddc646596f32ec57a15d20dc1982fc510fa5aec2 Mon Sep 17 00:00:00 2001 From: bberger Date: Thu, 18 Oct 2018 14:25:52 -0700 Subject: [PATCH 1/2] Improve performance by more than 400X Scenarios: * IEnumerable option with 9000 entries: 15+ hours -> 150ms * Options class w/20 string options all set on command line: 960ms -> 2.2ms The speedups were all achieved by memoizing IEnumerables that are enumerated multiple times. --- src/CommandLine/Core/InstanceBuilder.cs | 31 ++++++++++++++---------- src/CommandLine/Core/OptionMapper.cs | 2 +- src/CommandLine/Core/TokenPartitioner.cs | 14 ++++++----- src/CommandLine/Core/Tokenizer.cs | 6 ++--- 4 files changed, 30 insertions(+), 23 deletions(-) diff --git a/src/CommandLine/Core/InstanceBuilder.cs b/src/CommandLine/Core/InstanceBuilder.cs index fd9ac4d7..01597ac4 100644 --- a/src/CommandLine/Core/InstanceBuilder.cs +++ b/src/CommandLine/Core/InstanceBuilder.cs @@ -28,13 +28,15 @@ public static ParserResult Build( var typeInfo = factory.MapValueOrDefault(f => f().GetType(), typeof(T)); var specProps = typeInfo.GetSpecifications(pi => SpecificationProperty.Create( - Specification.FromProperty(pi), pi, Maybe.Nothing())); + Specification.FromProperty(pi), pi, Maybe.Nothing())) + .Memorize(); var specs = from pt in specProps select pt.Specification; var optionSpecs = specs .ThrowingValidate(SpecificationGuards.Lookup) - .OfType(); + .OfType() + .Memorize(); Func makeDefault = () => typeof(T).IsMutable() @@ -45,18 +47,19 @@ public static ParserResult Build( Func, ParserResult> notParsed = errs => new NotParsed(makeDefault().GetType().ToTypeInfo(), errs); + var argumentsList = arguments.Memorize(); Func> buildUp = () => { - var tokenizerResult = tokenizer(arguments, optionSpecs); + var tokenizerResult = tokenizer(argumentsList, optionSpecs); - var tokens = tokenizerResult.SucceededWith(); + var tokens = tokenizerResult.SucceededWith().Memorize(); var partitions = TokenPartitioner.Partition( tokens, name => TypeLookup.FindTypeDescriptorAndSibling(name, optionSpecs, nameComparer)); - var optionsPartition = partitions.Item1; - var valuesPartition = partitions.Item2; - var errorsPartition = partitions.Item3; + var optionsPartition = partitions.Item1.Memorize(); + var valuesPartition = partitions.Item2.Memorize(); + var errorsPartition = partitions.Item3.Memorize(); var optionSpecPropsResult = OptionMapper.MapValues( @@ -68,7 +71,7 @@ public static ParserResult Build( var valueSpecPropsResult = ValueMapper.MapValues( (from pt in specProps where pt.Specification.IsValue() orderby ((ValueSpecification)pt.Specification).Index select pt), - valuesPartition, + valuesPartition, (vals, type, isScalar) => TypeConverter.ChangeType(vals, type, isScalar, parsingCulture, ignoreValueCase)); var missingValueErrors = from token in errorsPartition @@ -78,7 +81,7 @@ public static ParserResult Build( .FromOptionSpecification()); var specPropsWithValue = - optionSpecPropsResult.SucceededWith().Concat(valueSpecPropsResult.SucceededWith()); + optionSpecPropsResult.SucceededWith().Concat(valueSpecPropsResult.SucceededWith()).Memorize(); var setPropertyErrors = new List(); @@ -130,11 +133,13 @@ join sp in specPropsWithValue on prms.Name.ToLower() equals sp.Property.Name.ToL return allErrors.Except(warnings).ToParserResult(instance); }; - var preprocessorErrors = arguments.Any() - ? arguments.Preprocess(PreprocessorGuards.Lookup(nameComparer)) - : Enumerable.Empty(); + var preprocessorErrors = ( + argumentsList.Any() + ? argumentsList.Preprocess(PreprocessorGuards.Lookup(nameComparer)) + : Enumerable.Empty() + ).Memorize(); - var result = arguments.Any() + var result = argumentsList.Any() ? preprocessorErrors.Any() ? notParsed(preprocessorErrors) : buildUp() diff --git a/src/CommandLine/Core/OptionMapper.cs b/src/CommandLine/Core/OptionMapper.cs index 0d89b134..49f4889c 100644 --- a/src/CommandLine/Core/OptionMapper.cs +++ b/src/CommandLine/Core/OptionMapper.cs @@ -43,7 +43,7 @@ select Tuple.Create( ((OptionSpecification)pt.Specification).FromOptionSpecification())))) : Tuple.Create(pt, Maybe.Nothing()); } - ); + ).Memorize(); return Result.Succeed( sequencesAndErrors.Select(se => se.Item1), sequencesAndErrors.Select(se => se.Item2).OfType>().Select(se => se.Value)); diff --git a/src/CommandLine/Core/TokenPartitioner.cs b/src/CommandLine/Core/TokenPartitioner.cs index ebe6c0ca..987a8e01 100644 --- a/src/CommandLine/Core/TokenPartitioner.cs +++ b/src/CommandLine/Core/TokenPartitioner.cs @@ -19,14 +19,16 @@ public static IEnumerable tokens, Func> typeLookup) { + IEqualityComparer tokenComparer = ReferenceEqualityComparer.Default; + var tokenList = tokens.Memorize(); - var switches = Switch.Partition(tokenList, typeLookup).Memorize(); - var scalars = Scalar.Partition(tokenList, typeLookup).Memorize(); - var sequences = Sequence.Partition(tokenList, typeLookup).Memorize(); + var switches = Switch.Partition(tokenList, typeLookup).ToSet(tokenComparer); + var scalars = Scalar.Partition(tokenList, typeLookup).ToSet(tokenComparer); + var sequences = Sequence.Partition(tokenList, typeLookup).ToSet(tokenComparer); var nonOptions = tokenList - .Where(t => !switches.Contains(t, ReferenceEqualityComparer.Default)) - .Where(t => !scalars.Contains(t, ReferenceEqualityComparer.Default)) - .Where(t => !sequences.Contains(t, ReferenceEqualityComparer.Default)).Memorize(); + .Where(t => !switches.Contains(t)) + .Where(t => !scalars.Contains(t)) + .Where(t => !sequences.Contains(t)).Memorize(); var values = nonOptions.Where(v => v.IsValue()).Memorize(); var errors = nonOptions.Except(values, (IEqualityComparer)ReferenceEqualityComparer.Default).Memorize(); diff --git a/src/CommandLine/Core/Tokenizer.cs b/src/CommandLine/Core/Tokenizer.cs index e0e09fd3..fb241579 100644 --- a/src/CommandLine/Core/Tokenizer.cs +++ b/src/CommandLine/Core/Tokenizer.cs @@ -36,7 +36,7 @@ public static Result, Error> Tokenize( select token) .Memorize(); - var normalized = normalize(tokens); + var normalized = normalize(tokens).Memorize(); var unkTokens = (from t in normalized where t.IsName() && nameLookup(t.Text) == NameLookupResult.NoOptionFound select t).Memorize(); @@ -60,12 +60,12 @@ public static Result, Error> ExplodeOptionList( Result, Error> tokenizerResult, Func> optionSequenceWithSeparatorLookup) { - var tokens = tokenizerResult.SucceededWith(); + var tokens = tokenizerResult.SucceededWith().Memorize(); var replaces = tokens.Select((t, i) => optionSequenceWithSeparatorLookup(t.Text) .MapValueOrDefault(sep => Tuple.Create(i + 1, sep), - Tuple.Create(-1, '\0'))).SkipWhile(x => x.Item1 < 0); + Tuple.Create(-1, '\0'))).SkipWhile(x => x.Item1 < 0).Memorize(); var exploded = tokens.Select((t, i) => replaces.FirstOrDefault(x => x.Item1 == i).ToMaybe() From 65c6310392f7ac7d2fbbe9ca04eba9c7d75f2a0f Mon Sep 17 00:00:00 2001 From: bberger Date: Thu, 18 Oct 2018 15:02:43 -0700 Subject: [PATCH 2/2] Inline .ToSet() method. I initially added .ToSet() to EnumerableExtensions.cs, not realizing it was a paket file and thus not directly editable in this repo. --- src/CommandLine/Core/TokenPartitioner.cs | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/CommandLine/Core/TokenPartitioner.cs b/src/CommandLine/Core/TokenPartitioner.cs index 987a8e01..cc1d8c26 100644 --- a/src/CommandLine/Core/TokenPartitioner.cs +++ b/src/CommandLine/Core/TokenPartitioner.cs @@ -11,20 +11,16 @@ namespace CommandLine.Core static class TokenPartitioner { public static - Tuple< - IEnumerable>>, // options - IEnumerable, // values - IEnumerable // errors - > Partition( + Tuple>>, IEnumerable, IEnumerable> Partition( IEnumerable tokens, Func> typeLookup) { IEqualityComparer tokenComparer = ReferenceEqualityComparer.Default; var tokenList = tokens.Memorize(); - var switches = Switch.Partition(tokenList, typeLookup).ToSet(tokenComparer); - var scalars = Scalar.Partition(tokenList, typeLookup).ToSet(tokenComparer); - var sequences = Sequence.Partition(tokenList, typeLookup).ToSet(tokenComparer); + var switches = new HashSet(Switch.Partition(tokenList, typeLookup), tokenComparer); + var scalars = new HashSet(Scalar.Partition(tokenList, typeLookup), tokenComparer); + var sequences = new HashSet(Sequence.Partition(tokenList, typeLookup), tokenComparer); var nonOptions = tokenList .Where(t => !switches.Contains(t)) .Where(t => !scalars.Contains(t))