From ae96fa2dca3c73a1955c3b8741c8d766ec57349a Mon Sep 17 00:00:00 2001
From: Jack Dermody <jack@worldtcg.net>
Date: Fri, 16 Sep 2016 19:19:29 +1000
Subject: [PATCH] naive bayes decision trees (first pass) data tables namespace
 fixes

---
 Bayesian/NaiveBayesClassifier.cs              |  97 +++++++
 Bayesian/Training/NaiveBayesTrainer.cs        |  78 ++++++
 BrightWire.Net4.csproj                        |  32 ++-
 Connectionist/Activation/Softmax.cs           |   2 +-
 .../Execution/BidirectionalExecution.cs       |   2 +-
 Connectionist/Execution/RecurrentExecution.cs |   2 +-
 Connectionist/Factory.cs                      |  15 +-
 Connectionist/Training/Batch/BatchTrainer.cs  |   2 +-
 .../Training/Batch/RecurrentBatchTrainer.cs   |  19 +-
 .../Training/Helper/RecurrentContext.cs       |   5 +-
 .../Layer/Recurrent/SimpleRecurrent.cs        |   1 +
 .../Training/Manager/BidirectionalManager.cs  |   8 +-
 .../Training/Manager/FeedForwardManager.cs    |  22 +-
 .../Training/Manager/RecurrentManager.cs      |   9 +-
 .../Training/Manager/RecurrentManagerBase.cs  |  21 +-
 .../Training/WeightInitialisation/Xavier.cs   |   1 +
 Helper/MiniBatch.cs                           |   2 +-
 .../SequenceToSequenceTrainingDataProvider.cs |  69 -----
 .../DataTableTrainingDataProvider.cs          |  72 +++++
 .../DenseSequentialTrainingDataProvider.cs    |   2 +-
 .../DenseTrainingDataProvider.cs              |   4 +-
 .../SequenceToSequenceTrainingDataProvider.cs |  98 +++++++
 .../SparseTrainingDataProvider.cs             |   4 +-
 Interface.cs                                  |  76 ++++-
 LICENSE                                       | 222 ++-------------
 LinearAlgebra/CpuVector.cs                    |  26 ++
 LinearAlgebra/GpuVector.cs                    |  25 ++
 Models/DecisionTree.cs                        |  72 +++++
 Models/ExecutionResults/FeedForwardOutput.cs  |   2 +-
 Models/ExecutionResults/RecurrentOutput.cs    |   2 +-
 Models/NaiveBayes.cs                          |  74 +++++
 TabularData/Analysis/DataTableAnalysis.cs     |  44 +++
 TabularData/Analysis/FrequencyAnalysis.cs     |  37 +++
 TabularData/Analysis/FrequencyCollector.cs    |  58 ++++
 TabularData/Analysis/NumericCollector.cs      | 110 ++++++++
 TabularData/Analysis/StringCollector.cs       |  56 ++++
 TabularData/DataTable.cs                      | 262 +++++-------------
 TabularData/Helper/CSVDataTableBuilder.cs     | 237 ++++++++++++++++
 TabularData/Helper/ClassBasedRowProcessor.cs  |  41 +++
 TabularData/Helper/ColumnTypeClassifier.cs    |  48 ++++
 TabularData/Helper/DataTableProjector.cs      |  66 +++++
 TabularData/Helper/DataTableRow.cs            |  46 +++
 TabularData/Helper/DataTableWriter.cs         |  69 +++++
 TabularData/IndexedDataTable.cs               |  92 ++++++
 TabularData/MemoryBasedDataTable.cs           |  52 ++++
 TabularData/MutableDataTable.cs               | 168 +++++++++++
 TabularData/NumericTable.cs                   |   2 +-
 TrainingData/Artificial/ReberGrammar.cs       |  14 +-
 TreeBased/DecisionTreeClassifier.cs           |  62 +++++
 TreeBased/Training/DecisionTreeTrainer.cs     | 258 +++++++++++++++++
 Unsupervised/Clustering/KMeans.cs             |   2 +-
 51 files changed, 2255 insertions(+), 535 deletions(-)
 create mode 100644 Bayesian/NaiveBayesClassifier.cs
 create mode 100644 Bayesian/Training/NaiveBayesTrainer.cs
 delete mode 100644 Helper/SequenceToSequenceTrainingDataProvider.cs
 create mode 100644 Helper/TrainingData/DataTableTrainingDataProvider.cs
 rename Helper/{ => TrainingData}/DenseSequentialTrainingDataProvider.cs (98%)
 rename Helper/{ => TrainingData}/DenseTrainingDataProvider.cs (95%)
 create mode 100644 Helper/TrainingData/SequenceToSequenceTrainingDataProvider.cs
 rename Helper/{ => TrainingData}/SparseTrainingDataProvider.cs (97%)
 create mode 100644 Models/DecisionTree.cs
 create mode 100644 Models/NaiveBayes.cs
 create mode 100644 TabularData/Analysis/DataTableAnalysis.cs
 create mode 100644 TabularData/Analysis/FrequencyAnalysis.cs
 create mode 100644 TabularData/Analysis/FrequencyCollector.cs
 create mode 100644 TabularData/Analysis/NumericCollector.cs
 create mode 100644 TabularData/Analysis/StringCollector.cs
 create mode 100644 TabularData/Helper/CSVDataTableBuilder.cs
 create mode 100644 TabularData/Helper/ClassBasedRowProcessor.cs
 create mode 100644 TabularData/Helper/ColumnTypeClassifier.cs
 create mode 100644 TabularData/Helper/DataTableProjector.cs
 create mode 100644 TabularData/Helper/DataTableRow.cs
 create mode 100644 TabularData/Helper/DataTableWriter.cs
 create mode 100644 TabularData/IndexedDataTable.cs
 create mode 100644 TabularData/MemoryBasedDataTable.cs
 create mode 100644 TabularData/MutableDataTable.cs
 create mode 100644 TreeBased/DecisionTreeClassifier.cs
 create mode 100644 TreeBased/Training/DecisionTreeTrainer.cs

diff --git a/Bayesian/NaiveBayesClassifier.cs b/Bayesian/NaiveBayesClassifier.cs
new file mode 100644
index 00000000..2234e49d
--- /dev/null
+++ b/Bayesian/NaiveBayesClassifier.cs
@@ -0,0 +1,97 @@
+using BrightWire.Models;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace BrightWire.Bayesian
+{
+    public class NaiveBayesClassifier : IRowProcessor
+    {
+        interface IProbabilityProvider
+        {
+            double GetProbability(IRow row);
+        }
+        class CategoricalColumn : IProbabilityProvider
+        {
+            readonly int _columnIndex;
+            readonly Dictionary<string, double> _probability;
+            readonly double _nullValue;
+
+            public CategoricalColumn(NaiveBayes.CategorialColumn summary, double nullValue = 0)
+            {
+                _nullValue = nullValue;
+                _columnIndex = summary.ColumnIndex;
+                _probability = summary.Probability.ToDictionary(d => d.Category, d => d.LogProbability);
+            }
+
+            public double GetProbability(IRow row)
+            {
+                double ret;
+                var val = row.GetField<string>(_columnIndex);
+                if (_probability.TryGetValue(val, out ret))
+                    return ret;
+                return _nullValue;
+            }
+        }
+        class ContinuousColumn : IProbabilityProvider
+        {
+            readonly NaiveBayes.ContinuousGaussianColumn _column;
+
+            public ContinuousColumn(NaiveBayes.ContinuousGaussianColumn column)
+            {
+                _column = column;
+            }
+
+            public double GetProbability(IRow row)
+            {
+                double x = row.GetField<double>(_column.ColumnIndex);
+                var exponent = Math.Exp(-1 * Math.Pow(x - _column.Mean, 2) / (2 * Math.Pow(_column.Variance, 2)));
+                return Math.Log(1.0 / Math.Sqrt(2 * Math.PI * _column.Variance) * exponent);
+            }
+        }
+        readonly List<Tuple<string, List<IProbabilityProvider>>> _classProbability = new List<Tuple<string, List<IProbabilityProvider>>>();
+        readonly List<Tuple<IRow, string>> _resultList = new List<Tuple<IRow, string>>();
+
+        public NaiveBayesClassifier(NaiveBayes model)
+        {
+            foreach (var cls in model.Class) {
+                List<IProbabilityProvider> list = new List<IProbabilityProvider>();
+                foreach (var col in cls.ColumnSummary) {
+                    if (col.Type == NaiveBayes.ColumnType.Categorical)
+                        list.Add(new CategoricalColumn(col as NaiveBayes.CategorialColumn));
+                    else if (col.Type == NaiveBayes.ColumnType.ContinuousGaussian)
+                        list.Add(new ContinuousColumn(col as NaiveBayes.ContinuousGaussianColumn));
+                }
+                _classProbability.Add(Tuple.Create(cls.Label, list));
+            }
+        }
+
+        public IEnumerable<KeyValuePair<string, double>> Classify(IRow row)
+        {
+            var ret = new Dictionary<string, double>();
+            foreach(var cls in _classProbability) {
+                double score = 0;
+                foreach (var item in cls.Item2)
+                    score += item.GetProbability(row);
+                ret.Add(cls.Item1, score);
+            }
+            return ret.OrderByDescending(kv => kv.Value);
+        }
+
+        public bool Process(IRow row)
+        {
+            var classification = Classify(row).First().Key;
+            _resultList.Add(Tuple.Create(row, classification));
+            return true;
+        }
+
+        public void Clear()
+        {
+            _resultList.Clear();
+        }
+
+        public IReadOnlyList<Tuple<IRow, string>> Results { get { return _resultList; } }
+    }
+}
diff --git a/Bayesian/Training/NaiveBayesTrainer.cs b/Bayesian/Training/NaiveBayesTrainer.cs
new file mode 100644
index 00000000..f5f76c56
--- /dev/null
+++ b/Bayesian/Training/NaiveBayesTrainer.cs
@@ -0,0 +1,78 @@
+using BrightWire.Helper;
+using BrightWire.Models;
+using BrightWire.TabularData;
+using BrightWire.TabularData.Analysis;
+using BrightWire.TabularData.Helper;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace BrightWire.Bayesian.Training
+{
+    public static class NaiveBayesTrainer
+    {
+        public static NaiveBayes Train(IDataTable table, int classColumnIndex)
+        {
+            // analyse the table to get the set of class values
+            var analysis = new DataTableAnalysis(table, classColumnIndex);
+            table.Process(analysis);
+
+            var classInfo = analysis.ColumnInfo.Single();
+            if (classInfo.DistinctValues == null)
+                throw new Exception("Too many class values");
+
+            // analyse the data per class
+            var classBasedFrequency = classInfo.DistinctValues.Select(cv => Tuple.Create<string, IRowProcessor>(cv.ToString(), new FrequencyAnalysis(table, classColumnIndex)));
+            var frequencyAnalysis = new ClassBasedRowProcessor(classBasedFrequency, classColumnIndex);
+            table.Process(frequencyAnalysis);
+
+            // create the per-class summaries from the frequency table
+            var classList = new List<NaiveBayes.ClassSummary>();
+            foreach(var classSummary in frequencyAnalysis.All) {
+                var classLabel = classSummary.Item1;
+                var frequency = classSummary.Item2 as FrequencyAnalysis;
+                var columnList = new List<NaiveBayes.IColumn>();
+                foreach(var column in frequency.ColumnInfo) {
+                    var continuous = column as NumberCollector;
+                    var categorical = column as FrequencyCollector;
+                    if(categorical != null) {
+                        var total = (double)categorical.Total;
+                        if (total > 0) {
+                            var list = new List<NaiveBayes.CategorialProbability>();
+                            foreach (var item in categorical.Frequency) {
+                                list.Add(new NaiveBayes.CategorialProbability {
+                                    Category = item.Key,
+                                    LogProbability = Math.Log(item.Value / total)
+                                });
+                            }
+                            columnList.Add(new NaiveBayes.CategorialColumn {
+                                ColumnIndex = categorical.ColumnIndex,
+                                Probability = list
+                            });
+                        }
+                    }else if(continuous != null) {
+                        var variance = continuous.Variance;
+                        if (variance.HasValue) {
+                            var mean = continuous.Mean;
+                            columnList.Add(new NaiveBayes.ContinuousGaussianColumn {
+                                ColumnIndex = continuous.ColumnIndex,
+                                Mean = mean,
+                                Variance = variance.Value
+                            });
+                        }
+                    }
+                }
+                classList.Add(new NaiveBayes.ClassSummary {
+                    Label = classLabel,
+                    ColumnSummary = columnList
+                });
+            }
+
+            return new NaiveBayes {
+                Class = classList
+            };
+        }
+    }
+}
diff --git a/BrightWire.Net4.csproj b/BrightWire.Net4.csproj
index 7ad84936..9f9892fe 100644
--- a/BrightWire.Net4.csproj
+++ b/BrightWire.Net4.csproj
@@ -7,7 +7,7 @@
     <ProjectGuid>{FD8AAEF6-2EDB-446C-BB19-5EBEE5CDE982}</ProjectGuid>
     <OutputType>Library</OutputType>
     <AppDesignerFolder>Properties</AppDesignerFolder>
-    <RootNamespace>BrightWire.Net4</RootNamespace>
+    <RootNamespace>BrightWire</RootNamespace>
     <AssemblyName>BrightWire.Net4</AssemblyName>
     <TargetFrameworkVersion>v4.6.1</TargetFrameworkVersion>
     <FileAlignment>512</FileAlignment>
@@ -87,6 +87,8 @@
   <ItemGroup>
     <Compile Include="Bayesian\HiddenMarkovModel2.cs" />
     <Compile Include="Bayesian\HiddenMarkovModel3.cs" />
+    <Compile Include="Bayesian\NaiveBayesClassifier.cs" />
+    <Compile Include="Bayesian\Training\NaiveBayesTrainer.cs" />
     <Compile Include="Connectionist\Activation\LeakyRELU.cs" />
     <Compile Include="Connectionist\Activation\RELU.cs" />
     <Compile Include="Connectionist\Activation\Sigmoid.cs" />
@@ -102,7 +104,9 @@
     <Compile Include="Connectionist\Execution\RecurrentExecution.cs" />
     <Compile Include="ErrorMetrics\Quadratic.cs" />
     <Compile Include="Helper\MiniBatch.cs" />
-    <Compile Include="Helper\SequenceToSequenceTrainingDataProvider.cs" />
+    <Compile Include="Helper\TrainingData\DataTableTrainingDataProvider.cs" />
+    <Compile Include="Helper\TrainingData\SequenceToSequenceTrainingDataProvider.cs" />
+    <Compile Include="Models\DecisionTree.cs" />
     <Compile Include="Models\ExecutionResults\FeedForwardOutput.cs" />
     <Compile Include="Models\ExecutionResults\RecurrentOutput.cs" />
     <Compile Include="Connectionist\Factory.cs" />
@@ -136,11 +140,11 @@
     <Compile Include="ExtensionMethods.cs" />
     <Compile Include="Helper\BigEndianBinaryReader.cs" />
     <Compile Include="Helper\BoundMath.cs" />
-    <Compile Include="Helper\DenseSequentialTrainingDataProvider.cs" />
-    <Compile Include="Helper\DenseTrainingDataProvider.cs" />
+    <Compile Include="Helper\TrainingData\DenseSequentialTrainingDataProvider.cs" />
+    <Compile Include="Helper\TrainingData\DenseTrainingDataProvider.cs" />
     <Compile Include="Helper\DisposableMatrixExecutionLine.cs" />
     <Compile Include="Helper\SequenceWindowBuilder.cs" />
-    <Compile Include="Helper\SparseTrainingDataProvider.cs" />
+    <Compile Include="Helper\TrainingData\SparseTrainingDataProvider.cs" />
     <Compile Include="Interface.cs" />
     <Compile Include="LinearAlgebra\CpuMatrix.cs" />
     <Compile Include="LinearAlgebra\CpuVector.cs" />
@@ -153,16 +157,33 @@
     <Compile Include="Models\FeedForwardNetwork.cs" />
     <Compile Include="Models\FloatArray.cs" />
     <Compile Include="Models\HMM.cs" />
+    <Compile Include="Models\NaiveBayes.cs" />
     <Compile Include="Models\NetworkLayer.cs" />
     <Compile Include="Models\RecurrentLayer.cs" />
     <Compile Include="Models\RecurrentNetwork.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
+    <Compile Include="TabularData\Analysis\DataTableAnalysis.cs" />
+    <Compile Include="TabularData\Analysis\FrequencyAnalysis.cs" />
+    <Compile Include="TabularData\Analysis\FrequencyCollector.cs" />
+    <Compile Include="TabularData\Analysis\NumericCollector.cs" />
+    <Compile Include="TabularData\Analysis\StringCollector.cs" />
     <Compile Include="TabularData\DataTable.cs" />
+    <Compile Include="TabularData\Helper\ClassBasedRowProcessor.cs" />
+    <Compile Include="TabularData\Helper\ColumnTypeClassifier.cs" />
+    <Compile Include="TabularData\Helper\CSVDataTableBuilder.cs" />
+    <Compile Include="TabularData\Helper\DataTableProjector.cs" />
+    <Compile Include="TabularData\Helper\DataTableRow.cs" />
+    <Compile Include="TabularData\Helper\DataTableWriter.cs" />
+    <Compile Include="TabularData\IndexedDataTable.cs" />
+    <Compile Include="TabularData\MemoryBasedDataTable.cs" />
+    <Compile Include="TabularData\MutableDataTable.cs" />
     <Compile Include="TabularData\NumericTable.cs" />
     <Compile Include="TrainingData\Artificial\BinaryIntegers.cs" />
     <Compile Include="TrainingData\Artificial\ReberGrammar.cs" />
     <Compile Include="TrainingData\Artificial\Xor.cs" />
     <Compile Include="TrainingData\MNIST.cs" />
+    <Compile Include="TreeBased\DecisionTreeClassifier.cs" />
+    <Compile Include="TreeBased\Training\DecisionTreeTrainer.cs" />
     <Compile Include="Unsupervised\Clustering\KMeans.cs" />
   </ItemGroup>
   <ItemGroup>
@@ -180,6 +201,7 @@
     <Content Include="LinearAlgebra\cuda\kernel.c" />
     <Content Include="LinearAlgebra\cuda\kernel.stub.c" />
   </ItemGroup>
+  <ItemGroup />
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
   <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
        Other similar extension points exist, see Microsoft.Common.targets.
diff --git a/Connectionist/Activation/Softmax.cs b/Connectionist/Activation/Softmax.cs
index 9e02dabd..84c8b2a1 100644
--- a/Connectionist/Activation/Softmax.cs
+++ b/Connectionist/Activation/Softmax.cs
@@ -4,7 +4,7 @@
 using System.Text;
 using System.Threading.Tasks;
 
-namespace BrightWire.Net4.Connectionist.Activation
+namespace BrightWire.Connectionist.Activation
 {
     public class Softmax : IActivationFunction
     {
diff --git a/Connectionist/Execution/BidirectionalExecution.cs b/Connectionist/Execution/BidirectionalExecution.cs
index fc104e2c..f4b14c1a 100644
--- a/Connectionist/Execution/BidirectionalExecution.cs
+++ b/Connectionist/Execution/BidirectionalExecution.cs
@@ -1,5 +1,5 @@
 using BrightWire.Helper;
-using BrightWire.Net4.Models.ExecutionResults;
+using BrightWire.Models.ExecutionResults;
 using System;
 using System.Collections.Generic;
 using System.Linq;
diff --git a/Connectionist/Execution/RecurrentExecution.cs b/Connectionist/Execution/RecurrentExecution.cs
index c9405b33..7a57135f 100644
--- a/Connectionist/Execution/RecurrentExecution.cs
+++ b/Connectionist/Execution/RecurrentExecution.cs
@@ -1,5 +1,5 @@
 using BrightWire.Helper;
-using BrightWire.Net4.Models.ExecutionResults;
+using BrightWire.Models.ExecutionResults;
 using System;
 using System.Collections.Generic;
 using System.Linq;
diff --git a/Connectionist/Factory.cs b/Connectionist/Factory.cs
index 128bda0c..7c2b2732 100644
--- a/Connectionist/Factory.cs
+++ b/Connectionist/Factory.cs
@@ -254,30 +254,33 @@ public IBidirectionalRecurrentExecution CreateBidirectional(BidirectionalNetwork
         public IFeedForwardTrainingManager CreateFeedForwardManager(
             INeuralNetworkTrainer trainer,
             string dataFile,
-            ITrainingDataProvider testData
+            ITrainingDataProvider testData,
+            int? autoAdjustOnNoChangeCount = null
         )
         {
-            return new FeedForwardManager(trainer, dataFile, testData);
+            return new FeedForwardManager(trainer, dataFile, testData, autoAdjustOnNoChangeCount);
         }
 
         public IRecurrentTrainingManager CreateRecurrentManager(
             INeuralNetworkRecurrentBatchTrainer trainer,
             string dataFile,
             ISequentialTrainingDataProvider testData,
-            int memorySize
+            int memorySize,
+            int? autoAdjustOnNoChangeCount = null
         )
         {
-            return new RecurrentManager(trainer, dataFile, testData, memorySize);
+            return new RecurrentManager(trainer, dataFile, testData, memorySize, autoAdjustOnNoChangeCount);
         }
 
         public IBidirectionalRecurrentTrainingManager CreateBidirectionalManager(
             INeuralNetworkBidirectionalBatchTrainer trainer,
             string dataFile,
             ISequentialTrainingDataProvider testData,
-            int memorySize
+            int memorySize,
+            int? autoAdjustOnNoChangeCount = null
         )
         {
-            return new BidirectionalManager(_lap, trainer, dataFile, testData, memorySize);
+            return new BidirectionalManager(_lap, trainer, dataFile, testData, memorySize, autoAdjustOnNoChangeCount);
         }
     }
 }
diff --git a/Connectionist/Training/Batch/BatchTrainer.cs b/Connectionist/Training/Batch/BatchTrainer.cs
index 2fb268db..919f07f9 100644
--- a/Connectionist/Training/Batch/BatchTrainer.cs
+++ b/Connectionist/Training/Batch/BatchTrainer.cs
@@ -6,7 +6,7 @@
 using System.Text;
 using System.Threading.Tasks;
 using BrightWire.Models;
-using BrightWire.Net4.Models.ExecutionResults;
+using BrightWire.Models.ExecutionResults;
 
 namespace BrightWire.Connectionist.Training.Batch
 {
diff --git a/Connectionist/Training/Batch/RecurrentBatchTrainer.cs b/Connectionist/Training/Batch/RecurrentBatchTrainer.cs
index b07c5ad5..3a3fdac2 100644
--- a/Connectionist/Training/Batch/RecurrentBatchTrainer.cs
+++ b/Connectionist/Training/Batch/RecurrentBatchTrainer.cs
@@ -131,34 +131,21 @@ public void TrainOnMiniBatch(ISequentialMiniBatch miniBatch, float[] memory, IRe
             // backpropagate, accumulating errors across the sequence
             using (var updateAccumulator = new UpdateAccumulator(trainingContext)) {
                 IMatrix curr = null;
-                var bpt = 0;
                 while (updateStack.Any()) {
                     var update = updateStack.Pop();
                     var isT0 = !updateStack.Any();
                     var actionStack = update.Item1;
 
                     // calculate error
-                    IMatrix previousError = null;
-                    if (curr != null && bpt < context.BackpropagationThroughTime) {
-                        using (var sum = curr.RowSums(1f / curr.ColumnCount))
-                            previousError = sum.ToColumnMatrix();
-                    }
                     var expectedOutput = update.Item2;
-                    if (expectedOutput != null) {
+                    if (expectedOutput != null)
                         curr = expectedOutput.Subtract(update.Item3);
-                        if (previousError != null)
-                            curr.AddInPlace(previousError);
-                    }
-                    else if (previousError != null)
-                        curr = previousError;
-                    else
-                        continue;
 
                     // backpropagate
                     beforeBackProp?.Invoke(curr);
                     while (actionStack.Any()) {
                         var backpropagationAction = actionStack.Pop();
-                        var shouldCalculateOutput = actionStack.Any() || isT0 || bpt < context.BackpropagationThroughTime;
+                        var shouldCalculateOutput = actionStack.Any() || isT0;
                         curr = backpropagationAction.Execute(curr, trainingContext, true, updateAccumulator);
                     }
                     afterBackProp?.Invoke(curr);
@@ -166,8 +153,6 @@ public void TrainOnMiniBatch(ISequentialMiniBatch miniBatch, float[] memory, IRe
                     // apply any filters
                     foreach (var filter in _filter)
                         filter.AfterBackPropagation(update.Item4, update.Item5, curr);
-
-                    ++bpt;
                 }
 
                 // adjust the initial memory against the error signal
diff --git a/Connectionist/Training/Helper/RecurrentContext.cs b/Connectionist/Training/Helper/RecurrentContext.cs
index 172aad0f..30610ddd 100644
--- a/Connectionist/Training/Helper/RecurrentContext.cs
+++ b/Connectionist/Training/Helper/RecurrentContext.cs
@@ -13,17 +13,14 @@ public class RecurrentContext : IRecurrentTrainingContext
         readonly ILinearAlgebraProvider _lap;
         readonly ITrainingContext _trainingContext;
         readonly List<INeuralNetworkRecurrentTrainerFilter> _filter = new List<INeuralNetworkRecurrentTrainerFilter>();
-        readonly int _backpropagationThroughTime;
 
-        public RecurrentContext(ILinearAlgebraProvider lap, ITrainingContext trainingContext, int backpropagationThroughTime = 0)
+        public RecurrentContext(ILinearAlgebraProvider lap, ITrainingContext trainingContext)
         {
             _lap = lap;
             _trainingContext = trainingContext;
-            _backpropagationThroughTime = backpropagationThroughTime;
         }
 
         public ITrainingContext TrainingContext { get { return _trainingContext; } }
-        public int BackpropagationThroughTime { get { return _backpropagationThroughTime; } }
 
         public void AddFilter(INeuralNetworkRecurrentTrainerFilter filter)
         {
diff --git a/Connectionist/Training/Layer/Recurrent/SimpleRecurrent.cs b/Connectionist/Training/Layer/Recurrent/SimpleRecurrent.cs
index 59cd0248..207889f3 100644
--- a/Connectionist/Training/Layer/Recurrent/SimpleRecurrent.cs
+++ b/Connectionist/Training/Layer/Recurrent/SimpleRecurrent.cs
@@ -43,6 +43,7 @@ public IMatrix Execute(IMatrix curr, ITrainingContext context, bool calculateOut
                     var prevDelta = delta.TransposeAndMultiply(_memoryUpdater.Layer.Weight);
                     var memoryUpdate = _memory.TransposeThisAndMultiply(delta);
                     var inputUpdate = _input.TransposeThisAndMultiply(delta);
+                    
                     //_input.Dispose();
                     _memory.Dispose();
                     _output.Dispose();
diff --git a/Connectionist/Training/Manager/BidirectionalManager.cs b/Connectionist/Training/Manager/BidirectionalManager.cs
index ff5a2d64..9b62d33e 100644
--- a/Connectionist/Training/Manager/BidirectionalManager.cs
+++ b/Connectionist/Training/Manager/BidirectionalManager.cs
@@ -24,7 +24,8 @@ internal class BidirectionalManager : RecurrentManagerBase, IBidirectionalRecurr
             INeuralNetworkBidirectionalBatchTrainer trainer,
             string dataFile,
             ISequentialTrainingDataProvider testData,
-            int memorySize) : base(testData)
+            int memorySize,
+            int? autoAdjustOnNoChangeCount = 5) : base(testData, autoAdjustOnNoChangeCount)
         {
             _lap = lap;
             _trainer = trainer;
@@ -72,6 +73,11 @@ public void Train(ISequentialTrainingDataProvider trainingData, int numEpochs, I
             recurrentContext.TrainingContext.RecurrentEpochComplete -= OnEpoch;
 
             // ensure best values are current
+            ApplyBestParams();
+        }
+
+        public override void ApplyBestParams()
+        {
             if (_bestOutput != null) {
                 _trainer.NetworkInfo = _bestOutput;
                 _forwardMemory = _bestOutput.ForwardMemory.Data;
diff --git a/Connectionist/Training/Manager/FeedForwardManager.cs b/Connectionist/Training/Manager/FeedForwardManager.cs
index 9eedb03a..836aa82b 100644
--- a/Connectionist/Training/Manager/FeedForwardManager.cs
+++ b/Connectionist/Training/Manager/FeedForwardManager.cs
@@ -14,16 +14,19 @@ public class FeedForwardManager : IFeedForwardTrainingManager
         readonly INeuralNetworkTrainer _trainer;
         readonly ITrainingDataProvider _testData;
         readonly string _dataFile;
+        readonly int? _autoAdjustOnNoChangeCount;
         readonly int _reportCadence;
         double _bestScore;
+        int _noChange = 0;
         FeedForwardNetwork _bestOutput = null;
 
-        public FeedForwardManager(INeuralNetworkTrainer trainer, string dataFile, ITrainingDataProvider testData, int reportCadence = 1)
+        public FeedForwardManager(INeuralNetworkTrainer trainer, string dataFile, ITrainingDataProvider testData, int? autoAdjustOnNoChangeCount = 5, int reportCadence = 1)
         {
             _reportCadence = reportCadence;
             _trainer = trainer;
             _dataFile = dataFile;
             _testData = testData;
+            _autoAdjustOnNoChangeCount = autoAdjustOnNoChangeCount;
 
             if (File.Exists(_dataFile)) {
                 using (var stream = new FileStream(_dataFile, FileMode.Open, FileAccess.Read))
@@ -62,6 +65,11 @@ public void Train(ITrainingDataProvider trainingData, int numEpochs, ITrainingCo
             context.EpochComplete -= OnEpochComplete;
 
             // ensure best values are current
+            ApplyBestParams();
+        }
+
+        public void ApplyBestParams()
+        {
             if (_bestOutput != null)
                 _trainer.NetworkInfo = _bestOutput;
         }
@@ -79,9 +87,19 @@ private void OnEpochComplete(ITrainingContext context)
             if((context.CurrentEpoch % _reportCadence) == 0)
                 context.WriteScore(score, errorMetric.DisplayAsPercentage, flag);
 
-            if(flag) {
+            if (flag) {
                 using (var stream = new FileStream(_dataFile, FileMode.Create, FileAccess.Write))
                     Serializer.Serialize(stream, _bestOutput);
+                _noChange = 0;
+            }
+            else
+                ++_noChange;
+
+            if(_autoAdjustOnNoChangeCount.HasValue && _noChange >= _autoAdjustOnNoChangeCount.Value) {
+                _noChange = 0;
+                ApplyBestParams();
+                context.ReduceTrainingRate();
+                Console.WriteLine("Reducing training rate to " + context.TrainingRate);
             }
         }
     }
diff --git a/Connectionist/Training/Manager/RecurrentManager.cs b/Connectionist/Training/Manager/RecurrentManager.cs
index e953351f..bbc58e4a 100644
--- a/Connectionist/Training/Manager/RecurrentManager.cs
+++ b/Connectionist/Training/Manager/RecurrentManager.cs
@@ -18,8 +18,8 @@ internal class RecurrentManager : RecurrentManagerBase, IRecurrentTrainingManage
         RecurrentNetwork _bestOutput;
         float[] _memory;
 
-        public RecurrentManager(INeuralNetworkRecurrentBatchTrainer trainer, string dataFile, ISequentialTrainingDataProvider testData, int memorySize)
-            : base(testData)
+        public RecurrentManager(INeuralNetworkRecurrentBatchTrainer trainer, string dataFile, ISequentialTrainingDataProvider testData, int memorySize, int? autoAdjustOnNoChangeCount = 5)
+            : base(testData, autoAdjustOnNoChangeCount)
         {
             _trainer = trainer;
             _dataFile = dataFile;
@@ -63,6 +63,11 @@ public void Train(ISequentialTrainingDataProvider trainingData, int numEpochs, I
             recurrentContext.TrainingContext.RecurrentEpochComplete -= OnEpochComplete;
 
             // ensure best values are current
+            ApplyBestParams();
+        }
+
+        public override void ApplyBestParams()
+        {
             if (_bestOutput != null) {
                 _trainer.NetworkInfo = _bestOutput;
                 _memory = _bestOutput.Memory.Data;
diff --git a/Connectionist/Training/Manager/RecurrentManagerBase.cs b/Connectionist/Training/Manager/RecurrentManagerBase.cs
index b51d4edf..ea255ea7 100644
--- a/Connectionist/Training/Manager/RecurrentManagerBase.cs
+++ b/Connectionist/Training/Manager/RecurrentManagerBase.cs
@@ -9,13 +9,18 @@
 
 namespace BrightWire.Connectionist.Training.Manager
 {
-    internal class RecurrentManagerBase
+    internal abstract class RecurrentManagerBase
     {
         protected readonly ISequentialTrainingDataProvider _testData;
+        readonly int? _autoAdjustOnNoChangeCount;
+        int _noChange = 0;
 
-        internal RecurrentManagerBase(ISequentialTrainingDataProvider testData)
+        public abstract void ApplyBestParams();
+
+        internal RecurrentManagerBase(ISequentialTrainingDataProvider testData, int? autoAdjustOnNoChangeCount)
         {
             _testData = testData;
+            _autoAdjustOnNoChangeCount = autoAdjustOnNoChangeCount;
         }
 
         protected double _GetScore(ISequentialTrainingDataProvider data, INeuralNetworkRecurrentBatchTrainer network, float[] memory, IRecurrentTrainingContext context)
@@ -43,6 +48,18 @@ protected bool _CalculateTestScore(ITrainingContext context, float[] memory, ISe
                 flag = true;
             }
             context.WriteScore(score, errorMetric.DisplayAsPercentage, flag);
+
+            if (flag)
+                _noChange = 0;
+            else
+                ++_noChange;
+
+            if (_autoAdjustOnNoChangeCount.HasValue && _noChange >= _autoAdjustOnNoChangeCount.Value) {
+                context.ReduceTrainingRate();
+                Console.WriteLine("Reducing training rate to " + context.TrainingRate);
+                ApplyBestParams();
+                _noChange = 0;
+            }
             return flag;
         }
 
diff --git a/Connectionist/Training/WeightInitialisation/Xavier.cs b/Connectionist/Training/WeightInitialisation/Xavier.cs
index 35c6bd70..d5fa998f 100644
--- a/Connectionist/Training/WeightInitialisation/Xavier.cs
+++ b/Connectionist/Training/WeightInitialisation/Xavier.cs
@@ -28,6 +28,7 @@ public float GetWeight(int inputSize, int outputSize, int i, int j)
             IContinuousDistribution distribution;
             if (!_distributionTable.TryGetValue(key, out distribution)) {
                 var stdDev = Math.Sqrt(6) / (inputSize + outputSize);
+                //var stdDev = Math.Sqrt(2 / (inputSize + outputSize));
                 _distributionTable.Add(key, distribution = new Normal(0, stdDev, _random));
             }
             return Convert.ToSingle(distribution.Sample());
diff --git a/Helper/MiniBatch.cs b/Helper/MiniBatch.cs
index a58674ab..b0b485c0 100644
--- a/Helper/MiniBatch.cs
+++ b/Helper/MiniBatch.cs
@@ -4,7 +4,7 @@
 using System.Text;
 using System.Threading.Tasks;
 
-namespace BrightWire.Net4.Helper
+namespace BrightWire.Helper
 {
     internal class MiniBatch : IMiniBatch
     {
diff --git a/Helper/SequenceToSequenceTrainingDataProvider.cs b/Helper/SequenceToSequenceTrainingDataProvider.cs
deleted file mode 100644
index de421990..00000000
--- a/Helper/SequenceToSequenceTrainingDataProvider.cs
+++ /dev/null
@@ -1,69 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
-namespace BrightWire.Net4.Helper
-{
-    public class SequenceToSequenceTrainingDataProvider : ISequentialTrainingDataProvider
-    {
-        readonly ILinearAlgebraProvider _lap;
-        readonly Tuple<int, int>[] _sequenceLength;
-        readonly Dictionary<int, List<Tuple<IReadOnlyList<float>[], IReadOnlyList<float>[], int>>> _inputData;
-        readonly int _inputSize, _totalCount;
-
-        //class MiniBatch : ISequentialMiniBatch
-        //{
-
-        //}
-
-        public SequenceToSequenceTrainingDataProvider(ILinearAlgebraProvider lap, IReadOnlyList<Tuple<IReadOnlyList<float>[], IReadOnlyList<float>[]>> data)
-        {
-            _lap = lap;
-            _totalCount = data.Count;
-
-            // group the data by sequence size
-            _inputData = data
-                .Select((a, ind) => Tuple.Create(a.Item1, a.Item2, ind))
-                .GroupBy(s => s.Item1.Length)
-                .OrderBy(g => g.Key)
-                .ToDictionary(g => g.Key, g => g.ToList())
-            ;
-            _sequenceLength = _inputData.Select(s => Tuple.Create(s.Key, s.Value.Count)).ToArray();
-
-            // find the dimensions of the input and output
-            var firstItem = _inputData.First().Value.First().Item1.First();
-            _inputSize = firstItem.Count;
-        }
-
-        public int Count { get { return _totalCount; } }
-
-        public int InputSize { get { return _inputSize; } }
-        public int OutputSize { get { return _inputSize; } }
-
-        public Tuple<int, int>[] Length
-        {
-            get
-            {
-                return _sequenceLength;
-            }
-        }
-
-        public ISequentialMiniBatch GetTrainingData(int sequenceLength, IReadOnlyList<int> rows)
-        {
-            var input = new IMatrix[sequenceLength];
-            var dataGroup = _inputData[sequenceLength];
-            var outputList = new List<IReadOnlyList<float>[]>();
-
-            foreach(var index in rows)
-                outputList.Add(dataGroup[index].Item2);
-
-            for (var k = 0; k < sequenceLength; k++)
-                input[k] = _lap.Create(rows.Count, _inputSize, (x, y) => dataGroup[rows[x]].Item1[k][y]);
-
-            //return new MiniBatch(input, output, rows.Select(r => dataGroup[r].Item2).ToArray());
-            return null;
-        }
-    }
-}
diff --git a/Helper/TrainingData/DataTableTrainingDataProvider.cs b/Helper/TrainingData/DataTableTrainingDataProvider.cs
new file mode 100644
index 00000000..7a81b9fb
--- /dev/null
+++ b/Helper/TrainingData/DataTableTrainingDataProvider.cs
@@ -0,0 +1,72 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace BrightWire.Helper.TrainingData
+{
+    public class DataTableTrainingDataProvider : ITrainingDataProvider
+    {
+        readonly IIndexableDataTable _table;
+        readonly int _inputSize, _outputSize;
+        readonly ILinearAlgebraProvider _lap;
+
+        public DataTableTrainingDataProvider(ILinearAlgebraProvider lap, IIndexableDataTable table)
+        {
+            _lap = lap;
+            _table = table;
+            _inputSize = table.ColumnCount - 1;
+            _outputSize = _GetClassificationSize(table.Columns.Last());
+        }
+
+        public int Count
+        {
+            get
+            {
+                return _table.RowCount;
+            }
+        }
+
+        public int InputSize
+        {
+            get
+            {
+                return _inputSize;
+            }
+        }
+
+        public int OutputSize
+        {
+            get
+            {
+                return _outputSize;
+            }
+        }
+
+        float _Get(IReadOnlyList<IRow> batchRow, int x, int y)
+        {
+            var row = batchRow[x];
+            return row.GetField<float>(y);
+        }
+
+        int _GetClassificationSize(IColumn column)
+        {
+            return column.NumDistinct;
+        }
+
+        float _GetClassification(IReadOnlyList<IRow> batchRow, int x, int y)
+        {
+            // TODO: compute per row and cache
+            return 0f;
+        }
+
+        public IMiniBatch GetTrainingData(IReadOnlyList<int> rows)
+        {
+            var batchRow = _table.GetRows(rows);
+            var input = _lap.Create(rows.Count, _inputSize, (x, y) => _Get(batchRow, rows[x], y));
+            var output = _lap.Create(rows.Count, _outputSize, (x, y) => _GetClassification(batchRow, rows[x], y));
+            return new MiniBatch(input, output);
+        }
+    }
+}
diff --git a/Helper/DenseSequentialTrainingDataProvider.cs b/Helper/TrainingData/DenseSequentialTrainingDataProvider.cs
similarity index 98%
rename from Helper/DenseSequentialTrainingDataProvider.cs
rename to Helper/TrainingData/DenseSequentialTrainingDataProvider.cs
index 982b7bea..3cfb088b 100644
--- a/Helper/DenseSequentialTrainingDataProvider.cs
+++ b/Helper/TrainingData/DenseSequentialTrainingDataProvider.cs
@@ -4,7 +4,7 @@
 using System.Text;
 using System.Threading.Tasks;
 
-namespace BrightWire.Net4.Helper
+namespace BrightWire.Helper.TrainingData
 {
     public class DenseSequentialTrainingDataProvider : ISequentialTrainingDataProvider
     {
diff --git a/Helper/DenseTrainingDataProvider.cs b/Helper/TrainingData/DenseTrainingDataProvider.cs
similarity index 95%
rename from Helper/DenseTrainingDataProvider.cs
rename to Helper/TrainingData/DenseTrainingDataProvider.cs
index 6f74b4f5..c08c5ebb 100644
--- a/Helper/DenseTrainingDataProvider.cs
+++ b/Helper/TrainingData/DenseTrainingDataProvider.cs
@@ -1,11 +1,11 @@
-using BrightWire.Net4.Helper;
+using BrightWire.Helper;
 using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
 
-namespace BrightWire.Helper
+namespace BrightWire.Helper.TrainingData
 {
     public class DenseTrainingDataProvider : ITrainingDataProvider
     {
diff --git a/Helper/TrainingData/SequenceToSequenceTrainingDataProvider.cs b/Helper/TrainingData/SequenceToSequenceTrainingDataProvider.cs
new file mode 100644
index 00000000..da913584
--- /dev/null
+++ b/Helper/TrainingData/SequenceToSequenceTrainingDataProvider.cs
@@ -0,0 +1,98 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace BrightWire.Helper.TrainingData
+{
+    public class SequenceToSequenceTrainingDataProvider : ISequentialTrainingDataProvider
+    {
+        readonly ILinearAlgebraProvider _lap;
+        readonly Tuple<int, int>[] _sequenceLength;
+        readonly Dictionary<int, List<Tuple<Tuple<float[], float[]>[], int>>> _inputData;
+        readonly int _inputSize, _outputSize, _totalCount;
+        readonly Func<IReadOnlyList<IMatrix>, IMatrix, IMatrix> _errorSignal;
+
+        class MiniBatch : ISequentialMiniBatch
+        {
+            readonly IMatrix[] _expectedOutput;
+            readonly Func<IReadOnlyList<IMatrix>, IMatrix, IMatrix> _errorSignal;
+
+            public int SequenceLength { get; private set; }
+            public int BatchSize { get; private set; }
+            public IMatrix[] Input { get; private set; }
+            public int[] CurrentRows { get; private set; }
+
+            public MiniBatch(IMatrix[] input, IMatrix[] output, int[] rows, Func<IReadOnlyList<IMatrix>, IMatrix, IMatrix> errorSignal)
+            {
+                Input = input;
+                SequenceLength = Input.Length;
+                BatchSize = Input[0].RowCount;
+                CurrentRows = rows;
+                _expectedOutput = output;
+                _errorSignal = errorSignal;
+            }
+
+            public void Dispose()
+            {
+                foreach (var item in _expectedOutput)
+                    item.Dispose();
+                foreach (var item in Input)
+                    item.Dispose();
+            }
+
+            public IMatrix GetExpectedOutput(IReadOnlyList<IMatrix> output, int k)
+            {
+                return _errorSignal(output, _expectedOutput[k]);
+                //return _expectedOutput[k];
+            }
+        }
+
+        public SequenceToSequenceTrainingDataProvider(ILinearAlgebraProvider lap, IReadOnlyList<Tuple<float[], float[]>[]> data, Func<IReadOnlyList<IMatrix>, IMatrix, IMatrix> errorSignal)
+        {
+            _lap = lap;
+            _totalCount = data.Count;
+            _errorSignal = errorSignal;
+
+            // group the data by sequence size
+            _inputData = data
+                .Select((a, ind) => Tuple.Create(a, ind))
+                .GroupBy(s => s.Item1.Length)
+                .OrderBy(g => g.Key)
+                .ToDictionary(g => g.Key, g => g.ToList())
+            ;
+            _sequenceLength = _inputData.Select(s => Tuple.Create(s.Key, s.Value.Count)).ToArray();
+
+            // find the dimensions of the input and output
+            var firstItem = _inputData.First().Value.First().Item1.First();
+            _inputSize = firstItem.Item1.Length;
+            _outputSize = firstItem.Item2.Length;
+        }
+
+        public int Count { get { return _totalCount; } }
+
+        public int InputSize { get { return _inputSize; } }
+        public int OutputSize { get { return _outputSize; } }
+
+        public Tuple<int, int>[] Length
+        {
+            get
+            {
+                return _sequenceLength;
+            }
+        }
+
+        public ISequentialMiniBatch GetTrainingData(int sequenceLength, IReadOnlyList<int> rows)
+        {
+            var input = new IMatrix[sequenceLength];
+            var output = new IMatrix[sequenceLength];
+            var dataGroup = _inputData[sequenceLength];
+            for (var k = 0; k < sequenceLength; k++) {
+                input[k] = _lap.Create(rows.Count, _inputSize, (x, y) => dataGroup[rows[x]].Item1[k].Item1[y]);
+                output[k] = _lap.Create(rows.Count, _outputSize, (x, y) => dataGroup[rows[x]].Item1[k].Item2[y]);
+            }
+            return new MiniBatch(input, output, rows.Select(r => dataGroup[r].Item2).ToArray(), _errorSignal);
+        }
+    }
+}
diff --git a/Helper/SparseTrainingDataProvider.cs b/Helper/TrainingData/SparseTrainingDataProvider.cs
similarity index 97%
rename from Helper/SparseTrainingDataProvider.cs
rename to Helper/TrainingData/SparseTrainingDataProvider.cs
index a7d903de..730046c6 100644
--- a/Helper/SparseTrainingDataProvider.cs
+++ b/Helper/TrainingData/SparseTrainingDataProvider.cs
@@ -1,11 +1,11 @@
-using BrightWire.Net4.Helper;
+using BrightWire.Helper;
 using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
 
-namespace BrightWire.Helper
+namespace BrightWire.Helper.TrainingData
 {
     public class SparseTrainingDataProvider : ITrainingDataProvider
     {
diff --git a/Interface.cs b/Interface.cs
index 843ae8b6..f4e8f978 100644
--- a/Interface.cs
+++ b/Interface.cs
@@ -79,6 +79,7 @@ public interface IVector : IDisposable
         float StdDev(float? mean);
         void Normalise(NormalisationType type);
         IVector Softmax();
+        IVector FindDistances(IVector[] data, DistanceMetric distance);
     }
 
     public interface IIndexableVector : IVector
@@ -231,7 +232,6 @@ public interface INeuralNetworkLayerUpdater : IDisposable
     public interface IRecurrentTrainingContext
     {
         ITrainingContext TrainingContext { get; }
-        int BackpropagationThroughTime { get; }
         void AddFilter(INeuralNetworkRecurrentTrainerFilter filter);
         void ExecuteForward(ISequentialMiniBatch miniBatch, float[] memory, Action<int, List<IMatrix>> onT);
         //void ExecuteForwardSingle(ISequentialMiniBatch miniBatch, float[] memory, int dataIndex, Action<int, List<IMatrix>> onT);
@@ -413,19 +413,22 @@ INeuralNetworkLayerDescriptor descriptor
         IFeedForwardTrainingManager CreateFeedForwardManager(
             INeuralNetworkTrainer trainer,
             string dataFile,
-            ITrainingDataProvider testData
+            ITrainingDataProvider testData,
+            int? autoAdjustOnNoChangeCount = null
         );
         IRecurrentTrainingManager CreateRecurrentManager(
             INeuralNetworkRecurrentBatchTrainer trainer, 
             string dataFile,
             ISequentialTrainingDataProvider testData,
-            int memorySize
+            int memorySize,
+            int? autoAdjustOnNoChangeCount = null
         );
         IBidirectionalRecurrentTrainingManager CreateBidirectionalManager(
             INeuralNetworkBidirectionalBatchTrainer trainer, 
             string dataFile,
             ISequentialTrainingDataProvider testData,
-            int memorySize
+            int memorySize,
+            int? autoAdjustOnNoChangeCount = null
         );
     }
 
@@ -619,18 +622,73 @@ public enum ColumnType
         Double,
         Float,
         Long,
+        Byte,
         Int,
         Date,
-        Boolean,
-        CategoryList,
-        WeightedCategoryList
+        Boolean
     }
 
     public interface IRow
     {
         IReadOnlyList<object> Data { get; }
         T GetField<T>(int index);
-        IEnumerable<float> GetNumericFields(IEnumerable<int> fields);
-        ColumnType GetType(int index);
+    }
+
+    public interface IColumn
+    {
+        string Name { get; }
+        ColumnType Type { get; }
+        int NumDistinct { get; }
+        bool IsContinuous { get; set; }
+    }
+
+    public interface IDataTable
+    {
+        int RowCount { get; }
+        int ColumnCount { get; }
+        IReadOnlyList<IColumn> Columns { get; }
+        void Process(IRowProcessor rowProcessor);
+    }
+
+    public interface IIndexableDataTable : IDataTable
+    {
+        IReadOnlyList<IRow> GetSlice(int offset, int count);
+        IReadOnlyList<IRow> GetRows(IEnumerable<int> rowIndex);
+        Tuple<IDataTable, IDataTable> Split(double trainPercentage = 0.8, bool shuffle = true);
+    }
+
+    public interface IRowProcessor
+    {
+        bool Process(IRow row);
+    }
+
+    public interface IColumnInfo
+    {
+        int ColumnIndex { get; }
+        IEnumerable<object> DistinctValues { get; }
+    }
+
+    public interface IStringColumnInfo : IColumnInfo
+    {
+        int MinLength { get; }
+        int MaxLength { get; }
+        string MostCommonString { get; }
+        int? NumDistinct { get; }
+    }
+
+    public interface INumericColumnInfo : IColumnInfo
+    {
+        double Min { get; }
+        double Max { get; }
+        double Mean { get; }
+        double? StdDev { get; }
+        double? Median { get; }
+        double? Mode { get; }
+        int? NumDistinct { get; }
+    }
+
+    public interface IFrequencyColumnInfo : IColumnInfo
+    {
+        IEnumerable<KeyValuePair<string, ulong>> Frequency { get; }
     }
 }
diff --git a/LICENSE b/LICENSE
index 8dada3ed..84d9b933 100644
--- a/LICENSE
+++ b/LICENSE
@@ -1,201 +1,21 @@
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "{}"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright {yyyy} {name of copyright owner}
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
+MIT License
+
+Copyright (c) 2016 Jack Dermody
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
\ No newline at end of file
diff --git a/LinearAlgebra/CpuVector.cs b/LinearAlgebra/CpuVector.cs
index 24cf14bc..79f61069 100644
--- a/LinearAlgebra/CpuVector.cs
+++ b/LinearAlgebra/CpuVector.cs
@@ -7,6 +7,7 @@
 using System.Linq;
 using BrightWire.Models;
 using BrightWire.Helper;
+using System.Threading.Tasks;
 
 namespace BrightWire.LinearAlgebra
 {
@@ -300,5 +301,30 @@ public IVector Softmax()
                 return new CpuVector(softmax.Divide(sum));
             return new CpuVector(softmax);
         }
+
+        Func<IVector, float> _GetDistanceFunc(DistanceMetric distance)
+        {
+            switch(distance) {
+                case DistanceMetric.Cosine:
+                    return CosineDistance;
+                case DistanceMetric.Euclidean:
+                    return EuclideanDistance;
+                case DistanceMetric.Manhattan:
+                    return ManhattanDistance;
+                case DistanceMetric.MeanSquared:
+                    return MeanSquaredDistance;
+                case DistanceMetric.SquaredEuclidean:
+                    return SquaredEuclidean;
+            }
+            throw new NotImplementedException();
+        }
+
+        public IVector FindDistances(IVector[] data, DistanceMetric distance)
+        {
+            var distanceFunc = _GetDistanceFunc(distance);
+            var ret = new float[data.Length];
+            Parallel.ForEach(data, (vec, ps, ind) => ret[ind] = distanceFunc(vec));
+            return new CpuVector(DenseVector.Create(data.Length, i => ret[i]));
+        }
     }
 }
diff --git a/LinearAlgebra/GpuVector.cs b/LinearAlgebra/GpuVector.cs
index d95b6d1a..c7021673 100644
--- a/LinearAlgebra/GpuVector.cs
+++ b/LinearAlgebra/GpuVector.cs
@@ -376,5 +376,30 @@ public IVector Softmax()
                 _cuda.Blas.Scale(1f / softmaxSum, softmax, 1);
             return new GpuVector(_cuda, _size, softmax);
         }
+
+        Func<IVector, float> _GetDistanceFunc(DistanceMetric distance)
+        {
+            switch (distance) {
+                //case DistanceMetric.Cosine:
+                //    return CosineDistance;
+                case DistanceMetric.Euclidean:
+                    return EuclideanDistance;
+                case DistanceMetric.Manhattan:
+                    return ManhattanDistance;
+                //case DistanceMetric.MeanSquared:
+                //    return MeanSquaredDistance;
+                //case DistanceMetric.SquaredEuclidean:
+                //    return SquaredEuclidean;
+            }
+            throw new NotImplementedException();
+        }
+
+        public IVector FindDistances(IVector[] data, DistanceMetric distance)
+        {
+            var distanceFunc = _GetDistanceFunc(distance);
+            var ret = new float[data.Length];
+            Parallel.ForEach(data, (vec, ps, ind) => ret[ind] = distanceFunc(vec));
+            return new GpuVector(_cuda, data.Length, i => ret[i]);
+        }
     }
 }
diff --git a/Models/DecisionTree.cs b/Models/DecisionTree.cs
new file mode 100644
index 00000000..f3362ac7
--- /dev/null
+++ b/Models/DecisionTree.cs
@@ -0,0 +1,72 @@
+using ProtoBuf;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using System.Xml;
+
+namespace BrightWire.Models
+{
+    [ProtoContract]
+    public class DecisionTree
+    {
+        [ProtoContract]
+        public class Node
+        {
+            [ProtoMember(1)]
+            public Node[] Children { get; set; }
+
+            [ProtoMember(2)]
+            public int ColumnIndex { get; set; }
+
+            [ProtoMember(3)]
+            public string MatchLabel { get; set; }
+
+            [ProtoMember(4)]
+            public double? Split { get; set; }
+
+            [ProtoMember(5)]
+            public string Classification { get; set; }
+
+
+            public void WriteTo(XmlWriter writer)
+            {
+                writer.WriteStartElement("node");
+                if (MatchLabel != null)
+                    writer.WriteAttributeString("matches", MatchLabel);
+
+                if (Split.HasValue)
+                    writer.WriteAttributeString("split", Split.Value.ToString());
+                if(ColumnIndex >= 0)
+                    writer.WriteAttributeString("column", ColumnIndex.ToString());
+                if (Classification != null)
+                    writer.WriteAttributeString("classification", Classification);
+
+                if (Children != null) {
+                    foreach (var child in Children)
+                        child.WriteTo(writer);
+                }
+                writer.WriteEndElement();
+            }
+        }
+
+        [ProtoMember(1)]
+        public int ClassColumnIndex { get; set; }
+
+        [ProtoMember(2)]
+        public Node Root { get; set; }
+
+        public string AsXml
+        {
+            get
+            {
+                var sb = new StringBuilder();
+                if (Root != null)
+                    Root.WriteTo(new XmlTextWriter(new StringWriter(sb)));
+                return sb.ToString();
+            }
+        }
+    }
+}
diff --git a/Models/ExecutionResults/FeedForwardOutput.cs b/Models/ExecutionResults/FeedForwardOutput.cs
index 36b9543a..7ca3866f 100644
--- a/Models/ExecutionResults/FeedForwardOutput.cs
+++ b/Models/ExecutionResults/FeedForwardOutput.cs
@@ -4,7 +4,7 @@
 using System.Text;
 using System.Threading.Tasks;
 
-namespace BrightWire.Net4.Models.ExecutionResults
+namespace BrightWire.Models.ExecutionResults
 {
     public class FeedForwardOutput : IFeedForwardOutput
     {
diff --git a/Models/ExecutionResults/RecurrentOutput.cs b/Models/ExecutionResults/RecurrentOutput.cs
index 07543ffc..1dfc5b05 100644
--- a/Models/ExecutionResults/RecurrentOutput.cs
+++ b/Models/ExecutionResults/RecurrentOutput.cs
@@ -4,7 +4,7 @@
 using System.Text;
 using System.Threading.Tasks;
 
-namespace BrightWire.Net4.Models.ExecutionResults
+namespace BrightWire.Models.ExecutionResults
 {
     public class RecurrentOutput : IRecurrentOutput
     {
diff --git a/Models/NaiveBayes.cs b/Models/NaiveBayes.cs
new file mode 100644
index 00000000..ee1046c1
--- /dev/null
+++ b/Models/NaiveBayes.cs
@@ -0,0 +1,74 @@
+using ProtoBuf;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace BrightWire.Models
+{
+    [ProtoContract]
+    public class NaiveBayes
+    {
+        public enum ColumnType
+        {
+            ContinuousGaussian,
+            Categorical
+        }
+        public interface IColumn
+        {
+            ColumnType Type { get; }
+            int ColumnIndex { get; }
+        }
+
+        [ProtoContract]
+        public class ContinuousGaussianColumn : IColumn
+        {
+            [ProtoMember(1)]
+            public int ColumnIndex { get; set; }
+
+            [ProtoMember(2)]
+            public double Variance { get; set; }
+
+            [ProtoMember(3)]
+            public double Mean { get; set; }
+
+            public ColumnType Type { get { return ColumnType.ContinuousGaussian; } }
+        }
+
+        [ProtoContract]
+        public class CategorialProbability
+        {
+            [ProtoMember(1)]
+            public string Category { get; set; }
+
+            [ProtoMember(2)]
+            public double LogProbability { get; set; }
+        }
+
+        [ProtoContract]
+        public class CategorialColumn : IColumn
+        {
+            [ProtoMember(1)]
+            public int ColumnIndex { get; set; }
+
+            [ProtoMember(2)]
+            public List<CategorialProbability> Probability { get; set; }
+
+            public ColumnType Type { get { return ColumnType.Categorical; } }
+        }
+
+        [ProtoContract]
+        public class ClassSummary
+        {
+            [ProtoMember(1)]
+            public string Label { get; set; }
+
+            [ProtoMember(2)]
+            public List<IColumn> ColumnSummary { get; set; }
+        }
+
+        [ProtoMember(1)]
+        public List<ClassSummary> Class { get; set; }
+    }
+}
diff --git a/TabularData/Analysis/DataTableAnalysis.cs b/TabularData/Analysis/DataTableAnalysis.cs
new file mode 100644
index 00000000..cc57cf5e
--- /dev/null
+++ b/TabularData/Analysis/DataTableAnalysis.cs
@@ -0,0 +1,44 @@
+using BrightWire.TabularData.Helper;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace BrightWire.TabularData.Analysis
+{
+    public class DataTableAnalysis : IRowProcessor
+    {
+        readonly List<IRowProcessor> _column = new List<IRowProcessor>();
+
+        public DataTableAnalysis(IDataTable table)
+        {
+            int index = 0;
+            foreach (var item in table.Columns) {
+                if (ColumnTypeClassifier.IsNumeric(item))
+                    _column.Add(new NumberCollector(index));
+                else if (item.Type == ColumnType.String)
+                    _column.Add(new StringCollector(index));
+                ++index;
+            }
+        }
+
+        public DataTableAnalysis(IDataTable table, int columnIndex)
+        {
+            var item = table.Columns[columnIndex];
+            if (ColumnTypeClassifier.IsNumeric(item))
+                _column.Add(new NumberCollector(columnIndex));
+            else if (item.Type == ColumnType.String)
+                _column.Add(new StringCollector(columnIndex));
+        }
+
+        public bool Process(IRow row)
+        {
+            foreach (var item in _column)
+                item.Process(row);
+            return true;
+        }
+
+        public IEnumerable<IColumnInfo> ColumnInfo { get { return _column.Cast<IColumnInfo>(); } }
+    }
+}
diff --git a/TabularData/Analysis/FrequencyAnalysis.cs b/TabularData/Analysis/FrequencyAnalysis.cs
new file mode 100644
index 00000000..481443bf
--- /dev/null
+++ b/TabularData/Analysis/FrequencyAnalysis.cs
@@ -0,0 +1,37 @@
+using BrightWire.TabularData.Helper;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace BrightWire.TabularData.Analysis
+{
+    public class FrequencyAnalysis : IRowProcessor
+    {
+        readonly List<IRowProcessor> _column = new List<IRowProcessor>();
+
+        public FrequencyAnalysis(IDataTable table, int ignoreColumnIndex = -1)
+        {
+            int index = 0;
+            foreach (var item in table.Columns) {
+                if (index != ignoreColumnIndex) {
+                    if (item.IsContinuous)
+                        _column.Add(new NumberCollector(index));
+                    else if (ColumnTypeClassifier.IsCategorical(item))
+                        _column.Add(new FrequencyCollector(index));
+                }
+                ++index;
+            }
+        }
+
+        public bool Process(IRow row)
+        {
+            foreach (var item in _column)
+                item.Process(row);
+            return true;
+        }
+
+        public IEnumerable<IColumnInfo> ColumnInfo { get { return _column.Cast<IColumnInfo>(); } }
+    }
+}
diff --git a/TabularData/Analysis/FrequencyCollector.cs b/TabularData/Analysis/FrequencyCollector.cs
new file mode 100644
index 00000000..19cd08bc
--- /dev/null
+++ b/TabularData/Analysis/FrequencyCollector.cs
@@ -0,0 +1,58 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace BrightWire.TabularData.Analysis
+{
+    class FrequencyCollector : IRowProcessor, IFrequencyColumnInfo
+    {
+        readonly int _index;
+        readonly Dictionary<string, ulong> _valueCount = new Dictionary<string, ulong>();
+        ulong _total = 0;
+
+        public FrequencyCollector(int index)
+        {
+            _index = index;
+        }
+
+        public int ColumnIndex
+        {
+            get
+            {
+                return _index;
+            }
+        }
+
+        public IEnumerable<object> DistinctValues
+        {
+            get
+            {
+                return _valueCount.Select(kv => kv.Key);
+            }
+        }
+
+        public IEnumerable<KeyValuePair<string, ulong>> Frequency
+        {
+            get
+            {
+                return _valueCount;
+            }
+        }
+
+        public ulong Total { get { return _total; } }
+
+        public bool Process(IRow row)
+        {
+            ulong count;
+            var val = row.GetField<string>(_index);
+            if (_valueCount.TryGetValue(val, out count))
+                _valueCount[val] = count + 1;
+            else
+                _valueCount.Add(val, 1);
+            ++_total;
+            return true;
+        }
+    }
+}
diff --git a/TabularData/Analysis/NumericCollector.cs b/TabularData/Analysis/NumericCollector.cs
new file mode 100644
index 00000000..93165ee3
--- /dev/null
+++ b/TabularData/Analysis/NumericCollector.cs
@@ -0,0 +1,110 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace BrightWire.TabularData.Analysis
+{
+    class NumberCollector : IRowProcessor, INumericColumnInfo
+    {
+        readonly int _index, _maxDistinct;
+        readonly Dictionary<double, ulong> _distinct = new Dictionary<double, ulong>();
+
+        double _mean = 0, _m2 = 0, _min = double.MaxValue, _max = double.MinValue, _mode = 0;
+        ulong _total = 0, _highestCount = 0;
+
+        public NumberCollector(int index, int maxDistinct = 131072 * 4)
+        {
+            _index = index;
+            _maxDistinct = maxDistinct;
+        }
+
+        public bool Process(IRow row)
+        {
+            var val = row.GetField<double>(_index);
+            ++_total;
+
+            // online std deviation and mean 
+            // https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Online_algorithm
+            var delta = val - _mean;
+            _mean += delta / _total;
+            _m2 += delta * (val - _mean);
+
+            // find the min and the max
+            if (val < _min)
+                _min = val;
+            if (val > _max)
+                _max = val;
+
+            // add to distinct values
+            if (_distinct.Count < _maxDistinct) {
+                ulong temp, count = 0;
+                if (_distinct.TryGetValue(val, out temp))
+                    _distinct[val] = count = temp + 1;
+                else
+                    _distinct.Add(val, count = 1);
+                if (count > _highestCount) {
+                    _highestCount = count;
+                    _mode = val;
+                }
+            }
+            return true;
+        }
+
+        public int ColumnIndex { get { return _index; } }
+        public double Min { get { return _min; } }
+        public double Max { get { return _max; } }
+        public double Mean { get { return _mean; } }
+        public double? Variance { get { return _total > 1 ? (_m2 / (_total - 1)) : (double?)null; } }
+        public double? StdDev {
+            get
+            {
+                var variance = Variance;
+                if (variance.HasValue)
+                    return Math.Sqrt(variance.Value);
+                return null;
+            }
+        }
+
+        public double? Median
+        {
+            get
+            {
+                double? ret = null;
+                if (_distinct.Count < _maxDistinct && _distinct.Any()) {
+                    ulong middle = _total / 2, count = 0;
+                    foreach (var item in _distinct.OrderBy(kv => kv.Key)) {
+                        top:
+                        if (count + item.Value >= middle) {
+                            if (ret.HasValue) {
+                                ret = (ret.Value + item.Key) / 2;
+                                break;
+                            }
+                            else {
+                                ret = item.Key;
+                                if (_total % 2 == 0)
+                                    break;
+                                middle = middle + 1;
+                                goto top;
+                            }
+                        }
+                        count += item.Value;
+                    }
+                }
+                return ret;
+            }
+        }
+        public double? Mode
+        {
+            get
+            {
+                if (_distinct.Count < _maxDistinct && _distinct.Any())
+                    return _mode;
+                return null;
+            }
+        }
+        public int? NumDistinct { get { return _distinct.Count < _maxDistinct ? _distinct.Count : (int?)null; } }
+        public IEnumerable<object> DistinctValues { get { return _distinct.Count < _maxDistinct ? _distinct.Select(kv => (object)kv.Key) : null; } }
+    }
+}
diff --git a/TabularData/Analysis/StringCollector.cs b/TabularData/Analysis/StringCollector.cs
new file mode 100644
index 00000000..da0d728f
--- /dev/null
+++ b/TabularData/Analysis/StringCollector.cs
@@ -0,0 +1,56 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace BrightWire.TabularData.Analysis
+{
+    class StringCollector : IRowProcessor, IStringColumnInfo
+    {
+        readonly int _index, _maxDistinct;
+        readonly Dictionary<string, ulong> _distinct = new Dictionary<string, ulong>();
+
+        int _minLength = int.MaxValue, _maxLength = int.MinValue;
+        ulong _total = 0, _highestCount = 0;
+        string _mode = null;
+
+        public StringCollector(int index, int maxDistinct = 131072 * 4)
+        {
+            _index = index;
+            _maxDistinct = maxDistinct;
+        }
+
+        public bool Process(IRow row)
+        {
+            var val = row.GetField<string>(_index);
+            var len = val.Length;
+            if (len < _minLength)
+                _minLength = len;
+            if (len > _maxLength)
+                _maxLength = len;
+            ++_total;
+
+            // add to distinct values
+            if (_distinct.Count < _maxDistinct) {
+                ulong temp, count = 0;
+                if (_distinct.TryGetValue(val, out temp))
+                    _distinct[val] = count = temp + 1;
+                else
+                    _distinct.Add(val, count = 1);
+                if (count > _highestCount) {
+                    _highestCount = count;
+                    _mode = val;
+                }
+            }
+            return true;
+        }
+
+        public int ColumnIndex { get { return _index; } }
+        public int MinLength { get { return _minLength; } }
+        public int MaxLength { get { return _maxLength; } }
+        public string MostCommonString { get { return _distinct.Count < _maxDistinct ? _mode : null; } }
+        public int? NumDistinct { get { return _distinct.Count < _maxDistinct ? _distinct.Count : (int?)null; } }
+        public IEnumerable<object> DistinctValues { get { return _distinct.Count < _maxDistinct ? _distinct.Select(kv => kv.Key) : null; } }
+    }
+}
diff --git a/TabularData/DataTable.cs b/TabularData/DataTable.cs
index 82c2cffb..f72b9de4 100644
--- a/TabularData/DataTable.cs
+++ b/TabularData/DataTable.cs
@@ -1,205 +1,80 @@
-using System;
+using BrightWire.TabularData.Helper;
+using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
 
-namespace BrightWire.Net4.TabularData
+namespace BrightWire.TabularData
 {
-    public class DataTable
+    public class DataTable : IDataTable
     {
-        public class Column
+        class Column : IColumn
         {
-            public ColumnType Type { get; private set; }
+            readonly ColumnType _type;
+            readonly string _name;
+            int _numDistinct;
+            bool? _isContinuous;
 
-            public Column(ColumnType type)
+            public Column(string name, ColumnType type, int numDistinct, bool? isContinuous)
             {
-                Type = type;
+                _name = name;
+                _type = type;
+                _numDistinct = numDistinct;
+                _isContinuous = isContinuous;
             }
-
-            public string TypeName
-            {
-                get
-                {
-                    switch (Type) {
-                        case ColumnType.Boolean:
-                            return "boolean";
-                        case ColumnType.String:
-                            return "string";
-                        case ColumnType.Double:
-                            return "double";
-                        case ColumnType.Float:
-                            return "float";
-                        case ColumnType.Int:
-                            return "int";
-                        case ColumnType.Date:
-                        case ColumnType.Long:
-                            return "long";
-                        case ColumnType.CategoryList:
-                            return "category-list";
-                        case ColumnType.WeightedCategoryList:
-                            return "weighted-category-list";
-                        default:
-                            return "null";
-                    }
-                }
-            }
-        }
-        class Row : IRow
-        {
-            readonly DataTable _table;
-            readonly List<object> _data;
-
-            public Row(DataTable table, IEnumerable<object> data)
+            public ColumnType Type { get { return _type; } }
+            public string Name { get { return _name; } }
+            public int NumDistinct
             {
-                _table = table;
-                _data = new List<object>(data);
+                get { return _numDistinct; }
+                set { _numDistinct = value; }
             }
-
-            public IReadOnlyList<object> Data { get { return _data; } }
-            public T GetField<T>(int index)
+            public bool IsContinuous
             {
-                var ret = _data[index];
-                var targetType = typeof(T);
-                if (ret.GetType() == targetType)
-                    return (T)ret;
-                return (T)Convert.ChangeType(ret, targetType);
+                get { return _isContinuous.HasValue ? _isContinuous.Value : ColumnTypeClassifier.IsContinuous(this); }
+                set { _isContinuous = value; }
             }
-            public IEnumerable<float> GetNumericFields(IEnumerable<int> fields)
-            {
-                return fields.Select(i => Convert.ToSingle(_data[i]));
-            }
-            public ColumnType GetType(int index)
-            {
-                if (index >= 0 && index < _table.ColumnTypes.Count)
-                    return _table.ColumnTypes[index].Type;
-                return ColumnType.Null;
-            }
-        }
-        readonly List<Column> _column = new List<Column>();
-        readonly List<Row> _data = new List<Row>();
-        readonly IReadOnlyList<long> _index = new List<long>();
-        readonly string _name;
-
-        public const int BLOCK_SIZE = 1024;
-
-        public DataTable(string name)
-        {
-            _name = name;
-        }
-
-        public string Name { get { return _name; } }
-        public IReadOnlyList<Column> ColumnTypes { get { return _column; } }
-        public IEnumerable<IRow> Rows { get { return _data; } }
-        public IEnumerable<object> GetColumn(int index)
-        {
-            foreach (var row in _data)
-                yield return row.Data[index];
-        }
-
-        public void Add(Column column)
-        {
-            _column.Add(column);
         }
 
-        public void AddRow(IEnumerable<object> data)
-        {
-            _data.Add(new Row(this, data));
-        }
-
-        public int Count { get { return _data.Count; } }
-
-        public void WriteTo(Stream stream)
-        {
-            // write the meta data
-            var writer = new BinaryWriter(stream, Encoding.UTF8, true);
-            writer.Write(_column.Count);
-            foreach (var column in _column)
-                writer.Write((byte)column.Type);
-
-            WriteDataTo(stream);
-        }
-
-        public static void WriteDataTo(BinaryWriter writer, ColumnType type, object val)
-        {
-            if (type == ColumnType.Date)
-                writer.Write(((DateTime)val).Ticks);
-            else if (type == ColumnType.Boolean)
-                writer.Write((bool)val);
-            else if (type == ColumnType.Double)
-                writer.Write((double)val);
-            else if (type == ColumnType.Float)
-                writer.Write((float)val);
-            else if (type == ColumnType.Int)
-                writer.Write((int)val);
-            else if (type == ColumnType.Long)
-                writer.Write((long)val);
-            else if (type == ColumnType.String)
-                writer.Write((string)val);
-            else if (type == ColumnType.CategoryList) {
-                var data = (int[])val;
-                writer.Write(data.Length);
-                foreach (var item in data) {
-                    writer.Write(item);
-                }
-            }
-            else if (type == ColumnType.WeightedCategoryList) {
-                var data = (Tuple<int, double>[])val;
-                writer.Write(data.Length);
-                foreach (var item in data) {
-                    writer.Write(item.Item1);
-                    writer.Write(item.Item2);
-                }
-            }
-        }
-
-        public int WriteDataTo(Stream stream)
-        {
-            int ret = 0;
-            var writer = new BinaryWriter(stream, Encoding.UTF8, true);
-            foreach (var row in _data) {
-                var index = 0;
-                var rowData = row.Data;
-                foreach (var column in _column) {
-                    var val = rowData[index++];
-                    WriteDataTo(writer, column.Type, val);
-                }
-                ++ret;
-            }
-            return ret;
-        }
+        readonly List<Column> _column = new List<Column>();
+        readonly long _dataOffset;
+        readonly protected Stream _stream;
 
-        public DataTable(Stream stream, IReadOnlyList<long> dataIndex, int maxRowCount)
+        public DataTable(Stream stream)
         {
-            // read the metadata
             var reader = new BinaryReader(stream, Encoding.UTF8, true);
             var columnCount = reader.ReadInt32();
             for (var i = 0; i < columnCount; i++) {
+                var name = reader.ReadString();
                 var type = (ColumnType)reader.ReadByte();
-                Add(new Column(type));
+                var numDistinct = reader.ReadInt32();
+                var continuousSpecified = reader.ReadBoolean();
+                bool? isContinuous = null;
+                if (continuousSpecified)
+                    isContinuous = reader.ReadBoolean();
+                _column.Add(new Column(name, type, numDistinct, isContinuous));
             }
 
-            _index = dataIndex;
-
-            // read the data
-            int index = 0;
-            while (stream.Position < stream.Length && index < maxRowCount) {
-                AddRow(_ReadRow(reader));
-                ++index;
-            }
+            _stream = stream;
+            _dataOffset = stream.Position;
         }
 
-        public IEnumerable<IRow> EnumerateRows(Stream stream)
+        public IReadOnlyList<IColumn> Columns { get { return _column; } }
+
+        public virtual int RowCount
         {
-            var reader = new BinaryReader(stream, Encoding.UTF8, true);
-            while (stream.Position < stream.Length) {
-                yield return new Row(this, _ReadRow(reader));
+            get
+            {
+                return -1;
             }
         }
+        public int ColumnCount { get { return _column.Count; } }
 
-        object _ReadColumn(ColumnType type, BinaryReader reader)
+        object _ReadColumn(Column column, BinaryReader reader)
         {
+            var type = column.Type;
             switch (type) {
                 case ColumnType.String:
                     return reader.ReadString();
@@ -215,51 +90,42 @@ object _ReadColumn(ColumnType type, BinaryReader reader)
                     return new DateTime(reader.ReadInt64());
                 case ColumnType.Long:
                     return reader.ReadInt64();
-                case ColumnType.CategoryList:
-                    return Enumerable.Range(0, reader.ReadInt32()).Select(i => reader.ReadInt32()).ToArray();
-                case ColumnType.WeightedCategoryList:
-                    return Enumerable.Range(0, reader.ReadInt32()).Select(i => Tuple.Create(reader.ReadInt32(), reader.ReadDouble())).ToArray();
+                case ColumnType.Byte:
+                    return reader.ReadByte();
+                //case ColumnType.CategoryList:
+                //    return Enumerable.Range(0, reader.ReadInt32()).Select(i => reader.ReadInt32()).ToArray();
+                //case ColumnType.WeightedCategoryList:
+                //    return Enumerable.Range(0, reader.ReadInt32()).Select(i => Tuple.Create(reader.ReadInt32(), reader.ReadDouble())).ToArray();
                 default:
                     return null;
             }
         }
 
-        object[] _ReadRow(BinaryReader reader)
+        protected object[] _ReadRow(BinaryReader reader)
         {
             var row = new object[_column.Count];
             for (var j = 0; j < _column.Count; j++)
-                row[j] = _ReadColumn(_column[j].Type, reader);
+                row[j] = _ReadColumn(_column[j], reader);
             return row;
         }
 
-        void _SkipRow(BinaryReader reader)
+        protected void _SkipRow(BinaryReader reader)
         {
             for (var j = 0; j < _column.Count; j++)
-                _ReadColumn(_column[j].Type, reader);
-        }
-
-        public void LoadData(Stream stream, int offset, int count, int fileSize)
-        {
-            var reader = new BinaryReader(stream, Encoding.UTF8, true);
-            stream.Seek(_index[offset / BLOCK_SIZE], SeekOrigin.Begin);
-
-            // seek to offset
-            for (var i = 0; i < offset % BLOCK_SIZE; i++)
-                _SkipRow(reader);
-
-            // read the data
-            for (var i = 0; i < count && stream.Position < fileSize && stream.Position < stream.Length; i++)
-                AddRow(_ReadRow(reader));
+                _ReadColumn(_column[j], reader);
         }
 
-        public void Clear()
+        public void Process(IRowProcessor rowProcessor)
         {
-            _data.Clear();
-        }
-
-        public NumericTable GetNumericColumns(params int[] columns)
-        {
-            return new NumericTable(Rows.Select(r => r.GetNumericFields(columns).ToArray()).ToList());
+            lock (_stream) {
+                _stream.Seek(_dataOffset, SeekOrigin.Begin);
+                var reader = new BinaryReader(_stream, Encoding.UTF8, true);
+                while (_stream.Position < _stream.Length) {
+                    var row = new DataTableRow(this, _ReadRow(reader));
+                    if (!rowProcessor.Process(row))
+                        break;
+                }
+            }
         }
     }
 }
diff --git a/TabularData/Helper/CSVDataTableBuilder.cs b/TabularData/Helper/CSVDataTableBuilder.cs
new file mode 100644
index 00000000..72a282f5
--- /dev/null
+++ b/TabularData/Helper/CSVDataTableBuilder.cs
@@ -0,0 +1,237 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace BrightWire.TabularData.Helper
+{
+    public class CSVDataTableBuilder
+    {
+        static HashSet<string> _booleanTrueFields = new HashSet<string>();
+        static HashSet<string> _booleanFalseFields = new HashSet<string>();
+        static CSVDataTableBuilder()
+        {
+            _booleanTrueFields.Add("Y");
+            _booleanFalseFields.Add("N");
+
+            _booleanTrueFields.Add("YES");
+            _booleanFalseFields.Add("NO");
+
+            _booleanTrueFields.Add("TRUE");
+            _booleanFalseFields.Add("FALSE");
+
+            _booleanTrueFields.Add("ON");
+            _booleanFalseFields.Add("OFF");
+        }
+
+        readonly char _delimiter;
+
+        public CSVDataTableBuilder(char delimiter = ',')
+        {
+            _delimiter = delimiter;
+        }
+
+        public IndexedDataTable Parse(StreamReader reader, Stream output = null, bool? hasHeader = null)
+        {
+            if (output == null)
+                output = new MemoryStream();
+
+            //preview the file
+            var lines = new List<string>();
+            while (!reader.EndOfStream && lines.Count < IndexedDataTable.BLOCK_SIZE) {
+                var line = reader.ReadLine();
+                if (String.IsNullOrEmpty(line))
+                    continue;
+                lines.Add(line);
+            }
+
+            if (lines.Any()) {
+                // use the preview to determine the data type
+                bool hasHeaderRow = hasHeader.HasValue ? hasHeader.Value : false;
+                var writer = _DetermineHeaders(lines, !hasHeader.HasValue, ref hasHeaderRow);
+
+                // add the preview lines
+                foreach (var line in lines.Skip(hasHeaderRow ? 1 : 0))
+                    _Add(line, writer);
+
+                // parse the remaining data
+                int pos = 0;
+                while (!reader.EndOfStream) {
+                    if (pos % IndexedDataTable.BLOCK_SIZE == 0)
+                        writer.WriteTo(output);
+                    var line = reader.ReadLine();
+                    if (String.IsNullOrEmpty(line))
+                        continue;
+                    _Add(line, writer);
+                    ++pos;
+                }
+                writer.WriteTo(output);
+                return writer.GetIndexedTable(output);
+            }
+            return null;
+        }
+
+        private DataTableWriter _DetermineHeaders(List<string> lines, bool checkForHeader, ref bool hasHeader)
+        {
+            // see if there is a header (all strings)
+            var firstLineTypes = _Parse(lines.First());
+            if (checkForHeader)
+                hasHeader = firstLineTypes.All(str => _DetermineType(str) == ColumnType.String);
+
+            // get the list of header names
+            var headerNames = new List<string>();
+            int index = 0;
+            foreach (var item in firstLineTypes)
+                headerNames.Add(hasHeader ? item : "_col" + index++);
+
+            // get the list of header types
+            var data = lines
+                .Skip(hasHeader ? 1 : 0)
+                .SelectMany(line => _Parse(line).Select((str, pos) => Tuple.Create(str, pos)))
+                .GroupBy(l => l.Item2, l => _DetermineType(l.Item1))
+                .OrderBy(g => g.Key)
+                .Select(g => g.Min(v => (int)v))
+                .Cast<ColumnType>()
+                .ToList()
+            ;
+
+            // add the columns
+            var ret = new DataTableWriter();
+            foreach (var column in headerNames.Zip(data, (name, type) => Tuple.Create(name, type)))
+                ret.AddColumn(column.Item1, column.Item2);
+
+            return ret;
+        }
+
+        ColumnType _DetermineType(string str)
+        {
+            double dbl;
+            float flt;
+            long lng;
+            int it;
+            DateTime date;
+            byte b;
+
+            if (byte.TryParse(str, out b))
+                return ColumnType.Byte;
+            else if (int.TryParse(str, out it))
+                return ColumnType.Int;
+            else if (long.TryParse(str, out lng))
+                return ColumnType.Long;
+            else if (float.TryParse(str, out flt))
+                return ColumnType.Float;
+            else if (double.TryParse(str, out dbl))
+                return ColumnType.Double;
+            else if (DateTime.TryParse(str, out date))
+                return ColumnType.Date;
+            else {
+                var upperStr = str.ToUpperInvariant();
+                if (_booleanTrueFields.Contains(upperStr) || _booleanFalseFields.Contains(upperStr))
+                    return ColumnType.Boolean;
+                return ColumnType.String;
+            }
+        }
+
+        IEnumerable<string> _Parse(string line)
+        {
+            bool inQuote = false;
+            var curr = new StringBuilder();
+
+            for (int i = 0, len = line.Length; i < len; i++) {
+                var ch = line[i];
+                if (ch == '"') {
+                    if (inQuote) {
+                        if (i + 1 < len && line[i + 1] == '"') {
+                            ++i;
+                            curr.Append(ch);
+                        }
+                        else {
+                            inQuote = false;
+                            continue;
+                        }
+                    }
+                    else {
+                        inQuote = true;
+                        continue;
+                    }
+                }
+                else if (ch == _delimiter && !inQuote) {
+                    yield return curr.ToString();
+                    curr.Clear();
+                }
+                else
+                    curr.Append(ch);
+            }
+            if (curr.Length > 0)
+                yield return curr.ToString();
+        }
+
+        private void _Add(string line, DataTableWriter writer)
+        {
+            var convertedData = writer.Columns
+                .Zip(_Parse(line), (c, str) => _Convert(ref c._type, str))
+                .ToList()
+            ;
+            writer.AddRow(convertedData);
+        }
+
+        private object _Convert(ref ColumnType type, string str)
+        {
+            double dbl;
+            float flt;
+            int i;
+            byte b;
+            long l;
+
+            switch (type) {
+                case ColumnType.Boolean:
+                    return _booleanTrueFields.Contains(str.ToUpperInvariant());
+
+                case ColumnType.Date:
+                    return DateTime.Parse(str);
+
+                case ColumnType.Double:
+                    return double.Parse(str);
+
+                case ColumnType.Long:
+                    return long.Parse(str);
+
+                case ColumnType.Float:
+                    if (float.TryParse(str, out flt))
+                        return flt;
+                    if(double.TryParse(str, out dbl)) {
+                        type = ColumnType.Double;
+                        return dbl;
+                    }
+                    return default(float);
+
+                case ColumnType.Int:
+                    if (int.TryParse(str, out i))
+                        return i;
+                    if(long.TryParse(str, out l)) {
+                        type = ColumnType.Long;
+                        return l;
+                    }
+                    return default(int);
+
+                case ColumnType.Byte:
+                    if (byte.TryParse(str, out b))
+                        return b;
+                    if (int.TryParse(str, out i)) {
+                        type = ColumnType.Int;
+                        return i;
+                    }
+                    if (long.TryParse(str, out l)) {
+                        type = ColumnType.Long;
+                        return l;
+                    }
+                    return default(byte);
+
+                default:
+                    return str;
+            }
+        }
+    }
+}
diff --git a/TabularData/Helper/ClassBasedRowProcessor.cs b/TabularData/Helper/ClassBasedRowProcessor.cs
new file mode 100644
index 00000000..1c535186
--- /dev/null
+++ b/TabularData/Helper/ClassBasedRowProcessor.cs
@@ -0,0 +1,41 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace BrightWire.TabularData.Helper
+{
+    public class ClassBasedRowProcessor : IRowProcessor
+    {
+        readonly int _classColumnIndex;
+        readonly Dictionary<string, List<IRowProcessor>> _columnProcessor = new Dictionary<string, List<IRowProcessor>>();
+
+        public ClassBasedRowProcessor(IEnumerable<Tuple<string, IRowProcessor>> classBasedProcessors, int classColumnIndex)
+        {
+            _classColumnIndex = classColumnIndex;
+
+            List<IRowProcessor> temp;
+            foreach (var item in classBasedProcessors) {
+                if (!_columnProcessor.TryGetValue(item.Item1, out temp))
+                    _columnProcessor.Add(item.Item1, temp = new List<IRowProcessor>());
+                temp.Add(item.Item2);
+            }
+        }
+
+        public bool Process(IRow row)
+        {
+            var classValue = row.GetField<string>(_classColumnIndex);
+            if (classValue != null) {
+                List<IRowProcessor> temp;
+                if(_columnProcessor.TryGetValue(classValue, out temp)) {
+                    foreach(var item in temp)
+                        item.Process(row);
+                }
+            }
+            return true;
+        }
+
+        public IEnumerable<Tuple<string, IRowProcessor>> All { get { return _columnProcessor.SelectMany(d => d.Value.Select(d2 => Tuple.Create(d.Key, d2))); } }
+    }
+}
diff --git a/TabularData/Helper/ColumnTypeClassifier.cs b/TabularData/Helper/ColumnTypeClassifier.cs
new file mode 100644
index 00000000..aa528ae8
--- /dev/null
+++ b/TabularData/Helper/ColumnTypeClassifier.cs
@@ -0,0 +1,48 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace BrightWire.TabularData.Helper
+{
+    public static class ColumnTypeClassifier
+    {
+        const int MAX_CATEGORICAL_VALUES = 256;
+
+        static HashSet<ColumnType> _continuousType = new HashSet<ColumnType> {
+            ColumnType.Date,
+            ColumnType.Double,
+            ColumnType.Float,
+            ColumnType.Int,
+            ColumnType.Long,
+            ColumnType.Byte,
+        };
+        static HashSet<ColumnType> _categoricalType = new HashSet<ColumnType> {
+            ColumnType.Boolean,
+            ColumnType.String,
+        };
+        static HashSet<ColumnType> _numericType = new HashSet<ColumnType> {
+            ColumnType.Boolean,
+            ColumnType.Date,
+            ColumnType.Double,
+            ColumnType.Float,
+            ColumnType.Int,
+            ColumnType.Long,
+            ColumnType.Byte
+        };
+
+        public static bool IsNumeric(IColumn column)
+        {
+            return _numericType.Contains(column.Type);
+        }
+        public static bool IsContinuous(IColumn column)
+        {
+            return _continuousType.Contains(column.Type);
+        }
+        public static bool IsCategorical(IColumn column)
+        {
+            return _categoricalType.Contains(column.Type);
+        }
+    }
+}
diff --git a/TabularData/Helper/DataTableProjector.cs b/TabularData/Helper/DataTableProjector.cs
new file mode 100644
index 00000000..c2184cdb
--- /dev/null
+++ b/TabularData/Helper/DataTableProjector.cs
@@ -0,0 +1,66 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace BrightWire.TabularData.Helper
+{
+    public class DataTableProjector : IRowProcessor
+    {
+        readonly IRowProcessor _destination;
+        readonly IReadOnlyList<int> _validColumn;
+
+        class ProjectedRow : IRow
+        {
+            readonly IReadOnlyList<object> _data;
+
+            public ProjectedRow(IReadOnlyList<object> data)
+            {
+                _data = data;
+            }
+
+            public IReadOnlyList<object> Data
+            {
+                get
+                {
+                    return _data;
+                }
+            }
+
+            public T GetField<T>(int index)
+            {
+                return DataTableRow.GetField<T>(_data, index);
+            }
+        }
+
+        public DataTableProjector(IRowProcessor destination, IEnumerable<int> columns)
+        {
+            _destination = destination;
+            _validColumn = columns.ToList();
+        }
+
+        public bool Process(IRow row)
+        {
+            var data = row.Data;
+            var newRow = new List<object>();
+            foreach(var item in _validColumn)
+                newRow.Add(data[item]);
+            return _destination.Process(new ProjectedRow(newRow));
+        }
+
+        public static IIndexableDataTable Project(IDataTable table, IEnumerable<int> columns)
+        {
+            var validColumn = new HashSet<int>(columns);
+            var writer = new DataTableWriter(table.Columns.Select((c, i) => Tuple.Create(c, i)).Where(c => validColumn.Contains(c.Item2)).Select(c => c.Item1));
+            var projector = new DataTableProjector(writer, columns);
+            table.Process(projector);
+            return writer.GetIndexedTable();
+        }
+
+        public static IIndexableDataTable Project(IDataTable table, params int[] columns)
+        {
+            return Project(table, columns);
+        }
+    }
+}
diff --git a/TabularData/Helper/DataTableRow.cs b/TabularData/Helper/DataTableRow.cs
new file mode 100644
index 00000000..f7a722c7
--- /dev/null
+++ b/TabularData/Helper/DataTableRow.cs
@@ -0,0 +1,46 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace BrightWire.TabularData.Helper
+{
+    public class DataTableRow : IRow
+    {
+        readonly IDataTable _table;
+        readonly List<object> _data;
+
+        public DataTableRow(IDataTable table, IEnumerable<object> data)
+        {
+            _table = table;
+            _data = new List<object>(data);
+        }
+
+        public IReadOnlyList<object> Data { get { return _data; } }
+
+        public static T GetField<T>(IReadOnlyList<object> data, int index)
+        {
+            var ret = data[index];
+            var targetType = typeof(T);
+            if (ret.GetType() == targetType)
+                return (T)ret;
+            return (T)Convert.ChangeType(ret, targetType);
+        }
+
+        public T GetField<T>(int index)
+        {
+            return GetField<T>(_data, index);
+        }
+        public IEnumerable<float> GetNumericFields(IEnumerable<int> fields)
+        {
+            return fields.Select(i => Convert.ToSingle(_data[i]));
+        }
+        public IColumn GetColumn(int index)
+        {
+            if (index >= 0 && index < _table.Columns.Count)
+                return _table.Columns[index];
+            return null;
+        }
+    }
+}
diff --git a/TabularData/Helper/DataTableWriter.cs b/TabularData/Helper/DataTableWriter.cs
new file mode 100644
index 00000000..f3f35119
--- /dev/null
+++ b/TabularData/Helper/DataTableWriter.cs
@@ -0,0 +1,69 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace BrightWire.TabularData.Helper
+{
+    public class DataTableWriter : IRowProcessor
+    {
+        readonly List<long> _index = new List<long>();
+        readonly MutableDataTable _dataTable;
+        int _rowCount = 0;
+        bool _hasWrittenHeader = false;
+
+        public DataTableWriter()
+        {
+            _dataTable = new MutableDataTable();
+        }
+        public DataTableWriter(IEnumerable<IColumn> columns)
+        {
+            _dataTable = new MutableDataTable(columns);
+        }
+
+        public IReadOnlyList<MutableDataTable.Column> Columns { get { return _dataTable.Columns2; } }
+        public IReadOnlyList<long> Index { get { return _index; } }
+        public int RowCount { get { return _rowCount; } }
+
+        public IndexedDataTable GetIndexedTable(Stream stream = null)
+        {
+            if (stream == null)
+                stream = new MemoryStream();
+
+            WriteTo(stream);
+            stream.Seek(0, SeekOrigin.Begin);
+            return new IndexedDataTable(stream, _index, _rowCount);
+        }
+
+        public void AddColumn(string name, ColumnType type)
+        {
+            _dataTable.Add(type, name);
+        }
+
+        public void AddRow(IEnumerable<object> row)
+        {
+            _dataTable.AddRow(row);
+        }
+
+        public void WriteTo(Stream stream)
+        {
+            if (!_hasWrittenHeader) {
+                _dataTable.WriteMetadata(stream);
+                _hasWrittenHeader = true;
+            }
+            if (_dataTable.RowCount > 0) {
+                _index.Add(stream.Position);
+                _rowCount += _dataTable.WriteData(stream);
+                _dataTable.Clear();
+            }
+        }
+
+        public bool Process(IRow row)
+        {
+            AddRow(row.Data);
+            return true;
+        }
+    }
+}
diff --git a/TabularData/IndexedDataTable.cs b/TabularData/IndexedDataTable.cs
new file mode 100644
index 00000000..160582d1
--- /dev/null
+++ b/TabularData/IndexedDataTable.cs
@@ -0,0 +1,92 @@
+using BrightWire.TabularData.Helper;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace BrightWire.TabularData
+{
+    public class IndexedDataTable : DataTable, IIndexableDataTable
+    {
+        public const int BLOCK_SIZE = 1024;
+
+        readonly IReadOnlyList<long> _index = new List<long>();
+        readonly int _rowCount;
+
+        public IndexedDataTable(Stream stream, IReadOnlyList<long> dataIndex, int rowCount) 
+            : base(stream)
+        {
+            _index = dataIndex;
+            _rowCount = rowCount;
+        }
+
+        public IReadOnlyList<long> Index { get { return _index; } }
+
+        public override int RowCount
+        {
+            get
+            {
+                return _rowCount;
+            }
+        }
+
+        public IReadOnlyList<IRow> GetSlice(int offset, int count)
+        {
+            var ret = new List<IRow>();
+            lock (_stream) {
+                var reader = new BinaryReader(_stream, Encoding.UTF8, true);
+                _stream.Seek(_index[offset / BLOCK_SIZE], SeekOrigin.Begin);
+
+                // seek to offset
+                for (var i = 0; i < offset % BLOCK_SIZE; i++)
+                    _SkipRow(reader);
+
+                // read the data
+                for (var i = 0; i < count && _stream.Position < _stream.Length; i++)
+                    ret.Add(new DataTableRow(this, _ReadRow(reader)));
+            }
+            return ret;
+        }
+
+        public IReadOnlyList<IRow> GetRows(IEnumerable<int> rowIndex)
+        {
+            // split the queries into blocks
+            HashSet<int> temp;
+            var blockMatch = new Dictionary<int, HashSet<int>>();
+            foreach (var row in rowIndex.OrderBy(r => r)) {
+                var block = row / BLOCK_SIZE;
+                if (!blockMatch.TryGetValue(block, out temp))
+                    blockMatch.Add(block, temp = new HashSet<int>());
+                temp.Add(row);
+            }
+
+            var ret = new List<IRow>();
+            lock (_stream) {
+                var reader = new BinaryReader(_stream, Encoding.UTF8, true);
+                foreach (var block in blockMatch.OrderBy(b => b.Key)) {
+                    _stream.Seek(_index[block.Key], SeekOrigin.Begin);
+                    var match = block.Value;
+
+                    for (int i = block.Key * BLOCK_SIZE, len = i + BLOCK_SIZE; i < len && _stream.Position < _stream.Length; i++) {
+                        if (match.Contains(i))
+                            ret.Add(new DataTableRow(this, _ReadRow(reader)));
+                    }
+                }
+            }
+            return ret;
+        }
+
+        public Tuple<IDataTable, IDataTable> Split(double trainPercentage = 0.8, bool shuffle = true)
+        {
+            var shuffled = Enumerable.Range(0, RowCount).Shuffle().ToList();
+            int trainingCount = Convert.ToInt32(RowCount * trainPercentage);
+
+            return Tuple.Create<IDataTable, IDataTable>(
+                new MemoryBasedDataTable(Columns, GetRows(shuffled.Take(trainingCount))), 
+                new MemoryBasedDataTable(Columns, GetRows(shuffled.Skip(trainingCount)))
+            );
+        }
+    }
+}
diff --git a/TabularData/MemoryBasedDataTable.cs b/TabularData/MemoryBasedDataTable.cs
new file mode 100644
index 00000000..6cb2c244
--- /dev/null
+++ b/TabularData/MemoryBasedDataTable.cs
@@ -0,0 +1,52 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace BrightWire.TabularData
+{
+    public class MemoryBasedDataTable : IDataTable
+    {
+        readonly IReadOnlyList<IColumn> _columns;
+        readonly IReadOnlyList<IRow> _data;
+
+        public MemoryBasedDataTable(IReadOnlyList<IColumn> columns, IReadOnlyList<IRow> data)
+        {
+            _columns = columns;
+            _data = data;
+        }
+
+        public int ColumnCount
+        {
+            get
+            {
+                return _columns.Count;
+            }
+        }
+
+        public IReadOnlyList<IColumn> Columns
+        {
+            get
+            {
+                return _columns;
+            }
+        }
+
+        public int RowCount
+        {
+            get
+            {
+                return _data.Count;
+            }
+        }
+
+        public void Process(IRowProcessor rowProcessor)
+        {
+            foreach(var item in _data) {
+                if (!rowProcessor.Process(item))
+                    break;
+            }
+        }
+    }
+}
diff --git a/TabularData/MutableDataTable.cs b/TabularData/MutableDataTable.cs
new file mode 100644
index 00000000..09578cb7
--- /dev/null
+++ b/TabularData/MutableDataTable.cs
@@ -0,0 +1,168 @@
+using BrightWire.TabularData.Helper;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace BrightWire.TabularData
+{
+    public class MutableDataTable : IDataTable
+    {
+        const int MAX_UNIQUE = 131072 * 4;
+
+        public class Column : IColumn
+        {
+            public bool? _isContinuous;
+            public ColumnType _type;
+            readonly HashSet<string> _uniqueValues = new HashSet<string>();
+            readonly string _name;
+
+            public Column(ColumnType type, string name)
+            {
+                _type = type;
+                _name = name;
+            }
+
+            public int NumDistinct
+            {
+                get
+                {
+                    return _uniqueValues.Count == MAX_UNIQUE ? 0 : _uniqueValues.Count;
+                }
+            }
+
+            public void Add(object value)
+            {
+                if (_uniqueValues.Count < MAX_UNIQUE)
+                    _uniqueValues.Add(value.ToString());
+            }
+
+            public ColumnType Type { get { return _type; } }
+            public string Name { get { return _name; } }
+
+            public bool IsContinuous
+            {
+                get { return _isContinuous.HasValue && _isContinuous.Value; }
+                set { _isContinuous = value; }
+            }
+        }
+        readonly List<Column> _column = new List<Column>();
+        readonly List<DataTableRow> _data = new List<DataTableRow>();
+
+        public MutableDataTable()
+        {
+        }
+        public MutableDataTable(IEnumerable<IColumn> columns)
+        {
+            foreach (var column in columns) {
+                var col = Add(column.Type, column.Name);
+                col.IsContinuous = column.IsContinuous;
+            }
+        }
+
+        public IReadOnlyList<IColumn> Columns { get { return _column; } }
+        public IReadOnlyList<Column> Columns2 { get { return _column; } }
+        public int RowCount { get { return _data.Count; } }
+        public int ColumnCount { get { return _column.Count; } }
+
+        public Column Add(ColumnType column, string name = "")
+        {
+            var ret = new Column(column, name);
+            _column.Add(ret);
+            return ret;
+        }
+
+        public IRow AddRow(IEnumerable<object> data)
+        {
+            var row = new DataTableRow(this, data);
+            _data.Add(row);
+
+            var data2 = row.Data;
+            for(int i = 0, len = data2.Count; i < len && i < _column.Count; i++)
+                _column[i].Add(data2[i]);
+            return row;
+        }
+
+        public void WriteMetadata(Stream stream)
+        {
+            var writer = new BinaryWriter(stream, Encoding.UTF8, true);
+            writer.Write(_column.Count);
+            foreach (var column in _column) {
+                writer.Write(column.Name);
+                writer.Write((byte)column.Type);
+                writer.Write(column.NumDistinct);
+                writer.Write(column._isContinuous.HasValue);
+                if (column._isContinuous.HasValue)
+                    writer.Write(column._isContinuous.Value);
+            }
+        }
+
+        void _WriteValueTo(BinaryWriter writer, Column column, object val)
+        {
+            var type = column.Type;
+
+            if (type == ColumnType.Date)
+                writer.Write(((DateTime)val).Ticks);
+            else if (type == ColumnType.Boolean)
+                writer.Write((bool)val);
+            else if (type == ColumnType.Double)
+                writer.Write((double)val);
+            else if (type == ColumnType.Float)
+                writer.Write((float)val);
+            else if (type == ColumnType.Int)
+                writer.Write((int)val);
+            else if (type == ColumnType.Long)
+                writer.Write((long)val);
+            else if (type == ColumnType.String)
+                writer.Write((string)val);
+            else if (type == ColumnType.Byte)
+                writer.Write((byte)val);
+            //else if (type == ColumnType.CategoryList) {
+            //    var data = (int[])val;
+            //    writer.Write(data.Length);
+            //    foreach (var item in data) {
+            //        writer.Write(item);
+            //    }
+            //}
+            //else if (type == ColumnType.WeightedCategoryList) {
+            //    var data = (Tuple<int, double>[])val;
+            //    writer.Write(data.Length);
+            //    foreach (var item in data) {
+            //        writer.Write(item.Item1);
+            //        writer.Write(item.Item2);
+            //    }
+            //}
+        }
+
+        public int WriteData(Stream stream)
+        {
+            int ret = 0;
+            var writer = new BinaryWriter(stream, Encoding.UTF8, true);
+            foreach (var row in _data) {
+                var index = 0;
+                var rowData = row.Data;
+                foreach (var column in _column) {
+                    var val = rowData[index++];
+                    _WriteValueTo(writer, column, val);
+                }
+                ++ret;
+            }
+            return ret;
+        }
+
+        public void Process(IRowProcessor rowProcessor)
+        {
+            foreach (var item in _data) {
+                if (!rowProcessor.Process(item))
+                    break;
+            }
+        }
+
+        public void Clear()
+        {
+            _data.Clear();
+        }
+    }
+}
diff --git a/TabularData/NumericTable.cs b/TabularData/NumericTable.cs
index ff96ace8..ce553174 100644
--- a/TabularData/NumericTable.cs
+++ b/TabularData/NumericTable.cs
@@ -5,7 +5,7 @@
 using System.Text;
 using System.Threading.Tasks;
 
-namespace BrightWire.Net4.TabularData
+namespace BrightWire.TabularData
 {
     public class NumericTable
     {
diff --git a/TrainingData/Artificial/ReberGrammar.cs b/TrainingData/Artificial/ReberGrammar.cs
index 716e6311..07caff9b 100644
--- a/TrainingData/Artificial/ReberGrammar.cs
+++ b/TrainingData/Artificial/ReberGrammar.cs
@@ -28,12 +28,16 @@ public static int GetIndex(char ch)
             var following = new Dictionary<string, HashSet<int>>();
             foreach (var str in strList) {
                 var sb = new StringBuilder();
+                string prev = null;
                 foreach (var ch in str) {
                     sb.Append(ch);
                     var key = sb.ToString();
-                    if (!following.TryGetValue(key, out temp))
-                        following.Add(key, temp = new HashSet<int>());
-                    temp.Add(_ch[ch]);
+                    if (prev != null) {
+                        if (!following.TryGetValue(prev, out temp))
+                            following.Add(prev, temp = new HashSet<int>());
+                        temp.Add(_ch[ch]);
+                    }
+                    prev = key;
                 }
             }
 
@@ -67,11 +71,11 @@ public ReberGrammar(bool stochastic = true)
             _rnd = stochastic ? new Random() : new Random(0);
         }
 
-        public IEnumerable<string> Get(int length)
+        public IEnumerable<string> Get(int? length)
         {
             while (true) {
                 var ret = Generate();
-                if (ret.Length == length)
+                if (!length.HasValue || (length.HasValue && ret.Length == length.Value))
                     yield return ret;
             }
         }
diff --git a/TreeBased/DecisionTreeClassifier.cs b/TreeBased/DecisionTreeClassifier.cs
new file mode 100644
index 00000000..f89bff79
--- /dev/null
+++ b/TreeBased/DecisionTreeClassifier.cs
@@ -0,0 +1,62 @@
+using BrightWire.Models;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace BrightWire.TreeBased
+{
+    public class DecisionTreeClassifier : IRowProcessor
+    {
+        readonly DecisionTree _tree;
+        readonly List<Tuple<string, string>> _resultList = new List<Tuple<string, string>>();
+        readonly int? _classColumnIndex;
+
+        public DecisionTreeClassifier(DecisionTree tree, int? classColumnIndex)
+        {
+            _tree = tree;
+            _classColumnIndex = classColumnIndex;
+        }
+
+        public string Classify(IRow row)
+        {
+            var p = _tree.Root;
+            while(p != null) {
+                if (p.ColumnIndex >= 0) {
+                    string findChild = null;
+                    if(p.Split.HasValue) {
+                        var val = row.GetField<double>(p.ColumnIndex);
+                        findChild = (val < p.Split.Value) ? "-" : "+";
+                    }else
+                        findChild = row.GetField<string>(p.ColumnIndex);
+                    if(findChild != null) {
+                        var child = p.Children.FirstOrDefault(c => c.MatchLabel == findChild);
+                        if(child != null) {
+                            p = child;
+                            continue;
+                        }
+                    }
+                }
+                return p.Classification;
+            }
+            return null;
+        }
+
+        public bool Process(IRow row)
+        {
+            string expectedValue = null;
+            if (_classColumnIndex.HasValue)
+                expectedValue = row.GetField<string>(_classColumnIndex.Value);
+            _resultList.Add(Tuple.Create(Classify(row), expectedValue));
+            return true;
+        }
+
+        public void Clear()
+        {
+            _resultList.Clear();
+        }
+
+        public IReadOnlyList<Tuple<string, string>> Results { get { return _resultList; } }
+    }
+}
diff --git a/TreeBased/Training/DecisionTreeTrainer.cs b/TreeBased/Training/DecisionTreeTrainer.cs
new file mode 100644
index 00000000..eae03725
--- /dev/null
+++ b/TreeBased/Training/DecisionTreeTrainer.cs
@@ -0,0 +1,258 @@
+using BrightWire.Models;
+using BrightWire.TabularData.Helper;
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace BrightWire.TreeBased.Training
+{
+    public static class DecisionTreeTrainer
+    {
+        class Attribute
+        {
+            readonly int _columnIndex;
+            readonly string _category;
+            readonly double? _split;
+
+            public Attribute(int columnIndex, string category)
+            {
+                _columnIndex = columnIndex;
+                _category = category;
+                _split = null;
+            }
+            public Attribute(int columnIndex, double split)
+            {
+                _columnIndex = columnIndex;
+                _category = null;
+                _split = split;
+            }
+            public override bool Equals(object obj)
+            {
+                var other = obj as Attribute;
+                if(other != null)
+                    return other._columnIndex == _columnIndex && other._category == _category && _split == other._split;
+                return false;
+            }
+            public override int GetHashCode()
+            {
+                if (_category != null)
+                    return _columnIndex.GetHashCode() ^ _category.GetHashCode();
+                return _columnIndex.GetHashCode() ^ _split.GetHashCode();
+            }
+            public override string ToString()
+            {
+                if (_category != null)
+                    return $"{_category} ({_columnIndex})";
+                return $"threshold: {_split} ({_columnIndex}";
+            }
+            public int ColumnIndex { get { return _columnIndex; } }
+            public string Category { get { return _category; } }
+            public double? Split { get { return _split; } }
+            public IReadOnlyDictionary<string, List<IRow>> Partition(IEnumerable<IRow> rows)
+            {
+                List<IRow> temp;
+                var ret = new Dictionary<string, List<IRow>>();
+                if (_category != null) {
+                    foreach (var item in rows) {
+                        var val = item.GetField<string>(_columnIndex);
+                        if (!ret.TryGetValue(val, out temp))
+                            ret.Add(val, temp = new List<IRow>());
+                        temp.Add(item);
+                    }
+                    return ret;
+                }else {
+                    var splitVal = _split.Value;
+                    foreach (var item in rows) {
+                        var val = item.GetField<double>(_columnIndex);
+                        var label = (val < splitVal) ? "-" : "+";
+                        if (!ret.TryGetValue(label, out temp))
+                            ret.Add(label, temp = new List<IRow>());
+                        temp.Add(item);
+                    }
+                }
+                return ret;
+            }
+        }
+        class TableInfo : IRowProcessor
+        {
+            readonly HashSet<int> _categorical = new HashSet<int>();
+            readonly HashSet<int> _continuous = new HashSet<int>();
+            readonly List<IRow> _data = new List<IRow>();
+            readonly int _classColumnIndex;
+
+            public TableInfo(IDataTable table, int classColumnIndex)
+            {
+                _classColumnIndex = classColumnIndex;
+                for (int i = 0, len = table.ColumnCount; i < len; i++) {
+                    if (i != _classColumnIndex) {
+                        var column = table.Columns[i];
+                        if (column.IsContinuous)
+                            _continuous.Add(i);
+                        else if (ColumnTypeClassifier.IsCategorical(column))
+                            _categorical.Add(i);
+                    }
+                }
+                table.Process(this);
+            }
+            bool IRowProcessor.Process(IRow row)
+            {
+                // load the entire dataset into memory
+                _data.Add(row);
+                return true;
+            }
+            public IEnumerable<int> CategoricalColumns { get { return _categorical; } }
+            public IEnumerable<int> ContinuousColumns { get { return _continuous; } }
+            public IReadOnlyList<IRow> Data { get { return _data; } }
+            public int ClassColumnIndex { get { return _classColumnIndex; } }
+        }
+        class Node
+        {
+            readonly TableInfo _tableInfo;
+            readonly IReadOnlyList<IRow> _data;
+            readonly Lazy<Dictionary<string, int>> _classCount;
+            Node _parent = null;
+            Attribute _attribute = null;
+            IReadOnlyList<Node> _children = null;
+            readonly string _matchLabel;
+
+            public Node(TableInfo tableInfo, IReadOnlyList<IRow> data, string matchLabel)
+            {
+                _data = data;
+                _tableInfo = tableInfo;
+                _matchLabel = matchLabel;
+                _classCount = new Lazy<Dictionary<string, int>>(() => data.GroupBy(d => d.Data[tableInfo.ClassColumnIndex].ToString()).ToDictionary(g => g.Key, g => g.Count()));
+            }
+
+            public DecisionTree.Node AsDecisionTreeNode()
+            {
+                var ret = new DecisionTree.Node {
+                    ColumnIndex = _attribute?.ColumnIndex ?? -1,
+                    MatchLabel = MatchLabel,
+                    Split = _attribute?.Split,
+                    Children = _children?.Select(c => c.AsDecisionTreeNode())?.ToArray(),
+                    Classification = PredictedClass
+                };
+                return ret;
+            }
+
+            public IReadOnlyCollection<Attribute> Attributes
+            {
+                get
+                {
+                    HashSet<double> temp;
+                    HashSet<string> temp2;
+                    var continuousValues = new Dictionary<int, HashSet<double>>();
+                    var categoricalValues = new Dictionary<int, HashSet<string>>();
+                    foreach(var item in _data) {
+                        foreach(var column in _tableInfo.CategoricalColumns) {
+                            if (!categoricalValues.TryGetValue(column, out temp2))
+                                categoricalValues.Add(column, temp2 = new HashSet<string>());
+                            temp2.Add(item.GetField<string>(column));
+                        }
+                        foreach (var column in _tableInfo.ContinuousColumns) {
+                            if (!continuousValues.TryGetValue(column, out temp))
+                                continuousValues.Add(column, temp = new HashSet<double>());
+                            temp.Add(item.GetField<double>(column));
+                        }
+                    }
+
+                    var ret = new HashSet<Attribute>();
+                    foreach(var column in categoricalValues) {
+                        if(column.Value.Count > 1) {
+                            foreach (var item in column.Value)
+                                ret.Add(new Attribute(column.Key, item));
+                        }
+                    }
+                    foreach (var column in continuousValues) {
+                        if (column.Value.Count > 1) {
+                            var orderedContinuous = column.Value.OrderBy(v => v).ToList();
+                            for (var i = 1; i < orderedContinuous.Count; i++) {
+                                var mid = (orderedContinuous[i - 1] + orderedContinuous[i]) / 2;
+                                ret.Add(new Attribute(column.Key, mid));
+                            }
+                        }
+                    }
+                    return ret;
+                }
+            }
+
+            public double Entropy
+            {
+                get
+                {
+                    double total = _classCount.Value.Sum(d => d.Value);
+                    double ret = 0;
+                    foreach(var item in _classCount.Value) {
+                        var probability = item.Value / total;
+                        ret -= probability * Math.Log(probability, 2);
+                    }
+                    return ret;
+                }
+            }
+
+            public IReadOnlyList<Node> SetAttribute(Attribute attribute, IReadOnlyList<Node> children)
+            {
+                _attribute = attribute;
+                _children = children;
+                foreach (var child in children)
+                    child._parent = this;
+                return children;
+            }
+
+            public int Depth
+            {
+                get
+                {
+                    var ret = 0;
+                    var p = _parent;
+                    while(p != null) {
+                        ++ret;
+                        p = p._parent;
+                    }
+                    return ret;
+                }
+            }
+            public bool IsLeaf { get { return _classCount.Value.Count <= 1; } }
+            public string PredictedClass { get { return _classCount.Value.OrderByDescending(kv => kv.Value).Select(kv => kv.Key).FirstOrDefault(); } }
+            public IReadOnlyList<IRow> Data { get { return _data; } }
+            public string MatchLabel { get { return _matchLabel; } }
+        }
+
+        public static DecisionTree Train(IDataTable table, int classColumnIndex)
+        {
+            var tableInfo = new TableInfo(table, classColumnIndex);
+            var root = new Node(tableInfo, tableInfo.Data, null);
+            var stack = new Stack<Node>();
+            stack.Push(root);
+
+            while(stack.Any()) {
+                var node = stack.Pop();
+                if (node.IsLeaf)
+                    continue;
+
+                var attributes = node.Attributes;
+                if (!attributes.Any())
+                    continue;
+
+                var nodeEntropy = node.Entropy;
+                double nodeTotal = node.Data.Count;
+                var scoreTable = new Dictionary<Tuple<Attribute, List<Node>>, double>();
+                foreach (var item in attributes) {
+                    var newChildren = item.Partition(node.Data).Select(d => new Node(tableInfo, d.Value, d.Key)).ToList();
+                    scoreTable.Add(Tuple.Create(item, newChildren), nodeEntropy - newChildren.Sum(c => c.Data.Count / nodeTotal * c.Entropy));
+                }
+                var bestSplit = scoreTable.OrderByDescending(kv => kv.Value).Select(kv => kv.Key).First();
+                foreach (var child in node.SetAttribute(bestSplit.Item1, bestSplit.Item2))
+                    stack.Push(child);
+            }
+
+            return new DecisionTree {
+                ClassColumnIndex = tableInfo.ClassColumnIndex,
+                Root = root.AsDecisionTreeNode()
+            };
+        }
+    }
+}
diff --git a/Unsupervised/Clustering/KMeans.cs b/Unsupervised/Clustering/KMeans.cs
index 34d836d4..b04ef1e6 100644
--- a/Unsupervised/Clustering/KMeans.cs
+++ b/Unsupervised/Clustering/KMeans.cs
@@ -4,7 +4,7 @@
 using System.Text;
 using System.Threading.Tasks;
 
-namespace BrightWire.Net4.Unsupervised.Clustering
+namespace BrightWire.Unsupervised.Clustering
 {
     public class KMeans
     {