(WIP) PCA Regression

code/Demos/Program.fs

@@ -4,65 +4,114 @@
 open System
 open SampleLoader
 open Classifiers
+open RegressionPredictors
 open kNN
 open Perceptron
 open DecisionTree
+open PCARegression
 
 
-// main
+// records
 
-// load training samples
-let samplesTraining = SampleLoader.LoadFromFile "hw3train.txt"
-
-// load test samples
-let samplesTest = SampleLoader.LoadFromFile "hw3test.txt"
-
-// create classifiers
 type BootstrapRecord<'T> = {
     Name : string;
     Classifier : IClassifier;
+    RegressionPredictor  : IRegressionPredictor;
     TrainingError : 'T
     ValidationError : 'T
     TestError : 'T
 }
-let defaultRecord = { Name = null; Classifier = new NullClassifier() :> IClassifier; TrainingError = 1.0; ValidationError = 1.0; TestError = 1.0; } 
-let mutable classifiers = [|
-    { defaultRecord with Name = "1-NN"; Classifier = new kNNClassifier(1) :> IClassifier; }
-    { defaultRecord with Name = "5-NN"; Classifier = new kNNClassifier(5) :> IClassifier; }
-    { defaultRecord with Name = "Perceptron Weighted"; Classifier = new PerceptronClassifier(0, 6, ClassificationMethod.Weighted) :> IClassifier; }
-    { defaultRecord with Name = "Decision Tree (Max Depth 1)"; Classifier = new DecisionTreeClassifier(1) :> IClassifier; }
-    { defaultRecord with Name = "Decision Tree (Max Depth 3)"; Classifier = new DecisionTreeClassifier(3) :> IClassifier; }
-    { defaultRecord with Name = "Decision Tree (Max Depth 7)"; Classifier = new DecisionTreeClassifier(7) :> IClassifier; }
-|]
-
-// train
-for c in classifiers do
-    c.Classifier.Train samplesTraining
-
-// measure training error
-classifiers <-
-    classifiers
-    |> Seq.map (fun c -> 
-        { 
-            c with TrainingError = MeasureError c.Classifier samplesTraining; 
-        })
-    |> Seq.toArray
-
-// measure test error
-classifiers <-
-    classifiers
-    |> Seq.map (fun c ->
-        {
-            c with TestError = MeasureError c.Classifier samplesTest
-        })
-    |> Seq.toArray
-
-
-// print classifiers and error
-for c in classifiers do
-    System.Console.WriteLine("Classifier: {0}. Training Error: {1:P3}. Test Error: {2:P3}.", c.Name, c.TrainingError, c.TestError)
+let defaultRecord = 
+    { 
+        Name = null; 
+        Classifier = new NullClassifier() :> IClassifier; 
+        RegressionPredictor = new NullRegressionPredictor() :> IRegressionPredictor; 
+        TrainingError = 1.0; 
+        ValidationError = 1.0; 
+        TestError = 1.0; 
+    } 
+
+
+//
+// main
+//
+
+// load training samples
+let samplesTraining = SampleLoader.LoadFromFile "hw3train.txt"
+
+// load test samples
+let samplesTest = SampleLoader.LoadFromFile "hw3test.txt"
+
+
+let performSupervisedTests = false
+let performUnsupervisedTests = true
+
+
+//
+// supervised
+//
+
+if performSupervisedTests then
+    let mutable classifiers = [|
+        { defaultRecord with Name = "1-NN"; Classifier = new kNNClassifier(1) :> IClassifier; }
+        { defaultRecord with Name = "5-NN"; Classifier = new kNNClassifier(5) :> IClassifier; }
+        { defaultRecord with Name = "Perceptron Weighted"; Classifier = new PerceptronClassifier(0, 6, ClassificationMethod.Weighted) :> IClassifier; }
+        { defaultRecord with Name = "Decision Tree (Max Depth 1)"; Classifier = new DecisionTreeClassifier(1) :> IClassifier; }
+        { defaultRecord with Name = "Decision Tree (Max Depth 3)"; Classifier = new DecisionTreeClassifier(3) :> IClassifier; }
+        { defaultRecord with Name = "Decision Tree (Max Depth 7)"; Classifier = new DecisionTreeClassifier(7) :> IClassifier; }
+    |]
+
+    // train
+    for c in classifiers do
+        c.Classifier.Train samplesTraining
+
+    // measure training error
+    classifiers <-
+        classifiers
+        |> Seq.map (fun c -> 
+            { 
+                c with TrainingError = MeasureError c.Classifier samplesTraining; 
+            })
+        |> Seq.toArray
+
+    // measure test error
+    classifiers <-
+        classifiers
+        |> Seq.map (fun c ->
+            {
+                c with TestError = MeasureError c.Classifier samplesTest
+            })
+        |> Seq.toArray
+
+
+    // print classifiers and error
+    for c in classifiers do
+        System.Console.WriteLine("Classifier: {0}. Training Error: {1:P3}. Test Error: {2:P3}.", c.Name, c.TrainingError, c.TestError)
 
 
-// done. wait for input.
+
+//
+// unsupervised
+//
+
+if performUnsupervisedTests then
+
+    let mutable regressionPredictor = [|
+        { defaultRecord with Name = "PCA"; RegressionPredictor = new PCARegressionPredictor() :> IRegressionPredictor; }
+    |]
+
+
+    // train
+    for c in regressionPredictor do
+        c.RegressionPredictor.Train samplesTraining
+
+
+
+
+//
+// done. 
+// 
+
+// wait for input.
 let mutable key = Console.ReadLine()
 

code/F-AI/Classifiers.fs

@@ -115,5 +115,5 @@ type NullClassifier() =
     interface IClassifier with
         member self.Train samples =
             ()
-        member self.Classify point =
+        member self.Classify sample =
             failwith "Not implemented"

code/F-AI/F-AI.fsproj

@@ -51,7 +51,7 @@
     <Compile Include="kNN.fs" />
     <Compile Include="Perceptron.fs" />
     <Compile Include="DecisionTree.fs" />
-    <Compile Include="PCA.fs" />
+    <Compile Include="PCARegression.fs" />
   </ItemGroup>
   <ItemGroup>
     <Reference Include="MathNet.Numerics">

code/F-AI/MathNet.fs

@@ -35,6 +35,68 @@ open MathNet.Numerics.LinearAlgebra.Double
 
 type Vector = Vector<float>
 type Matrix = Matrix<float>
+type MatrixDense = DenseMatrix
+type MatrixSparse = SparseMatrix
+
+
+// types
+
+//type MatrixEnumerated(rows:Vector seq) = 
+//    class
+//        inherit MathNet.Numerics.LinearAlgebra.Generic.Matrix<float>(
+//            rows |> Seq.length, 
+//            (rows |> Seq.head).Count)
+//        
+//        override self.DoDivide(scalar:float, matrix:Matrix) = 
+//            raise (System.NotImplementedException("Not supported."))
+//
+//        override self.DoPointwiseDivide(matrixA, matrixB) = 
+//            raise (System.NotImplementedException("Not supported."))
+//
+//        override self.DoSubtract(matrixA:Matrix, matrixB:Matrix) =
+//            raise (System.NotImplementedException("Not supported."))
+//
+//        override self.DoMultiply(matrixA:Matrix, matrixB:Matrix) = 
+//            raise (System.NotImplementedException("Not supported."))
+//
+//        override self.DoMultiply(scalar:float, matrix:Matrix) =
+//            raise (System.NotImplementedException("Not supported."))
+//
+//        override self.DoMultiply(vectorA:Vector, vectorB:Vector) = 
+//            raise (System.NotImplementedException("Not supported."))
+//
+//
+//        override self.DoAdd(matrixA:Matrix, matrixB:Matrix) =
+//            raise (System.NotImplementedException("Not supported."))
+//
+//        override self.At(i:int, j:int) =
+//            let value = rows |> Seq.skip(i) |> Seq.head |> (fun r -> r.Item(j))
+//            value
+//
+//        override self.At(i:int, j:int, value:float) = 
+//            raise (System.NotImplementedException("Not supported."))
+//
+//        override self.CreateMatrix(i, j) = 
+//            raise (System.NotImplementedException("Not supported."))
+//
+//        override self.CreateVector(size) = 
+//            raise (System.NotImplementedException("Not supported."))
+//
+//        override self.ConjugateTranspose() = 
+//            raise (System.NotImplementedException("Not supported."))
+//
+//        override self.L1Norm() =
+//            raise (System.NotImplementedException("Not supported."))
+//
+//        override self.FrobeniusNorm() =
+//            raise (System.NotImplementedException("Not supported."))
+//
+//        override self.InfinityNorm() = 
+//            raise (System.NotImplementedException("Not supported."))
+//
+//        override self.Trace() = 
+//            raise (System.NotImplementedException("Not supported."))
+//    end
 
 
 // functions
@@ -48,4 +110,15 @@ let vector (values:float seq) =
 let vectorFromCount (count:int) = 
     new DenseVector(count) :> Vector
 
-
+let matrixFromRows (rows:Vector seq) =
+    let numRows = rows |> Seq.length
+    let numCols = rows |> Seq.head |> (fun e->e.Count)
+
+    let matrix = new MatrixDense(numRows, numCols)
+    for indexedRow in rows |> Seq.mapi (fun i r -> i,r) do
+        let index = fst indexedRow
+        let row = snd indexedRow
+        matrix.SetRow(index, row)
+
+    matrix
+

code/F-AI/PCARegression.fs

@@ -0,0 +1,67 @@
+
+//    This file is part of F-AI.
+//
+//    F-AI is free software: you can redistribute it and/or modify
+//    it under the terms of the GNU Lesser General Public License as published by
+//    the Free Software Foundation, either version 3 of the License, or
+//    (at your option) any later version.
+//
+//    F-AI is distributed in the hope that it will be useful,
+//    but WITHOUT ANY WARRANTY; without even the implied warranty of
+//    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+//    GNU Lesser General Public License for more details.
+//
+//    You should have received a copy of the GNU Lesser General Public License
+//    along with F-AI.  If not, see <http://www.gnu.org/licenses/>.
+
+
+
+//
+//    PCA learner.
+//
+
+module PCARegression
+
+
+// namespaces
+
+open System
+open MathNet
+open Primitives
+open Statistics
+open RegressionPredictors
+
+
+// types
+
+type PCARegressionPredictor() =
+
+    let mutable eigenVectors = Array.empty<Vector>
+    let mutable eigenValues = Array.empty<float>
+
+    interface IRegressionPredictor with
+        member self.Train samples = 
+
+            // convert samples to matrix
+            let samplesAsMatrix = matrixFromRows (samples |> Seq.map (fun s -> s.Features))
+
+            // calculate covariance matrix
+            let covarianceMatrix = Statistics.CovarianceMatrix(samplesAsMatrix.RowEnumerator() |> Seq.map (fun r -> snd r))
+
+            // calculate eigen vectors/values
+            let eigenDecomposition = new MathNet.Numerics.LinearAlgebra.Double.Factorization.DenseEvd(samplesAsMatrix)
+
+            assert (eigenDecomposition.EigenValues() |> Seq.forall (fun e -> e.Imaginary = 0.0))
+
+            let eigenPairs = Seq.zip (eigenDecomposition.EigenValues()) (eigenDecomposition.EigenVectors().RowEnumerator())
+            let orderedEigenPairs = eigenPairs |> Seq.sortBy (fun p -> -(fst p).Real)
+            eigenValues <- orderedEigenPairs |> Seq.map (fun p -> (fst p).Real) |> Seq.toArray
+            eigenVectors <- orderedEigenPairs |> Seq.map (fun p -> (snd (snd p))) |> Seq.toArray
+
+            () // todo
+
+        member self.Predict sample =
+            sample // todo
+
+
+

code/F-AI/RegressionPredictors.fs

@@ -33,4 +33,14 @@ open Primitives
 
 type IRegressionPredictor =
     abstract member Train : seq<Sample> -> unit
-    abstract member Predict : Sample -> Sample
+    abstract member Predict : Sample -> Sample
+
+
+// types
+
+type NullRegressionPredictor() = 
+    interface IRegressionPredictor with
+        member self.Train samples =
+            ()
+        member self.Predict sample =
+            failwith "Not implemented"