Neural net learner now supports automatic mean and variance normaliza…

…tion of it's input by applying

(x_i - mean_i)/stdev_i  where i is the ith input (and is calculated from the input data set)

updated classifiers so they now support a generic weighting scheme: the data set examples are duplicated
by the weight ratio

BaseClassifier.pas

@@ -61,8 +61,11 @@   TCustomLearnerExample = class(TCustomExample)
   private
     fClassVal : integer;
   public
+    function Clone : TCustomLearnerExample; virtual;
+
     property ClassVal : integer read fClassVal write fClassVal;
   end;
+  TCustomLearnerExampleClass = class of TCustomLearnerExample;
 
 // #############################################################
 // #### List of examples - used in the training phase
@@ -85,7 +88,6 @@   TCustomLearnerExampleList = class(TObjectList)
     fRandom : TRandomGenerator;
 
     function InternalRandomDataSet(LearningSet : TCustomLearnerExampleList; StartIdx, EndIdx : integer; numElements : integer) : TCustomLearnerExampleList;
-    function Rand : TRandomGenerator;
     procedure SetRandomAlg(const Value: TRandomAlgorithm);
 
     function GetExample(index : integer) : TCustomLearnerExample; virtual;
@@ -96,7 +98,11 @@   TCustomLearnerExampleList = class(TObjectList)
     function CreateRandomDataSet(Percentage : integer) : TCustomLearnerExampleList;
     function CreateRandomizedBalancedDataSet(Percentage : integer) : TCustomLearnerExampleList;
 
+    // clone without examples
+    function CloneBase : TCustomLearnerExampleList; virtual;
+
     procedure Shuffle;  // randomizes all the examples
+    function Rand : TRandomGenerator;
 
     property Example[index : integer] : TCustomLearnerExample read GetExample write SetExample; default;
     procedure Add(Exmpl : TCustomLearnerExample);
@@ -107,6 +113,7 @@   TCustomLearnerExampleList = class(TObjectList)
     constructor Create;
     destructor Destroy; override;
   end;
+  TCustomLearnerExampleListClass = class of TCustomLearnerExampleList;
 
 // #############################################################
 // #### Base classifier class
@@ -139,7 +146,7 @@   TCommonLearnerProps = class(TObject)
 type
   TCustomWeightedLearner = class(TCommonLearnerProps)
   private
-    fDataSet : TCustomLearnerExampleList;
+    fDataSet : TCustomLearnerExampleList; // by reference
   protected
     property DataSet : TCustomLearnerExampleList read fDataSet;
     function DoLearn(const weights : Array of double) : TCustomClassifier; virtual; abstract;
@@ -163,11 +170,22 @@   TCustomWeightedLearnerClass = class of TCustomWeightedLearner;
 // #### Base classifier which cannot handle weigthing in the example list
 type
   TCustomLearner = class(TCustomWeightedLearner)
+  private
+    fOrigDataSet : TCustomLearnerExampleList;
+
+    // creates a new example list and adds already existing examples from the given list:
+    // weighting is achieved by duplicating
+    // items. e.g. if count=3 and weights are 0.66, 0.17, 0.17, then the
+    // result is count=5, example[0] x 3, example[1] x 1, example[2] x 1
+    // algorithm is:
+    // take max weigth and divide by 100: -> use that as minimum allowed weight and
+    // discard all examples lower than that weight.
+    // take the remaining lowest weight -> this example is put one time into the resulting
+    // array.
+    procedure BuildWeightedList(const weights : Array of double);
   protected
     function DoUnweightedLearn : TCustomClassifier; virtual; abstract;
     function DoLearn(const weights : Array of double) : TCustomClassifier; override;
-  public
-
   end;
   TCustomLearnerClass = class of TCustomLearner;
 
@@ -522,6 +540,12 @@ procedure TCustomLearnerExampleList.Shuffle;
      end;
 end;
 
+function TCustomLearnerExampleList.CloneBase: TCustomLearnerExampleList;
+begin
+     Result := TCustomLearnerExampleListClass(Self.ClassType).Create;
+     Result.fRandomAlg := fRandomAlg;
+end;
+
 { TCustomWeightedLearner }
 
 procedure TCustomWeightedLearner.IdxQuickSort(const Values : TDoubleDynArray; var Idx : TIntegerDynArray; L, R : integer);
@@ -729,7 +753,73 @@ function TCustomWeightedLearner.Learn: TCustomClassifier;
 function TCustomLearner.DoLearn(
   const weights: array of double): TCustomClassifier;
 begin
-     Result := DoUnweightedLearn;
+     // build a new list and use it like the original one
+     BuildWeightedList(weights);
+     try
+        Result := DoUnweightedLearn;
+     finally
+            // cleanup the intermediate dataset
+            if Assigned(fOrigDataSet) then
+            begin
+                 fDataSet.Free;
+                 fDataSet := fOrigDataSet;
+                 fOrigDataSet := nil;
+            end;
+     end;
+end;
+
+procedure TCustomLearner.BuildWeightedList(const weights: array of double);
+var maxWeight : double;
+    counter: Integer;
+    minAllowedWeight : double;
+    minWeight : double;
+    numExmpl : integer;
+    i : integer;
+    exmpl : TCustomLearnerExample;
+begin
+     // search for the maximum:
+     if length(weights) = 0 then
+        exit;
+
+     maxWeight := weights[0];
+     for counter := 1 to Length(weights) - 1 do
+         maxWeight := max(maxWeight, weights[counter]);
+
+     minAllowedWeight := maxWeight/100;
+     // find minimum
+     minWeight := maxWeight;
+     for counter := 0 to Length(weights) - 1 do
+     begin
+          if (weights[counter] < minWeight) and (weights[counter] >= minAllowedWeight) then
+             minWeight := weights[counter];
+     end;
+
+     // check if evenly distributed - if so do nothing
+     if minWeight = maxWeight then
+        exit;
+
+     fOrigDataSet := fDataSet;
+     fDataSet := fOrigDataSet.CloneBase;
+     fDataSet.fRandomAlg := fOrigDataSet.fRandomAlg;
+
+     // build resulting example list and multiple add the examples according to the weighting
+     for counter := 0 to Length(weights) - 1 do
+     begin
+          // remove all examples lower than the given weight
+          if weights[counter] < minWeight
+          then
+              continue
+          else
+          begin
+               numExmpl := ceil(weights[counter]/minWeight);
+               // clone and add examples
+               for i := 0 to numExmpl - 1 do
+               begin
+                    exmpl := TCustomLearnerExample(fOrigDataSet.GetExample(counter).Clone);
+                    fDataSet.Add(exmpl);
+               end;
+          end;
+     end;
 end;
 
 { TCustomExample }
@@ -756,4 +846,12 @@ function TCustomClassifier.Classify(Example: TCustomExample): integer;
      Result := Classify(Example, conf);
 end;
 
+{ TCustomLearnerExample }
+
+function TCustomLearnerExample.Clone: TCustomLearnerExample;
+begin
+     Result := TCustomLearnerExampleClass(self.ClassType).Create(fFeatureVec, False);
+     Result.fClassVal := fClassVal;
+end;
+
 end.

BaseMatrixExamples.pas

@@ -65,6 +65,9 @@   TMatrixLearnerExampleList = class(TCustomLearnerExampleList)
     fOwnsMatrix : boolean;
   public
     property Matrix : TDoubleMatrix read fMatrix;
+
+    function CloneBase : TCustomLearnerExampleList; override;
+
     constructor Create(AItemList : TDoubleMatrix; ClassVals : Array of Integer; ownsMatrix : boolean = True);
     destructor Destroy; override;
   end;
@@ -144,6 +147,13 @@ destructor TMatrixLearnerExampleList.Destroy;
      inherited;
 end;
 
+function TMatrixLearnerExampleList.CloneBase: TCustomLearnerExampleList;
+begin
+     Result := inherited CloneBase;
+     TMatrixLearnerExampleList(Result).fMatrix := Matrix;
+     TMatrixLearnerExampleList(Result).fOwnsMatrix := False;
+end;
+
 { TMatrixLearnerExample }
 
 constructor TMatrixLearnerExample.Create(AItemList: TDoubleMatrix; ItemIdx : integer;

DataIO/Haar2DDataSet.pas

@@ -31,6 +31,8 @@   THaar2DLearnerList = class(TCustomLearnerExampleList)
     procedure AddIntImg(img : TIntegralImage);
     procedure AddExample(list : THaar2DFeatureList; classVal : integer);
 
+    function CloneBase : TCustomLearnerExampleList; override;
+
     constructor Create;
     destructor Destroy; override;
   end;
@@ -348,4 +350,11 @@ destructor THaar2DLearnerList.Destroy;
      inherited;
 end;
 
+function THaar2DLearnerList.CloneBase: TCustomLearnerExampleList;
+begin
+     Result := inherited CloneBase;
+     THaar2DLearnerList(Result).fIntImages.OwnsObjects := False;
+     THaar2DLearnerList(Result).fIntImages.Assign(fIntImages);  // just copy references
+end;
+
 end.

DataSetTraining.pas

@@ -87,6 +87,10 @@ function TDataSetLearnerWeighted.LeaveOneOutLearner(
     classVal : integer;
     conf : double;
 begin
+     // ##############################################
+     // #### The final classifier will be from the complete set
+     // but the meantrain and classify errors are built from the
+     // leave one out algorithm
      MeanTrainError := 0;
      MeanClassifyError := 0;
      SetLength(TrainErrors, fDataSet.Count);
@@ -153,7 +157,7 @@ function TDataSetLearnerWeighted.LeaveOneOutLearner(
      // #### Finally create a classifier from the whole list
      learner := fLearner.Create;
      try
-        learner.Init(trainList);
+        learner.Init(fDataSet);
         Result := learner.Learn(Weights);
      finally
             learner.Free;

FeatureExtractors/Haar1DLearner.pas

@@ -53,7 +53,7 @@ implementation
 type
   T1DHaarExample = class(TCustomLearnerExample)
   public
-    function Clone(OnlyReference : boolean) : T1DHaarExample;
+    function CloneHaar(OnlyReference : boolean) : T1DHaarExample;
   end;
 
 { T1DHaarExampleCreator }
@@ -121,7 +121,7 @@ function T1DHaarExampleCreator.BuildLerningExmapleList(OnlyReference : boolean)
      Result := TCustomLearnerExampleList.Create;
 
      for i := 0 to fExampleList.Count - 1 do
-         Result.Add(T1DHaarExample(fExampleList[i]).Clone(OnlyReference));
+         Result.Add(T1DHaarExample(fExampleList[i]).CloneHaar(OnlyReference));
 end;
 
 constructor T1DHaarExampleCreator.Create;
@@ -142,7 +142,8 @@ destructor T1DHaarExampleCreator.Destroy;
 
 { T1DHaarExample }
 
-function T1DHaarExample.Clone(OnlyReference: boolean): T1DHaarExample;
+
+function T1DHaarExample.CloneHaar(OnlyReference: boolean): T1DHaarExample;
 var fVec : T1DHaarFeatureVec;
     features : Array of double;
     haarType : Array of T1DHaarType;

SimpleClassifier/FischerBatchLDA.pas

@@ -50,7 +50,7 @@   TFisherAugmentedBaseProps = record
 // Phd: Incremental, Robust, and Efficient Linear Discriminant Analysis Learning, page 33
 // Augmented subspace learning without any vector reduction
 type
-  TFisherBatchLDALearner = class(TCustomWeightedLearner)
+  TFisherBatchLDALearner = class(TCustomLearner)
   protected
     fProps : TFisherAugmentedBaseProps;
     fPCA : TMatrixPCA;
@@ -59,7 +59,7 @@   TFisherBatchLDALearner = class(TCustomWeightedLearner)
     function PCAAndProject(data : TDoubleMatrix; var U : TDoubleMatrix; var mu : TDoubleMatrix; var A : TDoubleMatrix; var eigVals : TDoubleMatrix) : boolean;
     procedure CreateBaseLDAClassifier(var pcaU, pcaMu, eigVals, ldaV : TDoubleMatrix; var classCenters : TDoubleMatrixDynArr;
                                       const classLabels : TIntegerDynArray; const classIdx : TIntIntArray; numClasses : Integer);
-    function DoLearn(const weights : Array of double) : TCustomClassifier; override;
+    function DoUnweightedLearn : TCustomClassifier; override;
   public
     procedure SetProperties(const props : TFisherAugmentedBaseProps);
 
@@ -275,8 +275,7 @@ destructor TFisherBatchLDALearner.Destroy;
      inherited;
 end;
 
-function TFisherBatchLDALearner.DoLearn(
-  const weights: array of double): TCustomClassifier;
+function TFisherBatchLDALearner.DoUnweightedLearn : TCustomClassifier;
 var pcaU, pcaMu, ldaV, eigVals: TDoubleMatrix;
     classCenters: TDoubleMatrixDynArr;
     numClasses : integer;
@@ -361,9 +360,10 @@ function TFisherBatchLDALearner.ExtractMatrixData(
   var ownsMatrix: boolean): TDoubleMatrix;
 var x, y : integer;
 begin
-     if DataSet is TMatrixLearnerExampleList then
+     if ( DataSet is TMatrixLearnerExampleList ) and ( TMatrixLearnerExampleList(DataSet).Matrix.Width = DataSet.Count) then
      begin
           Result := TMatrixLearnerExampleList(DataSet).Matrix;
+          ownsMatrix := False;
      end
      else
      begin
@@ -372,9 +372,9 @@ function TFisherBatchLDALearner.ExtractMatrixData(
           for y := 0 to DataSet[0].FeatureVec.FeatureVecLen - 1 do
               for x := 0 to DataSet.Count - 1 do
                   Result[x, y] := DataSet[x].FeatureVec[y];
-     end;
 
-     ownsMatrix := not (DataSet is TMatrixLearnerExampleList);
+          ownsMatrix := True;
+     end;
 end;
 
 function TFisherBatchLDALearner.PCAAndProject(data: TDoubleMatrix; var U, mu,