Permalink
Browse files

fist commit

  • Loading branch information...
0 parents commit 21e35b2176cb7e5daa5a5f5eccc986fea2e01b10 @alexksikes committed Oct 22, 2012
Showing with 37,591 additions and 0 deletions.
  1. +10 −0 README
  2. +218 −0 clustering/Cluster.java
  3. +174 −0 clustering/HierarchicalClustering.java
  4. +166 −0 clustering/LICENSE
  5. +172 −0 clustering/TreeDrawing.java
  6. +101 −0 eann/DataSet.java
  7. +129 −0 eann/EANN.java
  8. +140 −0 eann/Edge.java
  9. +44 −0 eann/Example.java
  10. +230 −0 eann/GA.java
  11. BIN eann/Graph1.xls
  12. BIN eann/Graph2.xls
  13. BIN eann/Graph3.xls
  14. BIN eann/Graph4.xls
  15. BIN eann/Graph5.xls
  16. BIN eann/Graph6.xls
  17. +55 −0 eann/Hypothesis.java
  18. +44 −0 eann/HypothesisComparator.java
  19. +166 −0 eann/LICENSE
  20. +500 −0 eann/NeuralNet.java
  21. +20 −0 eann/NeuralTest.java
  22. +165 −0 eann/Neuron.java
  23. BIN eann/README.pdf
  24. +308 −0 eann/evaluationdata.txt
  25. +189 −0 eann/fixedANNHypothesis.java
  26. 0 eann/testEANN.txt
  27. +308 −0 eann/trainingdata.txt
  28. +247 −0 eann/varyANNHypothesis.java
  29. +157 −0 ga/GA.java
  30. +209 −0 ga/Hypothesis.java
  31. +28 −0 ga/HypothesisComparator.java
  32. +166 −0 ga/LICENSE
  33. +84 −0 ga/Queens8.java
  34. +29 −0 ga/README
  35. +166 −0 gibbs/LICENSE
  36. +479 −0 gibbs/PatternFinder.java
  37. +14 −0 gibbs/README
  38. +8 −0 gibbs/output.format.txt
  39. BIN gibbs/pattern_finder.jar
  40. +10 −0 gibbs/sequences1L=10.txt
  41. +20 −0 gibbs/sequences2L=18.txt
  42. +10 −0 gibbs/sequencesL=10.txt
  43. +103 −0 k-means/DataSet.java
  44. +44 −0 k-means/Example.java
  45. +166 −0 k-means/LICENSE
  46. +179 −0 k-means/PointND.java
  47. +4 −0 k-means/README
  48. BIN k-means/bestModel.xls
  49. +1,501 −0 k-means/data.txt
  50. +398 −0 k-means/kMeans.java
  51. BIN k-means/kmeans.jar
  52. +402 −0 k-means/output.txt
  53. +1 −0 k-means/testClustering.txt
  54. +102 −0 knn/DataSet.java
  55. +45 −0 knn/Example.java
  56. +166 −0 knn/LICENSE
  57. +478 −0 knn/kNN.java
  58. +37 −0 knn/kNNComparator.java
  59. +105 −0 knn/kNNExample.java
  60. +20,001 −0 knn/project.test.txt
  61. +5,001 −0 knn/project.train.txt
  62. +102 −0 neural-net/DataSet.java
  63. +140 −0 neural-net/Edge.java
  64. +45 −0 neural-net/Example.java
  65. +166 −0 neural-net/LICENSE
  66. +484 −0 neural-net/NeuralNet.java
  67. +20 −0 neural-net/NeuralTest.java
  68. +165 −0 neural-net/Neuron.java
  69. +4 −0 neural-net/README
  70. +308 −0 neural-net/evaluationdata.txt
  71. BIN neural-net/myANN.dat
  72. BIN neural-net/neural_learn.jar
  73. +308 −0 neural-net/trainingdata.txt
  74. +104 −0 shotgun/Ensemble.java
  75. +166 −0 shotgun/LICENSE
  76. +225 −0 shotgun/Model.java
  77. +817 −0 shotgun/ModelSelection.java
  78. +889 −0 shotgun/Predictions.java
  79. +149 −0 shotgun/Targets.java
  80. BIN shotgun/ensembleSelection.xerox.ppt
  81. BIN shotgun/shotgun11.jar
10 README
@@ -0,0 +1,10 @@
+Some very old Java code implementing a couple of machine learning algorithms:
+
+* k-nearest neighbor
+* neural-nets
+* alglomerative clustering
+* genetic algortithm
+* gibbs sampling
+* k-means
+* evolutionary neural-nets
+* ensemble selection (shotgun)
218 clustering/Cluster.java
@@ -0,0 +1,218 @@
+/**
+ * @author Alex Ksikes
+ **/
+
+import java.io.*;
+import java.util.*;
+
+/**
+ * A cluster consists of a set of points and a representation.
+**/
+public class Cluster
+{
+
+ public static double[][] distances; // holds pairwise distances between points in diagonal form
+ private Vector points; // holds the points a cluster has
+ private int size; // number of points of a cluster
+ private int id; // unique id of a cluster
+ private String representation; // rep-n of a cluster in balanced parenthesis form
+
+ /**
+ * Builds a cluster with one point.
+ **/
+ public Cluster(int point, int id)
+ {
+ this.points=new Vector();
+ points.add(new Integer(point));
+ this.size=1;
+ this.id=id;
+ this.representation="("+String.valueOf(point)+")";
+ }
+
+ /**
+ * Merge two clusters.
+ **/
+ public void merge(Cluster c)
+ {
+ points.addAll(c.getPoints());
+ size=points.size();
+ updateRepresentation(c);
+ }
+
+ /**
+ * Returns the mean distance between this and a cluster c.
+ **/
+ public double meanDistance(Cluster c)
+ {
+ double distance=0;
+ for (int i=0; i<size; i++)
+ {
+ for (int j=0;j<c.getSize(); j++)
+ {
+ distance=distance + dist(getPoint(i),c.getPoint(j));
+ }
+ }
+ return distance/(size*c.getSize());
+ }
+
+ /**
+ * Returns the minimum distance between this and a cluster c.
+ **/
+ public double minimumDistance(Cluster c)
+ {
+ double currentDist;
+ double minDist=dist(getPoint(0),c.getPoint(0));
+ for (int i=0; i<size; i++)
+ {
+ for (int j=0;j<c.getSize(); j++)
+ {
+ currentDist=dist(getPoint(i),c.getPoint(j));
+ if (currentDist<minDist)
+ minDist=currentDist;
+ }
+ }
+ return minDist;
+ }
+
+ /**
+ * Compute the mean internal distance of this cluster.
+ **/
+ public double meanInternalDistance()
+ {
+ if (size==1)
+ return 0;
+ else
+ return meanDistance(this)*size/(size-1);
+ }
+
+ /**
+ * Get the point at the specified index i of this cluster.
+ **/
+ public int getPoint(int i)
+ {
+ return ((Integer) points.get(i)).intValue();
+ }
+
+ /**
+ * Update the balanced parenthesis form of this cluster
+ * after merging with a cluster c.
+ **/
+ public void updateRepresentation(Cluster c)
+ {
+ setRepresentation("(" + representation + "" + c.getRepresentation() + ")");
+ }
+
+ public void setRepresentation(String representation)
+ {
+ this.representation=representation;
+ }
+
+ public String getRepresentation()
+ {
+ return representation;
+ }
+
+ public Vector getPoints()
+ {
+ return points;
+ }
+
+ public int getSize()
+ {
+ return size;
+ }
+
+ public int getId()
+ {
+ return id;
+ }
+
+ public String toString()
+ {
+ return representation;
+ }
+
+ public boolean equals(Cluster c)
+ {
+ return (id==c.getId());
+ }
+
+ /**
+ * Make the distance array of pairwise distances in diagonal form.
+ **/
+ public static void setDistances(String filename)
+ {
+ try
+ {
+ BufferedReader bf=new BufferedReader(new FileReader(filename));
+ // read the first line that contains the number of points
+ String newLine = bf.readLine();
+ StringTokenizer st = new StringTokenizer(newLine);
+ distances=new double[Integer.parseInt(st.nextToken())][];
+ int numPoints;
+ // for each point
+ for (int i=0; i < distances.length; i++)
+ {
+ newLine=bf.readLine();
+ st=new StringTokenizer(newLine);
+ // ...read its distance from other points
+ numPoints=st.countTokens()-1;
+ double[] tempDist=new double[numPoints];
+ st.nextToken();
+ for (int j=0; j < numPoints; j++)
+ {
+ tempDist[j]=Double.parseDouble(st.nextToken());
+ }
+ distances[i]=tempDist;
+ }
+ }
+ catch (IOException e)
+ {
+ System.err.println(e.toString());
+ e.printStackTrace();
+ }
+ }
+
+ /**
+ * Returns the distance between point i and j.
+ **/
+ private static double dist(int i, int j)
+ {
+ if (i<j)
+ return distances[i][j-i-1];
+ else if(i==j)
+ return 0;
+ else
+ return distances[j][i-j-1];
+ }
+
+ /**
+ * Scale the data set of distances.
+ * New value=(value-min)/(max-min).
+ **/
+ public static void scaleDistances()
+ {
+ // looking for the max and min...
+ double current=distances[0][0];
+ double max=0, min=0;
+ for (int i=0;i<distances.length;i++)
+ {
+ for (int j=0;j<distances[i].length;j++)
+ {
+ current=distances[i][j];
+ if (current>max)
+ max=current;
+ if (current<min)
+ min=current;
+ }
+ }
+ // scaling the distances
+ for (int i=0;i<distances.length;i++)
+ {
+ for (int j=0;j<distances[i].length;j++)
+ {
+ distances[i][j]=(distances[i][j]-min)/(max-min);
+ }
+ }
+ }
+}
174 clustering/HierarchicalClustering.java
@@ -0,0 +1,174 @@
+/**
+ * @author Alex Ksikes
+ **/
+
+import java.io.*;
+import java.util.*;
+
+/**
+ * Implements agglomerative hirarchical clustering.
+**/
+public class HierarchicalClustering
+{
+
+ Vector clusters; // keeps track of clusters
+ int size; // number of clusters considered at a given momment
+
+ /**
+ * Default constructor
+ **/
+ public HierarchicalClustering()
+ {
+ }
+
+ /**
+ * Initialize the algorithm.
+ **/
+ public void init(String filename)
+ {
+ Cluster.setDistances(filename);
+
+// // for experiments: scale the distances
+// Cluster.scaleDistances();
+
+ this.clusters=new Vector();
+ // each point is in its own cluster
+ this.size=Cluster.distances.length;
+ for (int i=0; i<size; i++)
+ clusters.add(new Cluster(i,i));
+ }
+
+ /**
+ * Returns the cluster at the specified index i.
+ **/
+ public Cluster getCluster(int i)
+ {
+ return ((Cluster) clusters.get(i));
+ }
+
+ /**
+ * Returns the two closest clusters.
+ * Assumes at least two clusters are left.
+ **/
+ public Cluster[] findNearest()
+ {
+ Cluster[] closest=new Cluster[2];
+ Cluster c1=getCluster(0);
+ Cluster c2=getCluster(1);
+ double minMeanDist=c1.meanDistance(c2);
+
+// // for experiments
+// double minMeanDist=c1.minimumDistance(c2);
+
+ double current;
+ for (int i=0; i<size; i++)
+ {
+ for (int j=i+1; j<size; j++)
+ {
+ current=getCluster(i).meanDistance(getCluster(j));
+ if (current<minMeanDist)
+ {
+ c1=getCluster(i);
+ c2=getCluster(j);
+ minMeanDist=current;
+ }
+ }
+ }
+ closest[0]=c1;
+ closest[1]=c2;
+ return closest;
+ }
+
+ /**
+ * Merge two clusters.
+ **/
+ public void merge(Cluster c1, Cluster c2)
+ {
+ c1.merge(c2);
+ clusters.remove(c2);
+ size=clusters.size();
+ }
+
+ /**
+ * Run the agglomerative clustering algorithm.
+ * The input specifies the name of the file to write results in.
+ **/
+ public void run(String exp) throws IOException
+ {
+ File outputFile=new File(exp);
+ FileWriter out=new FileWriter(outputFile);
+ Cluster[] closest=new Cluster[2];
+ for (int i=size; i>1; i--)
+ {
+ closest=findNearest();
+ reportResults(out,i,closest[0],closest[1]);
+ merge(closest[0],closest[1]);
+ quickPrint(closest[0],closest[1]);
+ }
+ reportResults(out,1,closest[0],closest[0]);
+ out.close();
+ }
+
+ /**
+ * Print the merging of clusters together with the representation of the merged
+ * cluster.
+ **/
+ public void quickPrint(Cluster c1, Cluster c2)
+ {
+ System.out.println("Number of Clusters = "+size+" Merging... "+c2.getId()+" into "+c1.getId());
+ System.out.println("Cluster "+c1.getId()+" is now = "+c1.toString());
+ }
+
+ /**
+ * Write into a file the mean internal distance of the merged cluster,
+ * the mean distance and minimum distance between two clusters and
+ * the mean point happiness of all clusters.
+ **/
+ public void reportResults(FileWriter out, int numClusters, Cluster c1, Cluster c2) throws IOException
+ {
+ double meanInternal=c1.meanInternalDistance();
+ double meanDist=c1.meanDistance(c2);
+ double meanHappiness=computeMeanHappiness();
+ double minDist=c1.minimumDistance(c2);
+ out.write(numClusters + " " + meanInternal + " " + meanDist + " " + minDist
+ + " " + meanHappiness + "\r");
+ }
+
+ /**
+ * Return the mean distance between clusters.
+ **/
+ public double computeMeanHappiness()
+ {
+ double sum=0;
+ int denom=0; // may change that for closed form
+ for (int i=0;i<size;i++)
+ {
+ for (int j=i+1;j<size;j++)
+ {
+ sum=sum + getCluster(i).meanDistance(getCluster(j));
+ denom++;
+ }
+ }
+ return sum/denom;
+ }
+
+ /**
+ * Entry point to this program.
+ * Input is a data set containing the distances between points in diagonal form
+ * and the name of the file to store the results in.
+ **/
+ public static void main(String[] args)
+ {
+ if (args.length!=2)
+ {
+ System.out.println("Wrong usage. Type java HierarchicalClustering [dataFile] [saveFile]");
+ }
+ HierarchicalClustering algo=new HierarchicalClustering();
+ algo.init(args[0]);
+ try
+ {
+ algo.run(args[1]);
+ }
+ catch(IOException e){}
+ }
+}
166 clustering/LICENSE
@@ -0,0 +1,166 @@
+ GNU LESSER GENERAL PUBLIC LICENSE
+ Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+
+ This version of the GNU Lesser General Public License incorporates
+the terms and conditions of version 3 of the GNU General Public
+License, supplemented by the additional permissions listed below.
+
+ 0. Additional Definitions.
+
+ As used herein, "this License" refers to version 3 of the GNU Lesser
+General Public License, and the "GNU GPL" refers to version 3 of the GNU
+General Public License.
+
+ "The Library" refers to a covered work governed by this License,
+other than an Application or a Combined Work as defined below.
+
+ An "Application" is any work that makes use of an interface provided
+by the Library, but which is not otherwise based on the Library.
+Defining a subclass of a class defined by the Library is deemed a mode
+of using an interface provided by the Library.
+
+ A "Combined Work" is a work produced by combining or linking an
+Application with the Library. The particular version of the Library
+with which the Combined Work was made is also called the "Linked
+Version".
+
+ The "Minimal Corresponding Source" for a Combined Work means the
+Corresponding Source for the Combined Work, excluding any source code
+for portions of the Combined Work that, considered in isolation, are
+based on the Application, and not on the Linked Version.
+
+ The "Corresponding Application Code" for a Combined Work means the
+object code and/or source code for the Application, including any data
+and utility programs needed for reproducing the Combined Work from the
+Application, but excluding the System Libraries of the Combined Work.
+
+ 1. Exception to Section 3 of the GNU GPL.
+
+ You may convey a covered work under sections 3 and 4 of this License
+without being bound by section 3 of the GNU GPL.
+
+ 2. Conveying Modified Versions.
+
+ If you modify a copy of the Library, and, in your modifications, a
+facility refers to a function or data to be supplied by an Application
+that uses the facility (other than as an argument passed when the
+facility is invoked), then you may convey a copy of the modified
+version:
+
+ a) under this License, provided that you make a good faith effort to
+ ensure that, in the event an Application does not supply the
+ function or data, the facility still operates, and performs
+ whatever part of its purpose remains meaningful, or
+
+ b) under the GNU GPL, with none of the additional permissions of
+ this License applicable to that copy.
+
+ 3. Object Code Incorporating Material from Library Header Files.
+
+ The object code form of an Application may incorporate material from
+a header file that is part of the Library. You may convey such object
+code under terms of your choice, provided that, if the incorporated
+material is not limited to numerical parameters, data structure
+layouts and accessors, or small macros, inline functions and templates
+(ten or fewer lines in length), you do both of the following:
+
+ a) Give prominent notice with each copy of the object code that the
+ Library is used in it and that the Library and its use are
+ covered by this License.
+
+ b) Accompany the object code with a copy of the GNU GPL and this license
+ document.
+
+ 4. Combined Works.
+
+ You may convey a Combined Work under terms of your choice that,
+taken together, effectively do not restrict modification of the
+portions of the Library contained in the Combined Work and reverse
+engineering for debugging such modifications, if you also do each of
+the following:
+
+ a) Give prominent notice with each copy of the Combined Work that
+ the Library is used in it and that the Library and its use are
+ covered by this License.
+
+ b) Accompany the Combined Work with a copy of the GNU GPL and this license
+ document.
+
+ c) For a Combined Work that displays copyright notices during
+ execution, include the copyright notice for the Library among
+ these notices, as well as a reference directing the user to the
+ copies of the GNU GPL and this license document.
+
+ d) Do one of the following:
+
+ 0) Convey the Minimal Corresponding Source under the terms of this
+ License, and the Corresponding Application Code in a form
+ suitable for, and under terms that permit, the user to
+ recombine or relink the Application with a modified version of
+ the Linked Version to produce a modified Combined Work, in the
+ manner specified by section 6 of the GNU GPL for conveying
+ Corresponding Source.
+
+ 1) Use a suitable shared library mechanism for linking with the
+ Library. A suitable mechanism is one that (a) uses at run time
+ a copy of the Library already present on the user's computer
+ system, and (b) will operate properly with a modified version
+ of the Library that is interface-compatible with the Linked
+ Version.
+
+ e) Provide Installation Information, but only if you would otherwise
+ be required to provide such information under section 6 of the
+ GNU GPL, and only to the extent that such information is
+ necessary to install and execute a modified version of the
+ Combined Work produced by recombining or relinking the
+ Application with a modified version of the Linked Version. (If
+ you use option 4d0, the Installation Information must accompany
+ the Minimal Corresponding Source and Corresponding Application
+ Code. If you use option 4d1, you must provide the Installation
+ Information in the manner specified by section 6 of the GNU GPL
+ for conveying Corresponding Source.)
+
+ 5. Combined Libraries.
+
+ You may place library facilities that are a work based on the
+Library side by side in a single library together with other library
+facilities that are not Applications and are not covered by this
+License, and convey such a combined library under terms of your
+choice, if you do both of the following:
+
+ a) Accompany the combined library with a copy of the same work based
+ on the Library, uncombined with any other library facilities,
+ conveyed under the terms of this License.
+
+ b) Give prominent notice with the combined library that part of it
+ is a work based on the Library, and explaining where to find the
+ accompanying uncombined form of the same work.
+
+ 6. Revised Versions of the GNU Lesser General Public License.
+
+ The Free Software Foundation may publish revised and/or new versions
+of the GNU Lesser General Public License from time to time. Such new
+versions will be similar in spirit to the present version, but may
+differ in detail to address new problems or concerns.
+
+ Each version is given a distinguishing version number. If the
+Library as you received it specifies that a certain numbered version
+of the GNU Lesser General Public License "or any later version"
+applies to it, you have the option of following the terms and
+conditions either of that published version or of any later version
+published by the Free Software Foundation. If the Library as you
+received it does not specify a version number of the GNU Lesser
+General Public License, you may choose any version of the GNU Lesser
+General Public License ever published by the Free Software Foundation.
+
+ If the Library as you received it specifies that a proxy can decide
+whether future versions of the GNU Lesser General Public License shall
+apply, that proxy's public statement of acceptance of any version is
+permanent authorization for you to choose that version for the
+Library.
+
172 clustering/TreeDrawing.java
@@ -0,0 +1,172 @@
+/**
+ * @author Alex Ksikes
+ **/
+
+import java.awt.*;
+import java.io.*;
+
+/**
+ * This class makes a tree out of a balanced parenthesis string.
+ * Example of a balanced parenthesis string: ((1)((2)(4))).
+ * This class is used to draw a tree out of a cluster representation.
+ * Still need improvements (did this very quickly)
+**/
+public class TreeDrawing extends Canvas
+{
+
+ String balancedString; // the string we wish to draw as a tree.
+ int xInit,yInit,yInc;
+ Dimension d; // dimension of the canvas to draw a tree.
+
+ /**
+ * Default constructor
+ **/
+ public TreeDrawing(String balancedString)
+ {
+ if (!checkBalanced(balancedString))
+ {
+ System.out.println("String is not balanced!");
+ }
+ else
+ {
+ System.out.println("String balanced okay.");
+ this.balancedString=new String(balancedString);
+ this.d=new Dimension(1000,1000);
+ this.setSize(d);
+ xInit=800;
+ yInit=20;
+ yInc=10;
+ }
+ }
+
+ /**
+ * Check if string s is well balanced.
+ **/
+ public boolean checkBalanced(String s)
+ {
+ int count=0;
+ for (int i=0;i<s.length();i++)
+ {
+ if(s.charAt(i)=='(')
+ count++;
+ if(s.charAt(i)==')')
+ count--;
+ }
+ return count==0;
+ }
+
+ /**
+ * Get the left part of string s.
+ **/
+ public String getLeft(String s)
+ {
+ if (s.charAt(1)!='(')
+ {
+ return s;
+ }
+ else
+ {
+ int open=1;
+ int index=1;
+ char c;
+ while(open!=0)
+ {
+ index++;
+ c=s.charAt(index);
+ if (c=='(')
+ open++;
+ if (c==')')
+ open--;
+ }
+ //System.out.println("Left = "+s.substring(1,index+1));
+ return s.substring(1,index+1);
+ }
+ }
+
+ /**
+ * Get the right part of string s.
+ **/
+ public String getRight(String s)
+ {
+ if (s.charAt(s.length()-2)!=')')
+ {
+ return s;
+ }
+ else
+ {
+ String left=getLeft(s);
+ //System.out.println("Right = "+s.substring(left.length()+1,s.length()-1));
+ return s.substring(left.length()+1,s.length()-1);
+ }
+ }
+
+ /**
+ * Recursively draw a tree of string s at the coordinates (x,y).
+ **/
+ public void drawTree(Graphics g,String s,int x,int y)
+ {
+ g.drawOval(x,y,2,2);
+
+ if (s.charAt(1)!='(')
+ {
+ //System.out.println("I'm out with length= "+s.length());
+ return;
+ }
+ String left=getLeft(s);
+ String right=getRight(s);
+
+ // scales coordinates to fit the tree.
+ int xLeft=x-left.length()/25;
+ int xRight=x+right.length()/25;
+ //int yNew=y+yInc;
+ int yLeft=y+(int)Math.log(left.length()/3)+yInc;
+ int yRight=y+(int)Math.log(right.length()/3)+yInc;
+
+ System.out.println("Left = "+left);
+ System.out.println("Right = "+right);
+
+ g.drawLine(x,y,xLeft,yLeft);
+ g.drawLine(x,y,xRight,yRight);
+
+ drawTree(g,left,xLeft,yLeft);
+ drawTree(g,right,xRight,yRight);
+ }
+
+ public void paint(Graphics g)
+ {
+ drawTree(g,balancedString,xInit,yInit);
+ }
+
+
+
+ /**
+ * Entry point to this program.
+ * Input is a well formed balanced parenthesis string.
+ **/
+ public static void main(String[] args)
+ {
+ if (args.length!=1)
+ {
+ System.out.println("Wrong usage. Type java TreeDrawing [file]");
+ }
+ else
+ {
+ String data=new String();
+ try
+ {
+ BufferedReader bf=new BufferedReader(new FileReader(args[0]));
+ data=bf.readLine();
+ }
+ catch(IOException e){}
+ String test="(((((5)((1)(2)))((3)(4)))1)(((0)((5)(8)))(((9)(0))(3))))";
+ //String test2=((((165)(442))((396)(397)))(244));
+ System.out.println(data);
+
+ TreeDrawing tree=new TreeDrawing(data);
+ Frame f=new Frame();
+ f.setSize(1000,1000);
+ f.add(tree);
+ f.show();
+ }
+ }
+}
101 eann/DataSet.java
@@ -0,0 +1,101 @@
+import java.io.BufferedReader;
+import java.io.Serializable;
+import java.io.FileReader;
+import java.util.StringTokenizer;
+import java.io.IOException;
+
+/** This class takes in a txt file and creates a dataset that is easily accessible
+ * from the rest of the program
+ **/
+public class DataSet implements Serializable
+{
+ /********** Variables *************/
+ private int size;
+ private int attributeNum;
+ Example examples[];
+
+ /********** Methods ***************/
+
+ /** Constructor
+ * @param filename is the name of the file that contains the textual information
+ **/
+ public DataSet(String filename)
+ {
+ try
+ {
+ BufferedReader bf = new BufferedReader(new FileReader(filename));
+
+ // read the first line that contains the size of the dataset and number of attributes in each example
+ String newLine = bf.readLine();
+
+ StringTokenizer st = new StringTokenizer(newLine);
+ size = Integer.parseInt(st.nextToken());
+ attributeNum = Integer.parseInt(st.nextToken());
+ examples = new Example[size];
+
+ int classLabel;
+ double attributes[] = new double[attributeNum];
+
+ // read in the each of the example
+ for (int k=0;k<size;k++)
+ {
+ newLine = bf.readLine();
+ st = new StringTokenizer(newLine);
+ classLabel = Integer.parseInt(st.nextToken());
+ for (int i=0;i<attributeNum;i++)
+ {
+ attributes[i] = Double.parseDouble(st.nextToken());
+ }
+ examples[k] = new Example(classLabel,attributes);
+ }
+ }
+ catch (IOException e)
+ {
+ System.err.println(e.toString());
+ e.printStackTrace();
+ }
+ }
+
+ /** Accessor for the number of examples
+ * @return number of examples
+ **/
+ public int size()
+ {
+ return size;
+ }
+
+ /** Gets a particular example
+ * @param index of the example to retrieve
+ **/
+ public Example getExample(int index)
+ {
+ return examples[index];
+ }
+
+ /** Accessor for the classLabel
+ * @return classLabel for example index
+ **/
+ public int getClassLabel(int index)
+ {
+ return examples[index].getClassLabel();
+ }
+
+ /** Accessor for the number of attribute
+ * @return number of attribues in each example
+ **/
+ public int getAttributeNum()
+ {
+ return attributeNum;
+ }
+
+ /** Accessor for attributes
+ * @param i contains the numbering for the example in the dataset
+ * @param j contains the numbering for the attribute
+ * @return an attribute of example i attribute j
+ **/
+ public double getAttribute(int i, int j)
+ {
+ return examples[i].getAttribute(j);
+ }
+
+}
129 eann/EANN.java
@@ -0,0 +1,129 @@
+/**
+ * This class is the entry point to the program
+ * Author: Alex Ksikes
+ */
+
+import java.io.*;
+
+public class EANN {
+
+ // This is the main function
+ public static void main(String[] args) throws IOException
+ {
+ if (args.length!=3)
+ {
+ System.out.println("Wrong usage. Type java EANN [population size] [selection size]" +
+ "[mutation rate] [mode] [initial number of hidden neurons] [training set]");
+ }
+ // p is the number of hypotheses in population
+ int p=Integer.valueOf(args[0]).intValue();
+ // r is the fraction of the population to be replaced by Crossover at each step
+ double r=Double.valueOf(args[1]).doubleValue();
+ // m is the rate of mutation
+ double m=Double.valueOf(args[2]).doubleValue();
+ // mode is 0 if the number of hidden neurons is fixed and 1 otherwise
+ int mode=Integer.valueOf(args[3]).intValue();
+ // initNumHiddenNeurons is the maximum number of hidden neurons allowed for an initial population
+ int initNumHiddenNeurons=Integer.valueOf(args[4]).intValue();
+ // the training set
+ DataSet trainingSet=new DataSet(args[5]);
+ //
+ DataSet evaluationSet=new DataSet(args[6]);
+ // Write into files...
+ File outputFile=new File("testEANN.txt");
+ FileWriter out=new FileWriter(outputFile);
+
+ //////////////////////// to be erased //////////////////////////////
+// p=50;
+// r=0.6;
+// m=0.15;
+// mode=0;
+// initNumHiddenNeurons=5;
+ //////////////////////// to be erased //////////////////////////////
+
+ // Report the population size, crossover rate and mutation rate
+ System.out.println();
+ System.out.println("Evolutionary Artificial Neural Networks");
+ System.out.println("Population Size: " + p);
+ System.out.println("Crossover Rate: " + r);
+ System.out.println("Mutation Rate: " + m);
+ if (mode==0)
+ {
+ System.out.println("The Neural Nets have a fixed number of hidden neurons.");
+ System.out.println("Number of Hidden Neurons: " + initNumHiddenNeurons);
+ }
+ else
+ {
+ System.out.println("The Neural Nets have a varying number of hidden neurons.");
+ System.out.println("Initial Maximun Number of Hidden Neurons: " + initNumHiddenNeurons);
+ }
+ System.out.println();
+ out.write("Parameters are p=" + p + ", " + "r=" + r + " and m=" + m + "\r");
+
+ // The genetic algorithm
+ int iterationNum=0;
+ Hypothesis bestIndividual;
+ double bestFitness;
+ double bestAcc=0;
+ double evalAcc=0;
+ double aveFitness;
+ double aveNumNeurons;
+ double aveTrainingAcc;
+ double numNeurons=0;
+ // Randomly generate initial population
+ GA algorithm = new GA(p, r, m, mode, initNumHiddenNeurons, trainingSet, evaluationSet);
+ // For each individual compute fitness
+ algorithm.computeFitness();
+ // Get the best individual
+ bestIndividual=(algorithm.getPopulation())[0];
+ bestFitness=bestIndividual.getFitness();
+ aveFitness=algorithm.computeAveFitness();
+ //aveTrainingAcc=algorithm.computeAveTrainingAcc();
+ // Iterate till we get the very best individual
+ while (iterationNum!=100) // change this to number of iteration
+ {
+ // Select the very best members of the population to survive
+ algorithm.select();
+ // Make the best members reproduce themselves
+ algorithm.crossover();
+ // Add some mutations to new population
+ algorithm.mutate();
+ // The successor population becomes the current population
+ algorithm.setNextGeneration();
+ // For each individual compute fitness
+ algorithm.computeFitness();
+ // Get the best individual
+ bestIndividual=(algorithm.getPopulation())[0];
+ bestFitness=bestIndividual.getFitness();
+ bestAcc=bestIndividual.getTrainingAcc();
+ aveFitness=algorithm.computeAveFitness();
+ //aveTrainingAcc=algorithm.computeAveTrainingAcc();
+ evalAcc=bestIndividual.getEvalAcc();
+ numNeurons=bestIndividual.getNumHiddenNeurons();
+ // Report best results to the user at every five iterations
+ if (iterationNum%5==0)
+ {
+ aveNumNeurons=algorithm.computeAveNumHiddenNeurons();
+ System.out.println("Iteration: " + iterationNum);
+ System.out.println("Average Number of Hidden Neurons: " + aveNumNeurons);
+ System.out.println("Best Fitness: " + bestFitness);
+ System.out.println("Average Fitness: " + aveFitness);
+ System.out.println("Best solution: " + bestIndividual.toString());
+ System.out.println("Training Acc: " + bestAcc);
+ //System.out.println("Average Training Acc: " + aveTrainingAcc);
+ System.out.println("Evaluation Acc: "+ evalAcc);
+ System.out.println();
+ out.write("\n" + iterationNum + "," + bestAcc + "," + evalAcc + "," + numNeurons + "\r");
+ }
+ iterationNum++;
+ }
+ System.out.println("---------------------------------------------------------");
+ aveNumNeurons=algorithm.computeAveNumHiddenNeurons();
+ System.out.println("Training Acc: " + bestAcc);
+ System.out.println("Evaluation Acc: "+ evalAcc);
+ System.out.println("Solution is " + bestIndividual.toString());
+ out.write("\n" + iterationNum + "," + bestAcc + "," + evalAcc + "," + numNeurons + "\r");
+ out.close();
+ }
+
+}
140 eann/Edge.java
@@ -0,0 +1,140 @@
+/**
+ * Class Edge for the neural network
+ * Each edge has an assoicated weight that changes along the way
+ * Author: Alex Ksikes
+**/
+
+import java.lang.Math;
+import java.io.Serializable;
+
+public class Edge implements Serializable
+{
+ /*********** Variables **************/
+
+ private double weight; // the weight associated with this edge
+ private Neuron source; // the neuron it is connected to in layer i
+ private Neuron dest; // the neuron it is connected to in layer i+1
+ private int id; // identifier for the edge
+ private int mode; // the mode to use for update rule
+ private double deltaWeight; // the change in weight of this edge
+ private double deltaWeightOld; // keep track of the old value of the change in weight
+
+ // backpropation parameters
+ private double momentum = 0.2; // momentum term in order not to get stuck in local minima
+ private double learningRate = 0.3; // learning rate changes the speed of how convergence takes place
+
+ /*********** Methods *****************/
+
+ /**
+ * Edge Constructor
+ * @param parent is the neuron it is connected to in layer i
+ * @param child is the neuron it is connected to in layer i+1
+ * @param lr is the learning rate it is initialized to
+ * @param idd is the id of this Edge
+ * @param momentum is the momentum rate this edge is initialized to
+ * @param mode indicates the mode (Batch or Stochastic) to use for training
+ **/
+ public Edge(Neuron parent, Neuron child, double lr, int idd, double momentum, int mode)
+ {
+ this.source=parent;
+ this.dest=child;
+ this.learningRate=lr;
+ this.id=idd;
+ this.momentum=momentum;
+ this.mode=mode;
+ this.deltaWeight=0;
+ this.deltaWeightOld=0;
+ }
+
+ /**
+ * Resets the weight for the edge with some random value
+ * Chooses the weights uniformly between w and -w
+ * Where w=1/sqrt(n) and n=# of inputs going into the destination of this edge
+ **/
+ public void reset()
+ {
+ int n=getDest().getParentNum();
+ double w=1.0/Math.sqrt(n);
+ if (Math.random()>1/2.0)
+ weight=Math.random()*w;
+ else
+ weight=-1*Math.random()*w;
+ }
+
+ /**
+ * Sets the momentum of the neuron
+ **/
+ public void setMomentum(double mn)
+ {
+ this.momentum=mn;
+ }
+
+ /**
+ * Sets the learning rate of the neuron
+ **/
+ public void setLearningRate(double lr)
+ {
+ this.learningRate=lr;
+ }
+
+ /**
+ * Set the weight
+ **/
+ public void setWeight(double x)
+ {
+ this.weight=x;
+ }
+
+ /**
+ * Update the change in weight of this edge according to the error term
+ * @param errTerm is the error term that is propagated down
+ **/
+ public void updateDeltaWeight(double errterm)
+ {
+ Neuron s=getSource();
+ deltaWeight=deltaWeight + learningRate*(1-momentum)*errterm*s.getValue()
+ + momentum*deltaWeightOld;
+ }
+
+ /**
+ * The change in weight is saved and reinitialized
+ **/
+ public void initDeltaWeight()
+ {
+ deltaWeightOld=deltaWeight; // for momentum...
+ deltaWeight=0;
+ }
+
+ /**
+ * Update the weight of this edge
+ **/
+ public void updateWeight()
+ {
+ weight=weight + deltaWeight;
+ }
+
+ /**
+ * @return the weight for this edge
+ **/
+ public double getWeight()
+ {
+ return weight;
+ }
+
+ /**
+ * @return the source neuron for this edge
+ **/
+ public Neuron getSource()
+ {
+ return source;
+ }
+
+ /**
+ * @return the destination neuron for this edge
+ **/
+ public Neuron getDest()
+ {
+ return dest;
+ }
+
+}
44 eann/Example.java
@@ -0,0 +1,44 @@
+import java.io.Serializable;
+
+/** This Example class is a data structure that contains the class label
+ * and its attribute.
+ **/
+public class Example implements Serializable
+{
+ /********** Variables *************/
+
+ private int classLabel;
+ private double[] attributes;
+
+ /********** Methods ***************/
+
+ /** Constructor
+ * @param classLabel contains the class of this example
+ * @param attributes contains the array of attributes to be stored
+ **/
+ public Example(int classLabel, double[] attributes)
+ {
+ this.classLabel = classLabel;
+ this.attributes = new double[attributes.length];
+ for (int i=0;i<attributes.length;i++)
+ {
+ this.attributes[i]= attributes[i];
+ }
+ }
+
+ /** Accessor for the classLabel
+ * @return classLabel
+ **/
+ public int getClassLabel()
+ {
+ return classLabel;
+ }
+
+ /** Accessor for a single attribue in the attribute array
+ * @return attribute value in position index
+ **/
+ public double getAttribute(int index)
+ {
+ return attributes[index];
+ }
+}
230 eann/GA.java
@@ -0,0 +1,230 @@
+/*
+To do: implement sort procedure
+This class implements the operations of the Genetic Algorithm
+ * Author: Alex Ksikes
+*/
+import java.util.*;
+
+public class GA
+{
+ // p is the number of hypotheses in population
+ int p;
+ // r is the fraction of the population to be replaced by Crossover at each step
+ double r;
+ // m is the rate of mutation
+ double m;
+ // mode is 0 if the size of the number of hidden neurons is kept fixed or 1 o.w
+ int mode;
+ // Holds the current population
+ private Hypothesis[] population;
+ // Holds the successor population
+ private Hypothesis[] successorPopulation;
+
+ // Create initial population here
+ public GA(int p, double r, double m, int mode, int initNumHiddenNeurons, DataSet trainingSet, DataSet evaluationSet)
+ {
+ this.p=p;
+ this.r=r;
+ this.m=m;
+ this.mode=mode;
+
+ Hypothesis.trainingSet=trainingSet;
+ Hypothesis.evaluationSet=evaluationSet;
+ int numInputNeurons=trainingSet.getAttributeNum();
+ int numOutputNeurons=1;
+
+ // Set the number of input and output neurons
+ Hypothesis.numInputNeurons=numInputNeurons;
+ Hypothesis.numOutputNeurons=1;
+
+ // If mode is 0 we initialize the neural network used to evaluate each hypothesis
+ if(mode==0)
+ {
+ int[] numInLayer=new int[3];
+ numInLayer[0]=numInputNeurons;
+ numInLayer[1]=initNumHiddenNeurons;
+ numInLayer[2]=numOutputNeurons;
+ fixedANNHypothesis.numHiddenNeurons=initNumHiddenNeurons;
+ fixedANNHypothesis.neuralNet=new NeuralNet(3,numInLayer,0,0,1);
+ }
+
+ createInitialPopulation(initNumHiddenNeurons);
+ }
+
+ // Create initial population by randomly generating p hypotheses
+ // If mode is 0 creates p hypotheses of fixed size (maxNumberHiddenNeurons)
+ // If mode is 1 creates p hypotheses of size randomly varying between 2 and maxNumHiddenNeurons
+ private void createInitialPopulation(int initNumHiddenNeurons)
+ {
+ population=new Hypothesis[p];
+ successorPopulation=new Hypothesis[p];
+ Hypothesis h;
+ if (mode==0)
+ {
+ for (int i=0;i<p;i++)
+ {
+ h=new fixedANNHypothesis();
+ h.setToRandom();
+ population[i]=h;
+ successorPopulation[i]=new fixedANNHypothesis(); // initialize the successor array
+ }
+ }
+ if(mode==1)
+ {
+ Random value=new Random();
+ int randomSize;
+ for (int i=0;i<p;i++)
+ {
+ randomSize=value.nextInt(initNumHiddenNeurons-1)+2;
+ h=new varyANNHypothesis(randomSize);
+ h.setToRandom();
+ population[i]=h;
+ successorPopulation[i]=new varyANNHypothesis(); // initialize the successor array
+ }
+ }
+ }
+
+ // Compute fitness for the population
+ // and rank the population based on their fitness function
+ public void computeFitness()
+ {
+ for (int i=0;i<p;i++)
+ {
+ population[i].computeFitness();
+ }
+ HypothesisComparator c=new HypothesisComparator();
+ Arrays.sort(population,c);
+ }
+
+ // Returns the average fitness of the population
+ public double computeAveFitness()
+ {
+ double sum=0;
+ for (int i=0;i<p;i++)
+ {
+ sum=sum + population[i].getFitness();
+ }
+ return sum/(p*1.0);
+ }
+
+ // Returns the average training accuracy
+ public double computeAveTrainingAcc()
+ {
+ double sum=0;
+ for (int i=0;i<p;i++)
+ {
+ sum=sum + population[i].getTrainingAcc();
+ }
+ return sum/(p*1.0);
+ }
+
+ // Returns the average number of hidden neurons
+ public double computeAveNumHiddenNeurons()
+ {
+ double sum=0;
+ for (int i=0;i<p;i++)
+ {
+ sum=sum + population[i].getNumHiddenNeurons();
+ }
+ return sum/(p*1.0);
+ }
+
+ // Probabilisticaly select the (1-r)*p best hypotheses
+ // Assumes the population has been ranked
+ public void select()
+ {
+ Hypothesis h;
+ double[][][] representation;
+ int rank;
+ int numSelected=0;
+ double Prh;
+
+ // Operate rank selection
+ while (numSelected<(1-r)*p)
+ {
+ rank=(int) (Math.random()*p);
+ h=population[rank];
+ Prh= (p-rank) / ( p*(p+1.0)/2.0 );
+ if (Math.random()<Prh)
+ {
+ representation=h.getRepresentation();
+ successorPopulation[numSelected].setRepresentation(representation);
+ numSelected++;
+ }
+ }
+ }
+
+ // Probabilisticaly choose r*p/2 pairs of the best hypotheses
+ // and apply crossover on each pair
+ // Assumes the population has been ranked
+ public void crossover()
+ {
+ int j=(int) ((1-r)*p);
+ Hypothesis parent1;
+ Hypothesis parent2;
+ if (mode==0)
+ {
+ parent1=new fixedANNHypothesis();
+ parent2=new fixedANNHypothesis();
+ }
+ else
+ {
+ parent1=new varyANNHypothesis();
+ parent2=new varyANNHypothesis();
+ }
+ int rankP1, rankP2; // rank of the parent 1 and 2
+ double PrP1, PrP2; // proba of selection of parent 1 and 2
+ int numReproduced=0;
+
+ // Operate rank selection
+ while (numReproduced < r*p)
+ {
+ // choosing the first parent
+ rankP1=(int) (Math.random()*p);
+ PrP1= (p-rankP1) / ( p*(p+1.0)/2.0 );
+ // choosing the second parent
+ rankP2=(int) (Math.random()*p);
+ PrP2= (p-rankP2) / ( p*(p+1.0)/2.0 );
+ if (Math.random()<PrP1 && Math.random()<PrP2)
+ {
+ // apply crossover between parent 1 and parent 2
+ parent1.setRepresentation(population[rankP1].getRepresentation());
+ parent2.setRepresentation(population[rankP2].getRepresentation());
+ parent1.crossover(parent2);
+ successorPopulation[j].setRepresentation(parent1.getRepresentation());
+ numReproduced++;
+ // apply crossover between parent 2 and parent 1
+ parent1.setRepresentation(population[rankP1].getRepresentation());
+ parent2.setRepresentation(population[rankP2].getRepresentation());
+ parent2.crossover(parent1);
+ successorPopulation[j+1].setRepresentation(parent2.getRepresentation());
+ numReproduced++;
+ j=j+2;
+ }
+ }
+ }
+
+ // Mutate m*p random members of the successor population
+ public void mutate()
+ {
+ int index;
+ for (int i=0;i<m*p;i++)
+ {
+ index=(int)(Math.random()*p);
+ successorPopulation[index].mutate();
+ }
+ }
+
+ // Returns the array that holds the population
+ public Hypothesis[] getPopulation()
+ {
+ return population;
+ }
+
+ // Make the successor population the current population
+ public void setNextGeneration()
+ {
+ for (int i=0;i<p;i++)
+ population[i].setRepresentation(successorPopulation[i].getRepresentation());
+ }
+}
BIN eann/Graph1.xls
Binary file not shown.
BIN eann/Graph2.xls
Binary file not shown.
BIN eann/Graph3.xls
Binary file not shown.
BIN eann/Graph4.xls
Binary file not shown.
BIN eann/Graph5.xls
Binary file not shown.
BIN eann/Graph6.xls
Binary file not shown.
55 eann/Hypothesis.java
@@ -0,0 +1,55 @@
+/**
+ * This abstract class describes the behavior of an ANNHypothesis for use by the genetic algorithm
+ * Author: Alex Ksikes
+*/
+public abstract class Hypothesis
+{
+ /**************** Variables ********************/
+
+ static int numInputNeurons; // Number of input neurons
+ static int numOutputNeurons; // Number of output neurons
+ static DataSet trainingSet;
+ static DataSet evaluationSet;
+
+ /**************** Methods **********************/
+
+ // Give to this hypothesis a random representation
+ abstract public void setToRandom();
+
+ // Compute fitness for this Hypothesis
+ abstract public void computeFitness();
+
+ // Crossover with another Hypothesis
+ abstract public void crossover(Hypothesis otherParent);
+
+ // Mutate this Hypothesis
+ abstract public void mutate();
+
+ // Get the representation of this hypothesis
+ abstract public double[][][] getRepresentation();
+
+ // Set the representation of this hypothesis
+ abstract public void setRepresentation(double[][][] newRepresentation);
+
+ // Returns the fitness of this hypothesis
+ abstract public double getFitness();
+
+ // Pretty print this hypothesis
+ abstract public String toString();
+
+ // Set the weights of a neuron at the specified index location on the hidden layer
+ abstract public void setWeights(int index, double[][] newWeights);
+
+ // Returns the set of weights at the specified index location of the hidden layer
+ abstract public double[][] getWeights(int index);
+
+ // Returns the training accuracy of the corresponding neural net
+ abstract public double getTrainingAcc();
+
+ // Returns the number of hidden neurons
+ abstract public int getNumHiddenNeurons();
+
+ // Returns the validation accuracy
+ abstract public double getEvalAcc();
+
+}
44 eann/HypothesisComparator.java
@@ -0,0 +1,44 @@
+
+import java.util.*;
+
+public class HypothesisComparator implements Comparator
+{
+
+ public HypothesisComparator() {}
+
+ // Compare two hypotheses based on their fitness value
+ public int compare(Object o1,Object o2)
+ {
+ Hypothesis h1=(Hypothesis) o1;
+ Hypothesis h2=(Hypothesis) o2;
+ double fitness1=h1.getFitness();
+ double fitness2=h2.getFitness();
+ if (fitness1>fitness2)
+ {
+ return 1;
+ }
+ else if (fitness1==fitness2)
+ {
+ // we prefer smaller hypothesis
+ if(h1 instanceof varyANNHypothesis)
+ {
+ int sizeh1=((varyANNHypothesis) h1).getNumHiddenNeurons();
+ int sizeh2=((varyANNHypothesis) h2).getNumHiddenNeurons();
+ if (sizeh1>sizeh2)
+ {
+ return 1;
+ }
+ }
+ return 0;
+ }
+ else
+ {
+ return -1;
+ }
+ }
+
+ public boolean equals(Object o)
+ {
+ return(this.equals(o));
+ }
+}
166 eann/LICENSE
@@ -0,0 +1,166 @@
+ GNU LESSER GENERAL PUBLIC LICENSE
+ Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+
+ This version of the GNU Lesser General Public License incorporates
+the terms and conditions of version 3 of the GNU General Public
+License, supplemented by the additional permissions listed below.
+
+ 0. Additional Definitions.
+
+ As used herein, "this License" refers to version 3 of the GNU Lesser
+General Public License, and the "GNU GPL" refers to version 3 of the GNU
+General Public License.
+
+ "The Library" refers to a covered work governed by this License,
+other than an Application or a Combined Work as defined below.
+
+ An "Application" is any work that makes use of an interface provided
+by the Library, but which is not otherwise based on the Library.
+Defining a subclass of a class defined by the Library is deemed a mode
+of using an interface provided by the Library.
+
+ A "Combined Work" is a work produced by combining or linking an
+Application with the Library. The particular version of the Library
+with which the Combined Work was made is also called the "Linked
+Version".
+
+ The "Minimal Corresponding Source" for a Combined Work means the
+Corresponding Source for the Combined Work, excluding any source code
+for portions of the Combined Work that, considered in isolation, are
+based on the Application, and not on the Linked Version.
+
+ The "Corresponding Application Code" for a Combined Work means the
+object code and/or source code for the Application, including any data
+and utility programs needed for reproducing the Combined Work from the
+Application, but excluding the System Libraries of the Combined Work.
+
+ 1. Exception to Section 3 of the GNU GPL.
+
+ You may convey a covered work under sections 3 and 4 of this License
+without being bound by section 3 of the GNU GPL.
+
+ 2. Conveying Modified Versions.
+
+ If you modify a copy of the Library, and, in your modifications, a
+facility refers to a function or data to be supplied by an Application
+that uses the facility (other than as an argument passed when the
+facility is invoked), then you may convey a copy of the modified
+version:
+
+ a) under this License, provided that you make a good faith effort to
+ ensure that, in the event an Application does not supply the
+ function or data, the facility still operates, and performs
+ whatever part of its purpose remains meaningful, or
+
+ b) under the GNU GPL, with none of the additional permissions of
+ this License applicable to that copy.
+
+ 3. Object Code Incorporating Material from Library Header Files.
+
+ The object code form of an Application may incorporate material from
+a header file that is part of the Library. You may convey such object
+code under terms of your choice, provided that, if the incorporated
+material is not limited to numerical parameters, data structure
+layouts and accessors, or small macros, inline functions and templates
+(ten or fewer lines in length), you do both of the following:
+
+ a) Give prominent notice with each copy of the object code that the
+ Library is used in it and that the Library and its use are
+ covered by this License.
+
+ b) Accompany the object code with a copy of the GNU GPL and this license
+ document.
+
+ 4. Combined Works.
+
+ You may convey a Combined Work under terms of your choice that,
+taken together, effectively do not restrict modification of the
+portions of the Library contained in the Combined Work and reverse
+engineering for debugging such modifications, if you also do each of
+the following:
+
+ a) Give prominent notice with each copy of the Combined Work that
+ the Library is used in it and that the Library and its use are
+ covered by this License.
+
+ b) Accompany the Combined Work with a copy of the GNU GPL and this license
+ document.
+
+ c) For a Combined Work that displays copyright notices during
+ execution, include the copyright notice for the Library among
+ these notices, as well as a reference directing the user to the
+ copies of the GNU GPL and this license document.
+
+ d) Do one of the following:
+
+ 0) Convey the Minimal Corresponding Source under the terms of this
+ License, and the Corresponding Application Code in a form
+ suitable for, and under terms that permit, the user to
+ recombine or relink the Application with a modified version of
+ the Linked Version to produce a modified Combined Work, in the
+ manner specified by section 6 of the GNU GPL for conveying
+ Corresponding Source.
+
+ 1) Use a suitable shared library mechanism for linking with the
+ Library. A suitable mechanism is one that (a) uses at run time
+ a copy of the Library already present on the user's computer
+ system, and (b) will operate properly with a modified version
+ of the Library that is interface-compatible with the Linked
+ Version.
+
+ e) Provide Installation Information, but only if you would otherwise
+ be required to provide such information under section 6 of the
+ GNU GPL, and only to the extent that such information is
+ necessary to install and execute a modified version of the
+ Combined Work produced by recombining or relinking the
+ Application with a modified version of the Linked Version. (If
+ you use option 4d0, the Installation Information must accompany
+ the Minimal Corresponding Source and Corresponding Application
+ Code. If you use option 4d1, you must provide the Installation
+ Information in the manner specified by section 6 of the GNU GPL
+ for conveying Corresponding Source.)
+
+ 5. Combined Libraries.
+
+ You may place library facilities that are a work based on the
+Library side by side in a single library together with other library
+facilities that are not Applications and are not covered by this
+License, and convey such a combined library under terms of your
+choice, if you do both of the following:
+
+ a) Accompany the combined library with a copy of the same work based
+ on the Library, uncombined with any other library facilities,
+ conveyed under the terms of this License.
+
+ b) Give prominent notice with the combined library that part of it
+ is a work based on the Library, and explaining where to find the
+ accompanying uncombined form of the same work.
+
+ 6. Revised Versions of the GNU Lesser General Public License.
+
+ The Free Software Foundation may publish revised and/or new versions
+of the GNU Lesser General Public License from time to time. Such new
+versions will be similar in spirit to the present version, but may
+differ in detail to address new problems or concerns.
+
+ Each version is given a distinguishing version number. If the
+Library as you received it specifies that a certain numbered version
+of the GNU Lesser General Public License "or any later version"
+applies to it, you have the option of following the terms and
+conditions either of that published version or of any later version
+published by the Free Software Foundation. If the Library as you
+received it does not specify a version number of the GNU Lesser
+General Public License, you may choose any version of the GNU Lesser
+General Public License ever published by the Free Software Foundation.
+
+ If the Library as you received it specifies that a proxy can decide
+whether future versions of the GNU Lesser General Public License shall
+apply, that proxy's public statement of acceptance of any version is
+permanent authorization for you to choose that version for the
+Library.
+
500 eann/NeuralNet.java
@@ -0,0 +1,500 @@
+import java.util.*;
+import java.io.*;
+import java.util.zip.*;
+
+/**
+ * NeuralNet Class : A neural network consists of edges and neuron
+ * Author: Alex Ksikes
+**/
+public class NeuralNet implements Serializable
+{
+
+ /**************** Variables ********************/
+
+ final static int MAXNEURONS = 60; // maximum number of neurons in each layer
+ final static int BATCHMODE = 0; // batch mode for training
+ final static int STOCHASTICMODE = 1; // stochastic mode for training
+ Vector neurons; // set of neurons in this neural network
+ Vector edges; // set of edges in this neural network
+ int [][] neuronsIndex; // index of neurons. First component in this 2D array is the layer,
+ // second component is the layer numbering
+ int inputNum; // number of input neurons
+ int outputNum; // number of output neurons
+ int layerNo; // number of layers
+ double learningRate; // learning rate for the entire neural network
+ double momentum; // momentum term
+ double errAcc; // global error in the network
+ int mode; // the mode to use for training
+
+ /**************** Methods **********************/
+
+ /**
+ * Main Constructor
+ * Sets up the Multi-layer Neural Network
+ * @param layerNo indicates the number of layers
+ * @param []numInLayer contains the number of neurons in each layer,
+ * from input layer to hidden layers to output layer
+ * @param learningRate sets how fast the networks changes its weight
+ * @param momentum determine how much the weight change is based on past update
+ * @param mode indicates the mode (Batch or Stochastic) to use for training
+ * Step 1 : Creates all the neurons in the network, ordered by layers,
+ * keeping the index of the neuron in neuronsIndex[][]
+ * Step 2 : Calls MLPNetworkSetup to set up the edges between the neurons,
+ * determined by the layers they are in.
+ **/
+ public NeuralNet(int layerNo, int[] numInLayer, double learningRate, double momentum, int mode)
+ {
+ // Set up the neuron index array
+ neuronsIndex=new int[layerNo][];
+ for (int i=0; i<layerNo; i++)
+ neuronsIndex[i]=new int[numInLayer[i]];
+
+ this.layerNo=layerNo;
+ this.learningRate=learningRate;
+ this.momentum=momentum;
+ this.mode=mode;
+ this.inputNum=numInLayer[0];
+ this.outputNum=numInLayer[layerNo-1];
+ this.neurons=new Vector();
+
+ // Step 1
+ int id=0;
+ // in layer i
+ for (int i=0;i<layerNo;i++)
+ {
+ // and layer numbering j
+ for (int j=0;j<numInLayer[i];j++)
+ {
+ neurons.add(id,new Neuron(id));
+ neuronsIndex[i][j]=id; // keeping track of the index of each neuron
+ id++;
+ }
+ }
+ // Step 2
+ MLPNetworkSetup(numInLayer);
+ }
+
+ /**
+ * Set up the forward backward edges relationship in the neural network in MLP fashion
+ * @param numInLayer indicates the number of neuron in each layer
+ * Steps : Create an edge for each neuron in layer i and neuron in layer i+1.
+ * Add this edge to the set of forward edges for the neuron in layer i
+ * Add this edge to the set of backward edges for the neuron in layer i+1
+ * Call the edge reset function to set random initial weights.
+ **/
+ private void MLPNetworkSetup(int [] numInLayer)
+ {
+ this.edges=new Vector();
+ Neuron parent;
+ Neuron child;
+ Edge e;
+ int id=0;
+ // in layer i
+ for (int i=0;i<layerNo-1;i++)
+ {
+ // and layer numbering j
+ for (int j=0;j<numInLayer[i];j++)
+ {
+ parent=(Neuron) neurons.get(neuronsIndex[i][j]); // get neuron at this location
+ for (int k=0;k<numInLayer[i+1];k++)
+ {
+ child=(Neuron) neurons.get(neuronsIndex[i+1][k]);
+ e=new Edge(parent,child,learningRate,id,momentum,mode);
+ parent.addForwardEdge(e);
+ child.addBackwardEdge(e);
+ edges.add(id,e);
+ id++;
+ }
+ }
+ }
+ }
+
+ // Call the reset function on each edge
+ // Assumes a network has been built
+ public void init()
+ {
+ for (int i=0;i<edges.size();i++)
+ ((Edge) edges.get(i)).reset();
+ }
+
+ // Set the weights of the neuron at the specified layer and index in this neural net
+ // This method is used for the genetic algorithm in the decoding process
+ public void setNeuronWeights(int layer, int index, double[][] weights)
+ {
+ Neuron neuron=((Neuron) neurons.get(neuronsIndex[layer][index]));
+ Edge edge;
+ for (int i=0;i<inputNum;i++)
+ {
+ edge=((Edge) neuron.getBackwardEdges().get(i));
+ edge.setWeight(weights[i][0]);
+ }
+ for (int i=0;i<outputNum;i++)
+ {
+ edge=((Edge) neuron.getForwardEdges().get(i));
+ edge.setWeight(weights[0][i]);
+ }
+ }
+
+ /**
+ * Print out all the weights of all the edges
+ * A useful debugging tool to see whether your neural network is indeed changing the weights
+ **/
+ public void printWeight()
+ {
+ for (int i=0;i<edges.size();i++)
+ {
+ System.out.print("Weight of edge "+i+": "+ ((Edge) edges.get(i)) .getWeight()+" ");
+ }
+ }
+
+ /**
+ * run the network given an array of attributes from an example
+ * @param Example example contains the input attributes
+ * Step 1: Set all the input neurons [neurons in layer 0] with the attributes in this example
+ * Step 2: Calculate the value of each neuron beginning from the input layer to the output layer.
+ **/
+ private void runNetwork(Example example)
+ {
+ // Step 1
+ Neuron input;
+ double x;
+ for (int j=0;j<inputNum;j++)
+ {
+ input=(Neuron) neurons.get(neuronsIndex[0][j]);
+ x=example.getAttribute(j);
+ input.setValue(x);
+ }
+ // Step 2
+ Neuron n;
+ for (int i=1;i<layerNo;i++)
+ {
+ for (int j=0;j<neuronsIndex[i].length;j++)
+ {
+ n=(Neuron) neurons.elementAt(neuronsIndex[i][j]);
+ n.calValue();
+ }
+ }
+ }
+
+
+ /**
+ * Train the network using this example
+ * @param example contains the input attributes
+ * Step 1: run network on this training example
+ * Step 2: perform backpropagation based on the class label and network output for this example
+ **/
+ private void trainSingle(Example example)
+ {
+ runNetwork(example);
+ // compute the change in weight
+ backPropagation(example);
+ // update the weights after having seen this single example
+ updateWeights();
+ }
+
+ /**
+ * Train the network using all the examples in the training set
+ * @param example contains the input attributes
+ **/
+ private void trainAll(DataSet trainingSet)
+ {
+ Example example;
+ for (int i=0;i<trainingSet.size();i++)
+ {
+ example=trainingSet.getExample(i);
+ runNetwork(example);
+ // update the change in weight of each edge
+ backPropagation(example);
+ }
+ // now update the weights of each edge
+ updateWeights();
+ }
+
+ /**
+ * Update the weights of this ANN and reinitialize the change in weights
+ **/
+ public void updateWeights()
+ {
+ Edge e;
+ for (int i=0;i<edges.size();i++)
+ {
+ e=(Edge) edges.get(i);
+ e.updateWeight();
+ e.initDeltaWeight();
+ }
+ }
+
+ /**
+ * To test a single element
+ * Assume there is only one output neuron, so the class label is simply based on whether the output value
+ * is more than 0.5 or less than 0.5. If output neuron >0.5 return 1 else return 0.
+ * @param example contains the input attribute
+ * @return the class it should be in
+ **/
+ public int testSingle(Example example)
+ {
+ Neuron output;
+ runNetwork(example);
+ output=(Neuron) neurons.elementAt(neuronsIndex[layerNo-1][0]);
+ if (output.getValue()>0.5)
+ return 1;
+ else
+ return 0;
+ }
+
+ /**
+ * To test the accuracy level of the entire data set
+ * @param testSet contains all the example to test for accuracy
+ * @param return the percentage of correct classification
+ **/
+ public double testDataSet(DataSet testSet)
+ {
+ Example example;
+ int label;
+ int testValue;
+ int numCorrect=0;
+ int numExamples=testSet.size();
+ for (int i=0;i<numExamples;i++)
+ {
+ example=testSet.getExample(i);
+ testValue=testSingle(example);
+ label=example.getClassLabel();
+ if (testValue==label)
+ numCorrect++;
+ }
+ return ((1.0*numCorrect)/numExamples);
+ }
+
+ // Compute the root mean squared error of this network
+ public double computeRMS(DataSet trainingSet)
+ {
+ Example example;
+ int networkOutput;
+ int targetOutput;
+ int numExamples=trainingSet.size();
+ double sum=0;
+ for (int i=0;i<numExamples;i++)
+ {
+ example=trainingSet.getExample(i);
+ networkOutput=testSingle(example);
+ targetOutput=example.getClassLabel();
+ sum=Math.pow(networkOutput-targetOutput,2) + sum;
+ }
+ return Math.sqrt(sum/numExamples);
+ }
+
+ // Report the accuracy (RMS error and training and validation accuracy after every n epoch)
+ // and write into these data
+ public void reportAccuracy(DataSet trainingSet,DataSet evaluationSet,int epoch,int n,FileWriter out) throws IOException
+ {
+ double trainingAcc=0;
+ double validationAcc=0;
+ // report RMS error
+ System.out.println("RMS : " + computeRMS(trainingSet));
+ // report training and validation accuracy after every n epochs
+ if (epoch%n==0)
+ {
+ trainingAcc=testDataSet(trainingSet);
+ System.out.println("Training Acc : " + trainingAcc);
+ validationAcc=testDataSet(evaluationSet);
+ System.out.println("Validation Acc : " + validationAcc);
+ out.write("\n"+epoch+","+validationAcc+","+trainingAcc+"\r");
+ }
+ }
+
+ /**
+ * Allow user to use the entire data set to train for specified number of iterations
+ * @param trainingSet contains all the examples to be used for training
+ * @param evaluationSet contains all the examples for validation purpose
+ * @param epochNum indicates the number of iterations to train for
+ **/
+ public void iterativeTrain(DataSet trainingSet, DataSet evaluationSet, int epochNum, FileWriter out) throws IOException
+ {
+ int examplesNum=trainingSet.size();
+ Example example;
+ if (mode==STOCHASTICMODE)
+ {
+ for (int epoch=0;epoch<epochNum;epoch++)
+ {
+ for (int i=0;i<trainingSet.size();i++)
+ {
+ example=trainingSet.getExample(i);
+ trainSingle(example);
+ }
+ reportAccuracy(trainingSet,evaluationSet,epoch,20,out);
+ }
+ }
+ if (mode==BATCHMODE)
+ {
+ for (int epoch=0;epoch<epochNum;epoch++)
+ {
+ trainAll(trainingSet);
+ reportAccuracy(trainingSet,evaluationSet,epoch,20,out);
+ }
+ }
+ }
+
+ /**
+ * Perform back propagation : gradient descent search to learn the weights of the network
+ * Stochastic gradient descent
+ * @param example contains all the class label for this example
+ **/
+ public void backPropagation(Example example)
+ {
+ Neuron output;
+ Neuron n;
+ Edge e;
+ double targetVal;
+ double errorTerm;
+ // from output-to-hidden units
+ targetVal=example.getClassLabel();
+ for (int j=0;j<outputNum;j++)
+ {
+ output=(Neuron) neurons.get(neuronsIndex[layerNo-1][j]);
+ output.backErrorTrack(targetVal);
+ errorTerm=output.getErrorTerm();
+ for (int k=0;k<output.getParentNum();k++)
+ {
+ e=(Edge) (output.getBackwardEdges()).get(k);
+ e.updateDeltaWeight(errorTerm);
+ }
+ }
+ // from hidden-to-hidden units till hidden-to-input units
+ for (int i=layerNo-2; i>0; i--)
+ {
+ for (int j=0;j<neuronsIndex[i].length;j++)
+ {
+ n=(Neuron) neurons.elementAt(neuronsIndex[i][j]);
+ n.backErrorTrack();
+ errorTerm=n.getErrorTerm();
+ for (int k=0;k<n.getParentNum();k++)
+ {
+ e=(Edge) (n.getBackwardEdges()).get(k);
+ e.updateDeltaWeight(errorTerm);
+ }
+ }
+ }
+ }
+
+ /**
+ * Save NeuralNetwork to file
+ **/
+ public void save(String filename)
+ {
+ // Create a file dialog to query the user for a filename.
+ try {
+ // Create the necessary output streams to midistruct.
+ FileOutputStream fos = new FileOutputStream(filename);
+ GZIPOutputStream gzos = new GZIPOutputStream(fos);
+ ObjectOutputStream out = new ObjectOutputStream(gzos);
+ out.writeObject(this);
+ out.flush();
+ out.close();
+ }
+ catch (IOException e)
+ {
+ System.out.println(e);
+ }
+ }
+
+ /**
+ * Load NeuralNetwork from file
+ **/
+ public static NeuralNet load(String filename)
+ {
+ NeuralNet x;
+ try {
+ // Create necessary input streams
+ FileInputStream fis = new FileInputStream(filename);
+ GZIPInputStream gzis = new GZIPInputStream(fis);
+ ObjectInputStream in = new ObjectInputStream(gzis);
+ x = (NeuralNet)in.readObject();
+ in.close(); // Close the stream.
+ return x;
+ }
+ catch (Exception e)
+ {
+ System.out.println(e);
+ }
+ return null;
+ }
+
+
+ public Vector getNeurons()
+ {
+ return neurons;
+ }
+
+ public Vector getEdges()
+ {
+ return edges;
+ }
+
+ /** Start off point of this program **/
+ public static void main(String args[]) throws IOException
+ {
+ if (args.length!=3)
+ {
+ System.out.println("Wrong usage. Type java NeuralNet [trainingFile] [evaluationFile] [mode]");
+ }
+ else
+ {
+ // read in the datasets
+ DataSet trainingData = new DataSet(args[0]);
+ DataSet evaluationData = new DataSet(args[1]);
+ int mode = Integer.parseInt(args[2]);
+
+ // decide network topology
+ int layers[] = new int[3];
+ layers[0] = trainingData.getAttributeNum();
+ layers[1] = 9;
+ layers[2] = 1;
+ //layers[3] = 7;
+ //layers[4] = 3;
+ //layers[5] = 1;
+
+ // choose the learning rate and momentum parameters
+ double learningRate=0.1; // for batchmode take learning rate less than 0.05
+ double momentum=0.01; // for batchmode take momentum less than 0.001
+
+ // create a network
+ NeuralNet myNetwork = new NeuralNet(layers.length,layers, learningRate, momentum, mode);
+ // initial the weights
+ myNetwork.init();
+ System.out.println("MLP Neural Network Created");
+
+ // report parameters to user and write into files for graphs...
+ File outputFile=new File("testANN.txt");
+ FileWriter out=new FileWriter(outputFile);
+ for (int i=0;i<layers.length;i++)
+ {
+ System.out.println("Layer "+i+" has "+layers[i]+" elements");
+ out.write("\nLayer "+i+" has "+layers[i]+" elements\r");
+ }
+ System.out.println("Total number of neurons: " + myNetwork.getNeurons().size());
+ out.write("\nTotal number of neurons: " + myNetwork.getNeurons().size()+"\r");
+ System.out.println("Total number of edges: " + myNetwork.getEdges().size());
+ out.write("\nTotal number of edges: " + myNetwork.getEdges().size()+"\r");
+ if (mode==BATCHMODE)
+ {
+ System.out.println("Batchmode selected");
+ out.write("\nBatchmode selected \r");
+ }
+ else
+ {
+ System.out.println("Stochastic selected");
+ out.write("\nStochastic selected\r");
+ }
+ System.out.println("Learning Rate is "+learningRate);
+ out.write("\nLearning Rate is "+learningRate+"\r");
+ System.out.println("Momentum is "+momentum);
+ out.write("\nMomentum is "+momentum+"\r");
+
+ // training the network....
+ myNetwork.iterativeTrain(trainingData, evaluationData,69,out);
+ myNetwork.reportAccuracy(trainingData,evaluationData,1,1,out);
+ //myNetwork.save("myANN.dat");
+ out.close();
+ }
+ }
+}
20 eann/NeuralTest.java
@@ -0,0 +1,20 @@
+import java.io.*;
+
+// Class to test the mystery dataset on my saved network
+public class NeuralTest {
+
+ public static void main(String args[])
+ {
+ if (args.length!=2)
+ {
+ System.out.println("Wrong usage. Type java NeuralTest [evaluationFile] [savedANN]");
+ }
+ else
+ {
+ DataSet mysteryDataSet=new DataSet(args[0]);
+ NeuralNet myNetwork=NeuralNet.load(args[1]);
+ double accuracy=myNetwork.testDataSet(mysteryDataSet);
+ System.out.println("Accuracy of my saved ANN: " + accuracy);
+ }
+ }
+}
165 eann/Neuron.java
@@ -0,0 +1,165 @@
+import java.lang.Math;
+import java.util.Vector;
+import java.io.Serializable;
+
+/**
+ * Neuron class. Each neuron contains its own activation function and value.
+ * Author: Alex Ksikes
+ **/
+public class Neuron implements Serializable
+{
+
+ /************ Variables *************/
+
+ private double value; // value in this neuron
+ private double errorTerm; // error term of this neuron
+ private Vector fEdge; // set of forward edges
+ private Vector bEdge; // set of backward edges
+ private int id; // id of the neuron
+
+ /************ Methods *************/
+
+ /**
+ * Constructor for the neuron
+ * @param id of the neuron
+ **/
+ public Neuron(int idd)
+ {
+ id=idd;
+ fEdge=new Vector();
+ bEdge=new Vector();
+ }
+
+ /**
+ * Gets the id of the neuron
+ **/
+ public int getId()
+ {
+ return id;
+ }
+
+ public Vector getBackwardEdges()
+ {
+ return bEdge;
+ }
+
+ public Vector getForwardEdges()
+ {
+ return fEdge;
+ }
+
+ /**
+ * Activation function
+ * Here we simply call the sigmoid function
+ * @param f is the input to this activation function
+ **/
+ public double actf(double f)
+ {
+ return sigmoid(f);
+ }
+
+ public double sigmoid(double y)
+ {
+ return 1.0 / ( 1 + Math.exp(-1*y) );
+ }
+
+ /**
+ * Set the value for this neuron. Meant only for input neurons
+ **/
+ public void setValue(double x)
+ {
+ value=x;
+ }
+
+ /**
+ * @return the value in this neuron
+ **/
+ public double getValue()
+ {
+ return value;
+ }
+
+ /**
+ * calculates the value of this node based on weights of backedges and inputs of parents
+ **/
+ public void calValue()
+ {
+ // Compute weighted sum of its imput parents
+ double net=0;
+ Edge e;
+ for (int i=0; i<getParentNum(); i++)
+ {
+ e=(Edge) bEdge.get(i);
+ net=net + e.getWeight()* e.getSource().getValue();
+ }
+ // Apply activation function
+ value=actf(net);
+ }
+
+ /**
+ * @return the number of forward edges
+ **/
+ public int getChildNum()
+ {
+ return fEdge.size();
+ }
+
+ /**
+ * @return the number of backwards edges
+ **/
+ public int getParentNum()
+ {
+ return bEdge.size();
+ }
+
+ /**
+ * Add a new forward edge
+ **/
+ public void addForwardEdge(Edge e)
+ {
+ fEdge.add(e);
+ }
+
+ /**
+ * Add a new backward edge
+ **/
+ public void addBackwardEdge(Edge