diff --git a/bayes/pom.xml b/bayes/pom.xml new file mode 100755 index 0000000..3704d6e --- /dev/null +++ b/bayes/pom.xml @@ -0,0 +1,37 @@ + + + 4.0.0 + + ml-parent + com.enigmastation.ml + 4.0-SNAPSHOT + + com.enigmastation.ml + bayes + 4.0-SNAPSHOT + bayes + http://maven.apache.org + + UTF-8 + + + + org.testng + testng + 6.3.1 + test + + + org.apache.lucene + lucene-analyzers + 3.4.0 + + + ${parent.groupId} + common + ${project.version} + + + diff --git a/bayes/src/main/java/com/enigmastation/ml/bayes/Classifier.java b/bayes/src/main/java/com/enigmastation/ml/bayes/Classifier.java new file mode 100755 index 0000000..66af7c8 --- /dev/null +++ b/bayes/src/main/java/com/enigmastation/ml/bayes/Classifier.java @@ -0,0 +1,11 @@ +package com.enigmastation.ml.bayes; + +import java.util.Map; + +public interface Classifier { + Object classify(Object source); + Object classify(Object source, Object defaultClassification); + Object classify(Object source, Object defaultClassification, double strength); + Map getClassificationProbabilities(Object source); + void train(Object source, Object classification); +} diff --git a/bayes/src/main/java/com/enigmastation/ml/bayes/Tokenizer.java b/bayes/src/main/java/com/enigmastation/ml/bayes/Tokenizer.java new file mode 100755 index 0000000..0dfbebf --- /dev/null +++ b/bayes/src/main/java/com/enigmastation/ml/bayes/Tokenizer.java @@ -0,0 +1,7 @@ +package com.enigmastation.ml.bayes; + +import java.util.List; + +public interface Tokenizer { + List tokenize(Object source); +} diff --git a/bayes/src/main/java/com/enigmastation/ml/bayes/impl/PorterTokenizer.java b/bayes/src/main/java/com/enigmastation/ml/bayes/impl/PorterTokenizer.java new file mode 100755 index 0000000..9f9611f --- /dev/null +++ b/bayes/src/main/java/com/enigmastation/ml/bayes/impl/PorterTokenizer.java @@ -0,0 +1,35 @@ +package com.enigmastation.ml.bayes.impl; + +import com.enigmastation.ml.bayes.Tokenizer; +import org.apache.lucene.analysis.PorterStemFilter; +import org.apache.lucene.analysis.standard.StandardTokenizer; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.util.Version; + +import java.io.IOException; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.List; + +public class PorterTokenizer implements Tokenizer { + @Override + public List tokenize(Object source) { + List tokens = new ArrayList<>(source.toString().length() / 5); + org.apache.lucene.analysis.Tokenizer tokenizer = + new StandardTokenizer(Version.LUCENE_34, + new StringReader(source.toString())); + CharTermAttribute charTermAttribute = tokenizer.getAttribute(CharTermAttribute.class); + PorterStemFilter filter = new PorterStemFilter(tokenizer); + try { + while (filter.incrementToken()) { + String term = charTermAttribute.toString().toLowerCase(); + if (term.length() > 2) { + tokens.add(term); + } + } + } catch (IOException e) { + throw new RuntimeException("Should not happen: " + e.getMessage(), e); + } + return tokens; + } +} diff --git a/bayes/src/main/java/com/enigmastation/ml/bayes/impl/SimpleClassifier.java b/bayes/src/main/java/com/enigmastation/ml/bayes/impl/SimpleClassifier.java new file mode 100755 index 0000000..4e31079 --- /dev/null +++ b/bayes/src/main/java/com/enigmastation/ml/bayes/impl/SimpleClassifier.java @@ -0,0 +1,171 @@ +package com.enigmastation.ml.bayes.impl; + +import com.enigmastation.ml.bayes.Classifier; +import com.enigmastation.ml.bayes.Tokenizer; +import com.enigmastation.ml.common.collections.MapBuilder; +import com.enigmastation.ml.common.collections.ValueProvider; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +public class SimpleClassifier implements Classifier { + Map> features; + Map categories; + Tokenizer tokenizer = new PorterTokenizer(); + Map thresholds = new MapBuilder().defaultValue(1.0).build(); + + SimpleClassifier(Map> features, Map categories) { + this.features = features; + this.categories = categories; + } + + public SimpleClassifier() { + features = new MapBuilder().valueProvider(new ValueProvider() { + @Override + public Object getDefault(Object k) { + return new MapBuilder().defaultValue(0).build(); + } + }).build(); + categories = new MapBuilder().defaultValue(0).build(); + } + + @Override + public Object classify(Object source) { + return classify(source, "none"); + } + + @Override + public Object classify(Object source, Object defaultClassification) { + return classify(source, defaultClassification, 0.0); + } + + @Override + public Object classify(Object source, Object defaultClassification, double strength) { + Map probs = getClassificationProbabilities(source); + double max = 0.0; + Object category = null; + for (Map.Entry entry : probs.entrySet()) { + if (entry.getValue() > max) { + max = entry.getValue(); + category = entry.getKey(); + } + } + for (Map.Entry entry : probs.entrySet()) { + if (entry.getKey().equals(category)) { + continue; + } + + if ((entry.getValue() * getThreshold(category)) > probs.get(category)) { + return defaultClassification; + } + } + return category; + } + + @Override + public Map getClassificationProbabilities(Object source) { + Map probabilities = new HashMap(); + for (Object category : categories()) { + probabilities.put(category, docprob(source, category)); + } + return probabilities; + } + + @Override + public void train(Object source, Object classification) { + for (Object feature : tokenizer.tokenize(source)) { + incf(feature, classification); + } + incc(classification); + } + + private void incf(Object feature, Object category) { + Map cat = features.get(feature); + features.put(feature, cat); + cat.put(category, cat.get(category) + 1); + } + + private void incc(Object category) { + categories.put(category, categories.get(category) + 1); + } + + // the number of times a feature has occurred in a category + int fcount(Object feature, Object category) { + if (features.containsKey(feature) && features.get(feature).containsKey(category)) { + return features.get(feature).get(category); + } + return 0; + } + + int catcount(Object category) { + if (categories.containsKey(category)) { + return categories.get(category); + } + return 0; + } + + int totalcount() { + int sum = 0; + for (Integer i : categories.values()) { + sum += i; + } + return sum; + } + + Set categories() { + return categories.keySet(); + } + + double fprob(Object feature, Object category) { + if (catcount(category) == 0) { + return 0.0; + } + return (1.0 * fcount(feature, category)) / catcount(category); + } + + double weightedprob(Object feature, Object category, double weight, double assumedProbability) { + double basicProbability = fprob(feature, category); + //System.out.println("basic probability: "+basicProbability); + double totals = 0; + for (Object cat : categories()) { + totals += fcount(feature, cat); + } + //System.out.printf("((%f * %f)+(%f * %f))/(%f + %f) %n", weight, + //assumedProbability, totals, basicProbability, weight, totals); + return ((weight * assumedProbability) + (totals * basicProbability)) / (weight + totals); + } + + double weightedprob(Object feature, Object category, double weight) { + return weightedprob(feature, category, weight, 0.5); + } + + double weightedprob(Object feature, Object category) { + return weightedprob(feature, category, 1.0); + } + + /* naive bayes, very naive - and not what we usually need. */ + double docprob(Object corpus, Object category) { + double p = 1.0; + for (Object f : tokenizer.tokenize(corpus)) { + p *= weightedprob(f, category); + } + return p; + } + + double prob(Object corpus, Object category) { + double catprob = (1.0 * catcount(category)) / totalcount(); + double docprob = docprob(corpus, category); + return docprob * catprob; + } + + public void setThreshold(Object category, Double threshold) { + thresholds.put(category, threshold); + } + + public double getThreshold(Object category) { + return thresholds.get(category); + } + + +} diff --git a/bayes/src/main/java/com/enigmastation/ml/bayes/impl/SimpleTokenizer.java b/bayes/src/main/java/com/enigmastation/ml/bayes/impl/SimpleTokenizer.java new file mode 100755 index 0000000..758b336 --- /dev/null +++ b/bayes/src/main/java/com/enigmastation/ml/bayes/impl/SimpleTokenizer.java @@ -0,0 +1,20 @@ +package com.enigmastation.ml.bayes.impl; + +import com.enigmastation.ml.bayes.Tokenizer; + +import java.util.ArrayList; +import java.util.List; +import java.util.Scanner; + +public class SimpleTokenizer implements Tokenizer { + @Override + public List tokenize(Object source) { + String src=source.toString(); + List tokens=new ArrayList<>(src.length()/6); + Scanner scanner=new Scanner(src); + while(scanner.hasNext("\\S*")) { + tokens.add(scanner.next("\\S*")); + } + return tokens; + } +} diff --git a/bayes/src/test/java/com/enigmastation/ml/bayes/TokenizerTest.java b/bayes/src/test/java/com/enigmastation/ml/bayes/TokenizerTest.java new file mode 100755 index 0000000..796d262 --- /dev/null +++ b/bayes/src/test/java/com/enigmastation/ml/bayes/TokenizerTest.java @@ -0,0 +1,25 @@ +package com.enigmastation.ml.bayes; + +import com.enigmastation.ml.bayes.impl.PorterTokenizer; +import com.enigmastation.ml.bayes.impl.SimpleTokenizer; +import org.testng.annotations.Test; + +import java.util.List; + +import static org.testng.Assert.assertEquals; + +public class TokenizerTest { + @Test + public void testSimpleTokenizer() { + Tokenizer tokenizer = new SimpleTokenizer(); + List objects = tokenizer.tokenize("1 2 3 4 5 6 7"); + assertEquals(objects.size(), 7); + } + + @Test + public void testPorterTokenizer() { + Tokenizer porterTokenizer = new PorterTokenizer(); + porterTokenizer.tokenize("Now is the time for all good men to come to the aid of their finalizing country."); + assertEquals(porterTokenizer.tokenize("the quick brown fox jumps over the lazy dog's tail").size(), 10); + } +} diff --git a/bayes/src/test/java/com/enigmastation/ml/bayes/impl/SimpleClassifierTest.java b/bayes/src/test/java/com/enigmastation/ml/bayes/impl/SimpleClassifierTest.java new file mode 100755 index 0000000..35fae4c --- /dev/null +++ b/bayes/src/test/java/com/enigmastation/ml/bayes/impl/SimpleClassifierTest.java @@ -0,0 +1,66 @@ +package com.enigmastation.ml.bayes.impl; + +import org.testng.annotations.Test; + +import static org.testng.Assert.assertEquals; + +public class SimpleClassifierTest { + @Test + public void testInternalsOfTrain() { + SimpleClassifier c = new SimpleClassifier(); + c.train("the quick brown fox jumps over the lazy dog's tail", "good"); + c.train("make quick money in the online casino", "bad"); + assertEquals(c.fcount("quick", "good"), 1); + assertEquals(c.fcount("quick", "bad"), 1); + } + + @Test + public void testfprob() { + SimpleClassifier cl = getTrainedClassifier(); + assertEquals(cl.fprob("quick", "good"), 0.6666, 0.0001); + assertEquals(cl.weightedprob("monei", "good"), 0.25, 0.001); + train(cl); + assertEquals(cl.weightedprob("monei", "good"), 0.166, 0.001); + } + + @Test + public void testBayes() { + SimpleClassifier cl = getTrainedClassifier(); + assertEquals(cl.prob("quick rabbit", "good"), 0.15624, 0.0001); + assertEquals(cl.prob("quick rabbit", "bad"), 0.05, 0.001); + } + + @Test + public void testThreshold() { + SimpleClassifier cl = getTrainedClassifier(); + assertEquals(cl.getThreshold("bad"), 1.0, 0.01); + } + + @Test + public void testClassification() { + SimpleClassifier cl = getTrainedClassifier(); + assertEquals(cl.classify("quick rabbit", "unknown"), "good"); + assertEquals(cl.classify("quick money", "unknown"), "bad"); + cl.setThreshold("bad", 3.0); + assertEquals(cl.classify("quick money", "unknown"), "unknown"); + for (int i = 0; i < 10; i++) { + train(cl); + } + assertEquals(cl.classify("quick money", "unknown"), "bad"); + } + + private SimpleClassifier getTrainedClassifier() { + SimpleClassifier cl = new SimpleClassifier(); + + train(cl); + return cl; + } + + private void train(SimpleClassifier cl) { + cl.train("Nobody owns the water", "good"); + cl.train("The quick rabbit jumps fences", "good"); + cl.train("Buy pharmaceuticals now", "bad"); + cl.train("make quick money in the online casino", "bad"); + cl.train("the quick brown fox jumps", "good"); + } +} diff --git a/ci-bayes/src/test/java/com/enigmastation/classifier/corpus/CorpusTest.java b/ci-bayes/src/test/java/com/enigmastation/classifier/corpus/CorpusTest.java old mode 100644 new mode 100755 index 459b629..5a2e958 --- a/ci-bayes/src/test/java/com/enigmastation/classifier/corpus/CorpusTest.java +++ b/ci-bayes/src/test/java/com/enigmastation/classifier/corpus/CorpusTest.java @@ -1,307 +1,11 @@ package com.enigmastation.classifier.corpus; -import com.db4o.Db4oEmbedded; -import com.db4o.ObjectContainer; -import com.db4o.ObjectSet; -import com.enigmastation.classifier.ClassifierProbability; -import com.enigmastation.classifier.FisherClassifier; -import com.enigmastation.classifier.impl.FisherClassifierImpl; -import com.enigmastation.classifier.testing.MemoryMonitor; -import com.enigmastation.dao.CategoryDAO; -import com.enigmastation.dao.FeatureDAO; -import com.enigmastation.dao.db4o.CategoryDAOImpl; -import com.enigmastation.dao.db4o.FeatureDAOImpl; -import com.enigmastation.extractors.impl.SimpleWordLister; -import com.ice.tar.TarArchive; -import org.apache.tools.bzip2.CBZip2InputStream; -import org.testng.annotations.BeforeTest; -import org.testng.annotations.Test; - -import java.io.*; -import java.util.Date; - +/** + * Created by IntelliJ IDEA. + * User: joeo + * Date: 1/23/12 + * Time: 4:05 PM + * To change this template use File | Settings | File Templates. + */ public class CorpusTest { - @BeforeTest(alwaysRun = true) - public void setup() throws Throwable { - try { - FisherClassifierImpl classifier = new FisherClassifierImpl(); - classifier.setWordLister(new SimpleWordLister()); - FeatureDAO fdao=new FeatureDAOImpl(); - CategoryDAO cdao=new CategoryDAOImpl(); - classifier.setFeatureDAO(fdao); - classifier.setCategoryDAO(cdao); - classifier.init(); - - this.classifier = classifier; - - ObjectContainer db = Db4oEmbedded.openFile(Db4oEmbedded - .newConfiguration(), "dbFile"); - ObjectSet set = db.queryByExample(new Object()); - for (Object o : set) { - System.out.println("deleting object " + o + " from database"); - db.delete(o); - } - db.close(); - } catch(Throwable t) { - t.printStackTrace(); - throw t; - } - } - - FisherClassifier classifier; - - interface Manager { - void handleFile(Command c, String type, File corpus) throws IOException; - } - - interface Command { - String getType(); - - void execute(String s, String type); - } - - interface Selector { - boolean accept(int i); - } - - @Test(groups = {"fulltest"}) - public void runTest() throws IOException { - MemoryMonitor monitor = new MemoryMonitor(); - monitor.start(); - //monitor.setMonitoring(true); - savedOutput = new PrintWriter(sw); - long start = System.currentTimeMillis(); - buildClassifier(); - long end = System.currentTimeMillis(); - System.gc(); - savedOutput.printf("Training Runtime: %dms\n", (end - start)); - monitor.setMonitoring(true); - start = System.currentTimeMillis(); - testClassifier(); - end = System.currentTimeMillis(); - monitor.setMonitoring(false); - System.gc(); - savedOutput.printf("Testing Runtime: %dms\n", (end - start)); - showOutput(); - monitor.shutdown(); - // no need to empty the filesystem with maven, since we no longer pollute the same tree. - //emptyFilesystem(); - - // if this test takes longer than twenty seconds, something's wrong. We're running slow. - assert (end - start) < 20000L; - } - - private static void showOutput() { - System.err.println(sw.getBuffer().toString()); - } - - @SuppressWarnings("unused") - private void emptyFilesystem() throws IOException { - processFiles(new Manager() { - public void handleFile(Command c, String type, File corpus) throws IOException { - corpus.delete(); - } - }, new Command() { - public String getType() { - return "deleting"; - } - - public void execute(String s, String type) { - } - }, new Selector() { - public boolean accept(int i) { - return true; - } - }, false, true); - } - - static PrintWriter savedOutput; - static StringWriter sw = new StringWriter(); - - private void testClassifier() throws IOException { - - final int[] hits = new int[1]; - final int[] misses = new int[2]; - processFiles(new Manager() { - public void handleFile(Command c, String type, File corpus) throws IOException { - FileReader fr = new FileReader(corpus); - BufferedReader br = new BufferedReader(fr, 16384); - StringBuilder sb = new StringBuilder(); - String s; - while ((s = br.readLine()) != null) { - sb.append(s); - } - br.close(); - fr.close(); - c.execute(sb.toString(), type); - } - }, new Command() { - boolean normalized = false; - - public String getType() { - return "testing"; - } - - public void execute(String s, String type) { - String f = classifier.getClassification(s, "unknown"); - if (!f.equals(type)) { - System.out.println("should have gotten " + type + ", got " + f); - if (f.equals("unknown")) { - misses[1]++; - } else { - misses[0]++; - } - } else { - if (normalized == false) { - normalized = true; - ClassifierProbability[] probs = classifier.getProbabilities(s); - savedOutput.println("probabilities: "); - for (ClassifierProbability prob : probs) { - savedOutput.println(prob.toString()); - } - } - hits[0]++; - } - } - }, new Selector() { - public boolean accept(int i) { - return (i % 10) > 7; - } - }, false); - savedOutput.println("Hits: " + hits[0]); - savedOutput.println("Misses: " + (misses[0] + misses[1])); - savedOutput.println("Wrong classifications: " + misses[0]); - savedOutput.println("Unclear classifications: " + misses[1]); - savedOutput.println("Totals: " + (hits[0] + misses[0] + misses[1]) + " items"); - savedOutput.println("Hit rate: " + ((1.0 * hits[0]) / (hits[0] + misses[0] + misses[1])) + "%"); - } - - private void buildClassifier() throws IOException { - processFiles(new Manager() { - public void handleFile(Command c, String type, File corpus) throws IOException { - FileReader fr = new FileReader(corpus); - BufferedReader br = new BufferedReader(fr); - StringBuilder sb = new StringBuilder(); - String s; - while ((s = br.readLine()) != null) { - sb.append(s); - } - br.close(); - fr.close(); - c.execute(sb.toString(), type); - } - }, new Command() { - public String getType() { - return "training"; - } - - public void execute(String s, String type) { - classifier.train(s, type); - } - }, new Selector() { - public boolean accept(int i) { - return (i % 10) <= 7; - } - }, true, true); - } - - void processFiles(final Manager m, Command c, final Selector s, boolean expand) throws IOException { - processFiles(m, c, s, expand, false); - } - - void processFiles(final Manager m, final Command c, final Selector s, final boolean expand, boolean multithread) throws IOException { - - File file = new File(System.getProperty("user.dir") + "/target/test-classes/training"); - final File[] files = file.listFiles(new FilenameFilter() { - public boolean accept(File dir, String name) { - if (name.indexOf("bz2") != -1) { - if (name.indexOf("data") == -1) { - return true; - } - } - return false; - } - }); - ThreadGroup g = new ThreadGroup("CORPUS GROUP"); - if (expand) { - for (final File f : files) { - Thread e1 = new Thread(g, "Expando " + f.getName()) { - public void run() { - try { - final File x = new File(f.toString() + ".data"); - x.mkdir(); - - FileInputStream fis = new FileInputStream(f); - BufferedInputStream bis = new BufferedInputStream(fis); - //noinspection ResultOfMethodCallIgnored - bis.read(new byte[2]); - CBZip2InputStream bzip2 = new CBZip2InputStream(bis); - TarArchive archive = new TarArchive(bzip2); - archive.extractContents(x); - System.out.printf("%s finished expanding.\n", f.getName()); - } catch (IOException ioe) { - ioe.printStackTrace(); - } - } - }; - e1.start(); - } - while (g.activeCount() > 0) { - try { - Thread.sleep(2000); - System.out.printf("%s waiting for expansion to finish: %d threads\n", new Date(), g.activeCount()); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - - } - - for (final File f : files) { - Thread t = new Thread(g, f.getName()) { - public void run() { - try { - System.out.println(f); - final File x = new File(f.toString() + ".data"); - - System.out.printf("%s...%s!\n", f.getName(), c.getType()); - String type = f.toString().indexOf("ham") == -1 ? "spam" : "ham"; - int t = 0; - for (File dir : x.listFiles(new FilenameFilter() { - public boolean accept(File dir, String name) { - return name.indexOf("data") == -1; - } - })) { - for (File corpus : dir.listFiles()) { - if (s.accept(t)) { - m.handleFile(c, type, corpus); - } - if (++t % 100 == 0) { - System.out.printf("%s: %d mark\n", c.getType(), t); - } - } - } - } catch (IOException e) { - e.printStackTrace(); - } - } - }; - if (multithread) { - t.start(); - } else { - t.run(); - } - } - if (multithread) { - while (g.activeCount() > 0) { - try { - Thread.sleep(2000); - System.out.printf("%s: Waiting for %d threads to finish...\n", new Date().toString(), g.activeCount()); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - } - } - } diff --git a/ci-bayes/src/test/java/com/enigmastation/classifier/corpus/OriginalCorpusTest.java b/ci-bayes/src/test/java/com/enigmastation/classifier/corpus/OriginalCorpusTest.java new file mode 100755 index 0000000..add9c4f --- /dev/null +++ b/ci-bayes/src/test/java/com/enigmastation/classifier/corpus/OriginalCorpusTest.java @@ -0,0 +1,312 @@ +package com.enigmastation.classifier.corpus; + +import com.enigmastation.classifier.ClassifierProbability; +import com.enigmastation.classifier.FisherClassifier; +import com.enigmastation.classifier.impl.FisherClassifierImpl; +import com.enigmastation.classifier.testing.MemoryMonitor; +import com.enigmastation.dao.CategoryDAO; +import com.enigmastation.dao.FeatureDAO; +import com.enigmastation.dao.objectify.CategoryDAOImpl; +import com.enigmastation.dao.objectify.FeatureDAOImpl; +import com.enigmastation.extractors.impl.SimpleWordLister; +import com.google.appengine.tools.development.testing.LocalDatastoreServiceTestConfig; +import com.google.appengine.tools.development.testing.LocalServiceTestHelper; +import com.ice.tar.TarArchive; +import org.apache.tools.bzip2.CBZip2InputStream; +import org.testng.annotations.AfterTest; +import org.testng.annotations.BeforeTest; +import org.testng.annotations.Test; + +import java.io.*; +import java.util.Date; + +public class OriginalCorpusTest { + + private final LocalServiceTestHelper helper = + new LocalServiceTestHelper(new LocalDatastoreServiceTestConfig()); + + @AfterTest + public void tearDown() { + helper.tearDown(); + } + + @BeforeTest(alwaysRun = true) + public void setup() throws Throwable { + try { + FisherClassifierImpl classifier = new FisherClassifierImpl(); + classifier.setWordLister(new SimpleWordLister()); + FeatureDAO fdao = new FeatureDAOImpl(); + CategoryDAO cdao = new CategoryDAOImpl(); + classifier.setFeatureDAO(fdao); + classifier.setCategoryDAO(cdao); + classifier.init(); + + this.classifier = classifier; + helper.setUp(); + + } catch (Throwable t) { + t.printStackTrace(); + throw t; + } + } + + FisherClassifier classifier; + + interface Manager { + void handleFile(Command c, String type, File corpus) throws IOException; + } + + interface Command { + String getType(); + + void execute(String s, String type); + } + + interface Selector { + boolean accept(int i); + } + + @Test(groups = {"fulltest"}) + public void runTest() throws IOException { + MemoryMonitor monitor = new MemoryMonitor(); + monitor.start(); + //monitor.setMonitoring(true); + savedOutput = new PrintWriter(sw); + long start = System.currentTimeMillis(); + buildClassifier(); + long end = System.currentTimeMillis(); + System.gc(); + savedOutput.printf("Training Runtime: %dms\n", (end - start)); + monitor.setMonitoring(true); + start = System.currentTimeMillis(); + testClassifier(); + end = System.currentTimeMillis(); + monitor.setMonitoring(false); + System.gc(); + savedOutput.printf("Testing Runtime: %dms\n", (end - start)); + showOutput(); + monitor.shutdown(); + // no need to empty the filesystem with maven, since we no longer pollute the same tree. + //emptyFilesystem(); + + // if this test takes longer than twenty seconds, something's wrong. We're running slow. + assert (end - start) < 20000L; + } + + private static void showOutput() { + System.err.println(sw.getBuffer().toString()); + } + + @SuppressWarnings("unused") + private void emptyFilesystem() throws IOException { + processFiles(new Manager() { + public void handleFile(Command c, String type, File corpus) throws IOException { + corpus.delete(); + } + }, new Command() { + public String getType() { + return "deleting"; + } + + public void execute(String s, String type) { + } + }, new Selector() { + public boolean accept(int i) { + return true; + } + }, false, true + ); + } + + static PrintWriter savedOutput; + static StringWriter sw = new StringWriter(); + + private void testClassifier() throws IOException { + + final int[] hits = new int[1]; + final int[] misses = new int[2]; + processFiles(new Manager() { + public void handleFile(Command c, String type, File corpus) throws IOException { + FileReader fr = new FileReader(corpus); + BufferedReader br = new BufferedReader(fr, 16384); + StringBuilder sb = new StringBuilder(); + String s; + while ((s = br.readLine()) != null) { + sb.append(s); + } + br.close(); + fr.close(); + c.execute(sb.toString(), type); + } + }, new Command() { + boolean normalized = false; + + public String getType() { + return "testing"; + } + + public void execute(String s, String type) { + String f = classifier.getClassification(s, "unknown"); + if (!f.equals(type)) { + System.out.println("should have gotten " + type + ", got " + f); + if (f.equals("unknown")) { + misses[1]++; + } else { + misses[0]++; + } + } else { + if (normalized == false) { + normalized = true; + ClassifierProbability[] probs = classifier.getProbabilities(s); + savedOutput.println("probabilities: "); + for (ClassifierProbability prob : probs) { + savedOutput.println(prob.toString()); + } + } + hits[0]++; + } + } + }, new Selector() { + public boolean accept(int i) { + return (i % 10) > 7; + } + }, false + ); + savedOutput.println("Hits: " + hits[0]); + savedOutput.println("Misses: " + (misses[0] + misses[1])); + savedOutput.println("Wrong classifications: " + misses[0]); + savedOutput.println("Unclear classifications: " + misses[1]); + savedOutput.println("Totals: " + (hits[0] + misses[0] + misses[1]) + " items"); + savedOutput.println("Hit rate: " + ((1.0 * hits[0]) / (hits[0] + misses[0] + misses[1])) + "%"); + } + + private void buildClassifier() throws IOException { + processFiles(new Manager() { + public void handleFile(Command c, String type, File corpus) throws IOException { + FileReader fr = new FileReader(corpus); + BufferedReader br = new BufferedReader(fr); + StringBuilder sb = new StringBuilder(); + String s; + while ((s = br.readLine()) != null) { + sb.append(s); + } + br.close(); + fr.close(); + c.execute(sb.toString(), type); + } + }, new Command() { + public String getType() { + return "training"; + } + + public void execute(String s, String type) { + classifier.train(s, type); + } + }, new Selector() { + public boolean accept(int i) { + return (i % 10) <= 7; + } + }, true, true + ); + } + + void processFiles(final Manager m, Command c, final Selector s, boolean expand) throws IOException { + processFiles(m, c, s, expand, false); + } + + void processFiles(final Manager m, final Command c, final Selector s, final boolean expand, boolean multithread) throws IOException { + + File file = new File(System.getProperty("user.dir") + "/target/test-classes/training"); + final File[] files = file.listFiles(new FilenameFilter() { + public boolean accept(File dir, String name) { + if (name.indexOf("bz2") != -1) { + if (name.indexOf("data") == -1) { + return true; + } + } + return false; + } + }); + ThreadGroup g = new ThreadGroup("CORPUS GROUP"); + if (expand) { + for (final File f : files) { + Thread e1 = new Thread(g, "Expando " + f.getName()) { + public void run() { + try { + final File x = new File(f.toString() + ".data"); + x.mkdir(); + + FileInputStream fis = new FileInputStream(f); + BufferedInputStream bis = new BufferedInputStream(fis); + //noinspection ResultOfMethodCallIgnored + bis.read(new byte[2]); + CBZip2InputStream bzip2 = new CBZip2InputStream(bis); + TarArchive archive = new TarArchive(bzip2); + archive.extractContents(x); + System.out.printf("%s finished expanding.\n", f.getName()); + } catch (IOException ioe) { + ioe.printStackTrace(); + } + } + }; + e1.start(); + } + while (g.activeCount() > 0) { + try { + Thread.sleep(2000); + System.out.printf("%s waiting for expansion to finish: %d threads\n", new Date(), g.activeCount()); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + + } + + for (final File f : files) { + Thread t = new Thread(g, f.getName()) { + public void run() { + try { + System.out.println(f); + final File x = new File(f.toString() + ".data"); + + System.out.printf("%s...%s!\n", f.getName(), c.getType()); + String type = f.toString().indexOf("ham") == -1 ? "spam" : "ham"; + int t = 0; + for (File dir : x.listFiles(new FilenameFilter() { + public boolean accept(File dir, String name) { + return name.indexOf("data") == -1; + } + })) { + for (File corpus : dir.listFiles()) { + if (s.accept(t)) { + m.handleFile(c, type, corpus); + } + if (++t % 100 == 0) { + System.out.printf("%s: %d mark\n", c.getType(), t); + } + } + } + } catch (IOException e) { + e.printStackTrace(); + } + } + }; + if (multithread) { + t.start(); + } else { + t.run(); + } + } + if (multithread) { + while (g.activeCount() > 0) { + try { + Thread.sleep(2000); + System.out.printf("%s: Waiting for %d threads to finish...\n", new Date().toString(), g.activeCount()); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + } + +} diff --git a/ci-bayes/src/test/resources/jdbc-context.xml b/ci-bayes/src/test/resources/jdbc-context.xml new file mode 100755 index 0000000..2030f14 --- /dev/null +++ b/ci-bayes/src/test/resources/jdbc-context.xml @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ci-bayes/src/test/resources/jdbc.properties b/ci-bayes/src/test/resources/jdbc.properties new file mode 100755 index 0000000..e6a43a4 --- /dev/null +++ b/ci-bayes/src/test/resources/jdbc.properties @@ -0,0 +1,4 @@ +jdbc.driverClassName=org.hsqldb.jdbcDriver +jdbc.username=sa +jdbc.password= +jdbc.url=jdbc:hsqldb:file:testDB \ No newline at end of file diff --git a/common/pom.xml b/common/pom.xml new file mode 100755 index 0000000..2a84436 --- /dev/null +++ b/common/pom.xml @@ -0,0 +1,49 @@ + + + 4.0.0 + + ml-parent + com.enigmastation.ml + 4.0-SNAPSHOT + + com.enigmastation.ml + common + 4.0-SNAPSHOT + common + http://maven.apache.org + + UTF-8 + + + + + org.testng + testng + 6.3.1 + test + + + com.github.stephenc.high-scale-lib + high-scale-lib + 1.1.4 + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 2.3.2 + + 1.7 + 1.7 + true + true + + + + + diff --git a/common/src/main/java/com/enigmastation/ml/App.java b/common/src/main/java/com/enigmastation/ml/App.java new file mode 100755 index 0000000..e93781b --- /dev/null +++ b/common/src/main/java/com/enigmastation/ml/App.java @@ -0,0 +1,13 @@ +package com.enigmastation.ml; + +/** + * Hello world! + * + */ +public class App +{ + public static void main( String[] args ) + { + System.out.println( "Hello World!" ); + } +} diff --git a/common/src/main/java/com/enigmastation/ml/common/collections/ConstantValueProvider.java b/common/src/main/java/com/enigmastation/ml/common/collections/ConstantValueProvider.java new file mode 100755 index 0000000..9881e90 --- /dev/null +++ b/common/src/main/java/com/enigmastation/ml/common/collections/ConstantValueProvider.java @@ -0,0 +1,16 @@ +package com.enigmastation.ml.common.collections; + +import java.io.Serializable; + +public class ConstantValueProvider implements ValueProvider { + final V defaultValue; + + public ConstantValueProvider(V defaultValue) { + this.defaultValue = defaultValue; + } + + @Override + public V getDefault(Object k) { + return defaultValue; + } +} diff --git a/common/src/main/java/com/enigmastation/ml/common/collections/ConsumingMapDataStore.java b/common/src/main/java/com/enigmastation/ml/common/collections/ConsumingMapDataStore.java new file mode 100755 index 0000000..942ef47 --- /dev/null +++ b/common/src/main/java/com/enigmastation/ml/common/collections/ConsumingMapDataStore.java @@ -0,0 +1,10 @@ +package com.enigmastation.ml.common.collections; + +import java.util.Map; + +public class ConsumingMapDataStore extends MapDataStore { + @Override + protected void persistEntry(Map.Entry pair) { + // discards pair + } +} diff --git a/common/src/main/java/com/enigmastation/ml/common/collections/MapBuilder.java b/common/src/main/java/com/enigmastation/ml/common/collections/MapBuilder.java new file mode 100755 index 0000000..c62cd0c --- /dev/null +++ b/common/src/main/java/com/enigmastation/ml/common/collections/MapBuilder.java @@ -0,0 +1,60 @@ +package com.enigmastation.ml.common.collections; + +import java.util.Map; + +public class MapBuilder { + private ValueProvider valueProvider=null; + private int syncBlockSize = -1; + private MapDataStore dataStore=null; + + public MapBuilder dataStore(MapDataStore dataStore) { + if (this.dataStore != null) { + throw new IllegalStateException("dataStore already set"); + } + this.dataStore = dataStore; + return this; + } + + public MapBuilder syncBlockSize(int size) { + if (syncBlockSize != -1) { + throw new IllegalStateException("syncBlockSize already set"); + } + this.syncBlockSize = size; + return this; + } + + public MapBuilder defaultValue(V defaultValue) { + checkValueProviderState(); + this.valueProvider = new ConstantValueProvider(defaultValue); + return this; + } + + public MapBuilder valueProvider(ValueProvider valueProvider) { + checkValueProviderState(); + this.valueProvider = valueProvider; + return this; + } + + private void checkValueProviderState() { + if (valueProvider != null) { + throw new IllegalStateException("valueProvider already set"); + } + } + + public Map build() { + if (valueProvider == null) { + valueProvider = new ConstantValueProvider(null); + } + //noinspection unchecked + MapWithDefaultValue map = new MapWithDefaultValue((ValueProvider) valueProvider); + if (syncBlockSize != -1) { + map.setSyncBlockSize(syncBlockSize); + } + if (dataStore == null) { + dataStore = new ConsumingMapDataStore<>(); + } + //noinspection unchecked + map.setDataStore((MapDataStore) dataStore); + return map; + } +} diff --git a/common/src/main/java/com/enigmastation/ml/common/collections/MapDataStore.java b/common/src/main/java/com/enigmastation/ml/common/collections/MapDataStore.java new file mode 100755 index 0000000..6ed7434 --- /dev/null +++ b/common/src/main/java/com/enigmastation/ml/common/collections/MapDataStore.java @@ -0,0 +1,17 @@ +package com.enigmastation.ml.common.collections; + +import java.util.Iterator; +import java.util.Map; + +public abstract class MapDataStore { + public void persist(Map updates) { + Iterator it = updates.entrySet().iterator(); + while (it.hasNext()) { + Map.Entry pair = (Map.Entry) it.next(); + persistEntry(pair); + it.remove(); + } + } + + protected abstract void persistEntry(Map.Entry pair); +} diff --git a/common/src/main/java/com/enigmastation/ml/common/collections/MapWithDefaultValue.java b/common/src/main/java/com/enigmastation/ml/common/collections/MapWithDefaultValue.java new file mode 100755 index 0000000..d5afb3e --- /dev/null +++ b/common/src/main/java/com/enigmastation/ml/common/collections/MapWithDefaultValue.java @@ -0,0 +1,64 @@ +package com.enigmastation.ml.common.collections; + +import org.cliffc.high_scale_lib.NonBlockingHashMap; + +import java.util.Map; + +public class MapWithDefaultValue +extends NonBlockingHashMap { + ValueProvider valueProvider; + Map updates = new NonBlockingHashMap<>(); + int syncBlockSize = 0; + MapDataStore dataStore; + + public int getSyncBlockSize() { + return syncBlockSize; + } + + public void setSyncBlockSize(int syncBlockSize) { + this.syncBlockSize = syncBlockSize; + checkForSync(); + } + + public MapDataStore getDataStore() { + return dataStore; + } + + public void setDataStore(MapDataStore dataStore) { + this.dataStore = dataStore; + } + + public MapWithDefaultValue(ValueProvider valueProvider) { + this.valueProvider = valueProvider; + } + + public MapWithDefaultValue(int initial_sz, ValueProvider valueProvider) { + super(initial_sz); + this.valueProvider = valueProvider; + } + + public V get(Object key) { + V object = super.get(key); + if (object == null) { + object = valueProvider.getDefault(key); + } + return object; + } + + @Override + public V put(K key, V val) { + updates.put(key, val); + checkForSync(); + return super.put(key, val); + } + + private void checkForSync() { + if (updates.size() > syncBlockSize) { + synchronize(); + } + } + + public void synchronize() { + dataStore.persist(updates); + } +} diff --git a/common/src/main/java/com/enigmastation/ml/common/collections/ValueProvider.java b/common/src/main/java/com/enigmastation/ml/common/collections/ValueProvider.java new file mode 100755 index 0000000..a453640 --- /dev/null +++ b/common/src/main/java/com/enigmastation/ml/common/collections/ValueProvider.java @@ -0,0 +1,5 @@ +package com.enigmastation.ml.common.collections; + +public interface ValueProvider { + V getDefault(Object k); +} diff --git a/common/src/test/java/com/enigmastation/ml/common/collections/MapTest.java b/common/src/test/java/com/enigmastation/ml/common/collections/MapTest.java new file mode 100755 index 0000000..a2b2f94 --- /dev/null +++ b/common/src/test/java/com/enigmastation/ml/common/collections/MapTest.java @@ -0,0 +1,89 @@ +package com.enigmastation.ml.common.collections; + +import org.testng.annotations.Test; + +import java.util.HashMap; +import java.util.Map; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNull; + +public class MapTest { + @Test + public void testBasicMap() { + Map map = new MapBuilder().build(); + assertNull(map.get("foo")); + map.put("foo", 1); + assertEquals(map.get("foo"), new Integer(1)); + } + + @Test + public void testMapProvidedValue() { + Map map = new MapBuilder() + .valueProvider(new ValueProvider() { + @Override + public String getDefault(Object k) { + String key = k.toString(); + StringBuilder sb = new StringBuilder(); + for (char c : key.toCharArray()) { + sb.insert(0, c); + + } + return sb.toString(); + } + }).build(); + map.put("bar", "baz"); + assertEquals(map.get("foo"), "oof"); + assertEquals(map.get("foo"), "oof"); + assertEquals(map.get("bar"), "baz"); + } + + @Test + public void testMapSync() { + final Map updatedValues = new HashMap(); + Map map = new MapBuilder() + .dataStore(new MapDataStore() { + @Override + protected void persistEntry(Map.Entry pair) { + System.out.println(pair); + updatedValues.put(pair.getKey(), pair.getValue()); + } + }) + .build(); + map.put("foo", "bar"); + assertEquals(updatedValues.get("foo"), "bar"); + } + + @Test + public void testMapSyncWithBlockSize() { + final Map updatedValues = new HashMap(); + System.out.println("---------------"); + Map map = new MapBuilder() + .syncBlockSize(4) + .dataStore(new MapDataStore() { + @Override + protected void persistEntry(Map.Entry pair) { + System.out.println(pair); + updatedValues.put(pair.getKey(), pair.getValue()); + } + }) + .build(); + map.put("foo", "bar"); + assertNull(updatedValues.get("foo")); + for (int i = 0; i < 5; i++) { + map.put(Integer.toString(i), Integer.toString(i)); + } + assertEquals(updatedValues.get("foo"), "bar"); + assertEquals(updatedValues.get("3"), "3"); + // why null? Because the map is discarded before the sync is run. + // how to address? + // probably a finalizer. + assertNull(updatedValues.get("4")); + } + + @Test + public void testMapWithDouble() { + Map map = new MapBuilder().defaultValue(1.0).build(); + assertEquals(map.get("foo"), 1.0, 0.01); + } +} diff --git a/jdbc-dao/src/main/java/com/enigmastation/dao/jdbc/NeuronDAOImpl.java b/jdbc-dao/src/main/java/com/enigmastation/dao/jdbc/NeuronDAOImpl.java new file mode 100755 index 0000000..05eaff4 --- /dev/null +++ b/jdbc-dao/src/main/java/com/enigmastation/dao/jdbc/NeuronDAOImpl.java @@ -0,0 +1,124 @@ +package com.enigmastation.dao.jdbc; + +import com.enigmastation.dao.CategoryDAO; +import com.enigmastation.dao.impl.AbstractBaseDAO; +import com.enigmastation.dao.model.Category; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.RowCallbackHandler; +import org.springframework.stereotype.Repository; +import org.springframework.transaction.annotation.Transactional; + +import javax.annotation.PostConstruct; +import javax.sql.DataSource; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +/** + * User: joeo + * Date: 11/23/10 + * Time: 3:18 PM + *

+ * Copyright + */ +@Repository +public class NeuronDAOImpl extends AbstractBaseDAO implements CategoryDAO { + @Autowired + DataSource dataSource; + + JdbcTemplate template; + + @PostConstruct + public void init() { + template = new JdbcTemplate(dataSource); + try { + template.execute("create table category (id varchar primary key, createtime bigint, updatetime bigint, category varchar, _count bigint)"); + } catch (Exception ignored) { + } + try { + template.execute("create unique index cat_category on category(category)"); + } catch (Exception ignored) { + } + } + + @Transactional + public Category read(Category template) { + List categories = readMultiple(template); + if (categories.size() > 0) { + return categories.get(0); + } + return null; + } + + @Transactional + public Category write(Category object) { + boolean attemptUpdate = true; + int updates = 0; + if (object.getId() == null) { + object.setId(UUID.randomUUID().toString()); + object.setCreateTime(System.nanoTime()); + attemptUpdate = false; + } + object.setLastUpdateTime(System.nanoTime()); + if (attemptUpdate) { + updates = this.template.update("update category set createtime=?, updatetime=?, category=?, _count=? where id=?", + object.getCreateTime(), object.getLastUpdateTime(), + object.getCategory(), object.getCount(), + object.getId()); + } + if (updates == 0) { + this.template.update("insert into category (createtime, updatetime, category, _count, id) " + + "values (?,?,?,?,?)", object.getCreateTime(), object.getLastUpdateTime(), + object.getCategory(), object.getCount(), + object.getId()); + } + return object; + } + + @Transactional + public Category take(Category template) { + Category category = read(template); + remove(category); + return category; + } + + @Transactional + public List readMultiple(Category template) { + final List list = new ArrayList(); + this.template.query("select id, createtime, updatetime, category, _count from category "+ + "where id like ? and category like ?", + new Object[]{ + template.getId() == null ? "%" : template.getId(), + template.getCategory() == null ? "%" : template.getCategory(), + }, new RowCallbackHandler() { + public void processRow(ResultSet resultSet) throws SQLException { + Category category = new Category(); + category.setId(resultSet.getString(1)); + category.setCreateTime(resultSet.getLong(2)); + category.setLastUpdateTime(resultSet.getLong(3)); + category.setCategory(resultSet.getString(4)); + category.setCount(resultSet.getLong(5)); + list.add(category); + } + }); + return list; + } + + @Transactional + public List takeMultiple(Category template) { + List categories = readMultiple(template); + for (Category category : categories) { + remove(category); + } + return categories; + } + + @Transactional + private void remove(final Category category) { + //noinspection unchecked + this.template.update("delete from category where id=?", category.getId()); + } +} diff --git a/jdbc-dao/src/main/java/com/enigmastation/dao/jdbc/SynapseDAOImpl.java b/jdbc-dao/src/main/java/com/enigmastation/dao/jdbc/SynapseDAOImpl.java new file mode 100755 index 0000000..fe732ca --- /dev/null +++ b/jdbc-dao/src/main/java/com/enigmastation/dao/jdbc/SynapseDAOImpl.java @@ -0,0 +1,121 @@ +package com.enigmastation.dao.jdbc; + +import com.enigmastation.dao.NeuronDAO; +import com.enigmastation.dao.impl.AbstractBaseDAO; +import com.enigmastation.dao.model.Neuron; +import com.enigmastation.dao.model.Visibility; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.RowCallbackHandler; +import org.springframework.stereotype.Repository; +import org.springframework.transaction.annotation.Transactional; + +import javax.annotation.PostConstruct; +import javax.sql.DataSource; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +/** + * User: joeo + * Date: 11/23/10 + * Time: 3:18 PM + *

+ * Copyright + */ +@Repository +public class SynapseDAOImpl extends AbstractBaseDAO implements NeuronDAO { + @Autowired + DataSource dataSource; + + JdbcTemplate template; + + @PostConstruct + public void init() { + template = new JdbcTemplate(dataSource); + try { + template.execute("create table category (id varchar primary key, createtime bigint, updatetime bigint, payload varchar, visibility number)"); + } catch (Exception ignored) { + } + } + + @Transactional + public Neuron read(Neuron template) { + List categories = readMultiple(template); + if (categories.size() > 0) { + return categories.get(0); + } + return null; + } + + @Transactional + public Neuron write(Neuron object) { + boolean attemptUpdate = true; + int updates = 0; + if (object.getId() == null) { + object.setId(UUID.randomUUID().toString()); + object.setCreateTime(System.nanoTime()); + attemptUpdate = false; + } + object.setLastUpdateTime(System.nanoTime()); + if (attemptUpdate) { + updates = this.template.update("update neuron set createtime=?, updatetime=?, payload=?, visibility=? where id=?", + object.getCreateTime(), object.getLastUpdateTime(), + object.getPayload(), object.getVisibility().getStrength(), + object.getId()); + } + if (updates == 0) { + this.template.update("insert into category (createtime, updatetime, payload, visibility, id) " + + "values (?,?,?,?,?)", object.getCreateTime(), object.getLastUpdateTime(), + object.getPayload(), object.getVisibility(), + object.getId()); + } + return object; + } + + @Transactional + public Neuron take(Neuron template) { + Neuron neuron = read(template); + remove(neuron); + return neuron; + } + + @Transactional + public List readMultiple(Neuron template) { + final List list = new ArrayList(); + this.template.query("select id, createtime, updatetime, payload, visibility from category "+ + "where id like ? and payload like ?", + new Object[]{ + template.getId() == null ? "%" : template.getId(), + template.getPayload() == null ? "%" : template.getPayload(), + }, new RowCallbackHandler() { + public void processRow(ResultSet resultSet) throws SQLException { + Neuron category = new Neuron(); + category.setId(resultSet.getString(1)); + category.setCreateTime(resultSet.getLong(2)); + category.setLastUpdateTime(resultSet.getLong(3)); + category.setPayload(resultSet.getString(4)); + category.setVisibility(resultSet.getDouble(5) == Visibility.VISIBLE.getStrength()?Visibility.VISIBLE:Visibility.INVISIBLE); + list.add(category); + } + }); + return list; + } + + @Transactional + public List takeMultiple(Neuron template) { + List neurons = readMultiple(template); + for (Neuron category : neurons) { + remove(category); + } + return neurons; + } + + @Transactional + private void remove(final Neuron category) { + //noinspection unchecked + this.template.update("delete from neuron where id=?", category.getId()); + } +} diff --git a/objectify-dao/pom.xml b/objectify-dao/pom.xml new file mode 100755 index 0000000..f9f4aaf --- /dev/null +++ b/objectify-dao/pom.xml @@ -0,0 +1,44 @@ + + + 4.0.0 + + ci-bayes-parent + com.enigmastation.ci-bayes + 3.0.3 + + com.enigmastation.ci-bayes.dao + objectify-dao + 3.0.3 + objectify-dao + http://maven.apache.org + + UTF-8 + + + + org.testng + testng + 6.1.1 + test + + + com.enigmastation.ci-bayes + model + ${project.version} + + + + + + maven-compiler-plugin + 2.1 + + 1.5 + 1.5 + + + + + diff --git a/objectify-dao/src/main/java/com/enigmastation/dao/BaseEntity.java b/objectify-dao/src/main/java/com/enigmastation/dao/BaseEntity.java new file mode 100755 index 0000000..10da819 --- /dev/null +++ b/objectify-dao/src/main/java/com/enigmastation/dao/BaseEntity.java @@ -0,0 +1,11 @@ +package com.enigmastation.dao; + +/** + * Created by IntelliJ IDEA. + * User: joeo + * Date: 1/19/12 + * Time: 10:18 AM + * To change this template use File | Settings | File Templates. + */ +public class BaseEntity { +} diff --git a/objectify-dao/src/main/java/com/enigmastation/dao/objectify/CategoryDAOImpl.java b/objectify-dao/src/main/java/com/enigmastation/dao/objectify/CategoryDAOImpl.java new file mode 100755 index 0000000..012274d --- /dev/null +++ b/objectify-dao/src/main/java/com/enigmastation/dao/objectify/CategoryDAOImpl.java @@ -0,0 +1,11 @@ +package com.enigmastation.dao.objectify; + +/** + * Created by IntelliJ IDEA. + * User: joeo + * Date: 1/19/12 + * Time: 9:00 AM + * To change this template use File | Settings | File Templates. + */ +public class CategoryDAOImpl { +} diff --git a/objectify-dao/src/main/java/com/enigmastation/dao/objectify/FeatureDAOImpl.java b/objectify-dao/src/main/java/com/enigmastation/dao/objectify/FeatureDAOImpl.java new file mode 100755 index 0000000..2c0be1d --- /dev/null +++ b/objectify-dao/src/main/java/com/enigmastation/dao/objectify/FeatureDAOImpl.java @@ -0,0 +1,16 @@ +package com.enigmastation.dao.objectify; + +import com.enigmastation.dao.FeatureDAO; +import com.enigmastation.dao.model.Feature; +import org.springframework.stereotype.Repository; + +/** + * User: joeo + * Date: 11/23/10 + * Time: 3:18 PM + *

+ * Copyright + */ +@Repository +public class FeatureDAOImpl extends Db4OBaseDAO implements FeatureDAO { +} diff --git a/objectify-dao/src/main/java/com/enigmastation/dao/objectify/NeuronDAOImpl.java b/objectify-dao/src/main/java/com/enigmastation/dao/objectify/NeuronDAOImpl.java new file mode 100755 index 0000000..65abdff --- /dev/null +++ b/objectify-dao/src/main/java/com/enigmastation/dao/objectify/NeuronDAOImpl.java @@ -0,0 +1,16 @@ +package com.enigmastation.dao.objectify; + +import com.enigmastation.dao.NeuronDAO; +import com.enigmastation.dao.model.Neuron; +import org.springframework.stereotype.Repository; + +/** + * User: joeo + * Date: 11/5/10 + * Time: 7:03 AM + *

+ * Copyright + */ +@Repository +public class NeuronDAOImpl extends Db4OBaseDAO implements NeuronDAO { +} diff --git a/objectify-dao/src/main/java/com/enigmastation/dao/objectify/ObjectifyBaseDAO.java b/objectify-dao/src/main/java/com/enigmastation/dao/objectify/ObjectifyBaseDAO.java new file mode 100755 index 0000000..730b8b2 --- /dev/null +++ b/objectify-dao/src/main/java/com/enigmastation/dao/objectify/ObjectifyBaseDAO.java @@ -0,0 +1,11 @@ +package com.enigmastation.dao.objectify; + +/** + * Created by IntelliJ IDEA. + * User: joeo + * Date: 1/19/12 + * Time: 8:59 AM + * To change this template use File | Settings | File Templates. + */ +public class ObjectifyBaseDAO { +} diff --git a/objectify-dao/src/main/java/com/enigmastation/dao/objectify/SynapseDAOImpl.java b/objectify-dao/src/main/java/com/enigmastation/dao/objectify/SynapseDAOImpl.java new file mode 100755 index 0000000..4dca776 --- /dev/null +++ b/objectify-dao/src/main/java/com/enigmastation/dao/objectify/SynapseDAOImpl.java @@ -0,0 +1,16 @@ +package com.enigmastation.dao.objectify; + +import com.enigmastation.dao.SynapseDAO; +import com.enigmastation.dao.model.Synapse; +import org.springframework.stereotype.Repository; + +/** + * User: joeo + * Date: 11/5/10 + * Time: 7:03 AM + *

+ * Copyright + */ +@Repository +public class SynapseDAOImpl extends Db4OBaseDAO implements SynapseDAO { +} diff --git a/objectify-dao/src/test/java/com/enigmastation/dao/DAOTest.java b/objectify-dao/src/test/java/com/enigmastation/dao/DAOTest.java new file mode 100755 index 0000000..ff6b29a --- /dev/null +++ b/objectify-dao/src/test/java/com/enigmastation/dao/DAOTest.java @@ -0,0 +1,11 @@ +package com.enigmastation.dao; + +/** + * Created by IntelliJ IDEA. + * User: joeo + * Date: 1/19/12 + * Time: 10:09 AM + * To change this template use File | Settings | File Templates. + */ +public class DAOTest { +} diff --git a/objectify-dao/src/test/java/com/enigmastation/dao/TestEntity.java b/objectify-dao/src/test/java/com/enigmastation/dao/TestEntity.java new file mode 100755 index 0000000..ebb2511 --- /dev/null +++ b/objectify-dao/src/test/java/com/enigmastation/dao/TestEntity.java @@ -0,0 +1,11 @@ +package com.enigmastation.dao; + +/** + * Created by IntelliJ IDEA. + * User: joeo + * Date: 1/19/12 + * Time: 10:08 AM + * To change this template use File | Settings | File Templates. + */ +public class TestEntity { +} diff --git a/objectify-dao/src/test/java/com/enigmastation/dao/TestEntityDAO.java b/objectify-dao/src/test/java/com/enigmastation/dao/TestEntityDAO.java new file mode 100755 index 0000000..86ec324 --- /dev/null +++ b/objectify-dao/src/test/java/com/enigmastation/dao/TestEntityDAO.java @@ -0,0 +1,11 @@ +package com.enigmastation.dao; + +/** + * Created by IntelliJ IDEA. + * User: joeo + * Date: 1/19/12 + * Time: 10:08 AM + * To change this template use File | Settings | File Templates. + */ +public class TestEntityDAO { +}