Bog and messy(

Author: GreenWizard2015

src/Tests/Test_BasicHashingGenerator.java

@@ -0,0 +1,34 @@
+package Tests;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import YASL.Hashing.ChgCombined;
+import YASL.Hashing.IHasher;
+
+public class Test_BasicHashingGenerator {
+	@Test
+	public void uniqueSequence() {
+		ChgCombined<Integer> gen = new ChgCombined<Integer>( //
+		    (x, r) -> ~x * 23 * r, //
+		    x -> x, x -> x >> 16, x -> ~x //
+		);
+		final IHasher<Integer> hasher = gen.generate(5000, 3);
+		long notUnique = 0;
+		for (int i = 0; i < 1_000_000; i++) {
+			final int[] res = hasher.apply(i);
+			final Set<Integer> values = new HashSet<>();
+			for (int j = 0; j < res.length; j++) {
+				if (!values.add(res[j])) {
+					notUnique++;
+					break;
+				}
+			}
+		}
+		Assert.assertEquals(1, notUnique);
+
+	}
+}

src/Tests/Test_CountMinCounter.java

@@ -0,0 +1,33 @@
+package Tests;
+
+import java.util.Random;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import YASL.CExactCounter;
+import YASL.IItemsCounter;
+import YASL.Counters.CCMCounter;
+import YASL.Counters.CCountMinParams;
+import YASL.Hashing.ChgBasic;
+
+public class Test_CountMinCounter {
+	@Test
+	public void notLessThanExact() {
+		final int variations = 10000;
+		final Random r = new Random(0);
+		final IItemsCounter<Long> exact = new CExactCounter<>();
+		final IItemsCounter<Long> MC = new CCMCounter<Long>( //
+		    new CCountMinParams(0.01, 0.01), //
+		    new ChgBasic<>() //
+		);
+
+		for (long i = 0; i < 100_000; i++) {
+			final long value = r.nextInt(variations);
+			final int cnt = 1 + r.nextInt(10);
+			final long exactCount = exact.put(value, cnt);
+			final long minCount = MC.put(value, cnt);
+			Assert.assertTrue(exactCount <= minCount);
+		}
+	}
+}

src/Tests/Test_Estimator.java

@@ -4,9 +4,9 @@
 import org.junit.Test;
 
 import YASL.CEstimator;
-import YASL.CExactCollector;
 import YASL.CExactCounter;
 import YASL.IEstimator;
+import YASL.Collectors.CExactCollector;
 
 public class Test_Estimator {
 	@Test

src/Tests/Test_KTopCollector.java

@@ -4,7 +4,7 @@
 import org.junit.Test;
 
 import YASL.IEstimationCollector;
-import YASL.KTopCollector;
+import YASL.Collectors.KTopCollector;
 
 public class Test_KTopCollector {
 

src/Tests/Test_hgCombined.java

@@ -0,0 +1,61 @@
+package Tests;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import YASL.Hashing.ChgCombined;
+import YASL.Hashing.IHasher;
+
+public class Test_hgCombined {
+	@Test
+	public void returnHashesFirst() {
+		IHasher<Integer> gen = new ChgCombined<Integer>( //
+		    x -> 1, x -> 2 //
+		).generate(Integer.MAX_VALUE, 2);
+
+		Assert.assertArrayEquals( //
+		    new int[] { 1, 2 }, //
+		    gen.apply(1) //
+		);
+	}
+
+	@Test
+	public void respectRange() {
+		IHasher<Integer> gen = new ChgCombined<Integer>( //
+		    x -> 21, x -> -24//
+		).generate(5, 2);
+
+		Assert.assertArrayEquals( //
+		    new int[] { 1, 4 }, //
+		    gen.apply(1) //
+		);
+	}
+
+	@Test
+	public void useOnlyWhatNeed() {
+		IHasher<Integer> gen = new ChgCombined<Integer>( //
+		    x -> 1, //
+		    x -> {
+			    throw new RuntimeException("Don't touch me!");
+		    }//
+		).generate(5, 1);
+
+		Assert.assertArrayEquals( //
+		    new int[] { 1 }, //
+		    gen.apply(1) //
+		);
+	}
+
+	@Test
+	public void applySalting() {
+		IHasher<Integer> gen = new ChgCombined<Integer>( //
+		    (x, r) -> x * r * 4, //
+		    x -> x, x -> x + 1 //
+		).generate(Integer.MAX_VALUE, 5);
+
+		Assert.assertArrayEquals( //
+		    new int[] { 1, 2, 4, 5, 8 }, //
+		    gen.apply(1) //
+		);
+	}
+}

src/YASL/CExactCollector.java renamed to src/YASL/Collectors/CExactCollector.java

@@ -1,9 +1,13 @@
-package YASL;
+package YASL.Collectors;
 
 import java.util.HashMap;
 import java.util.Map;
 import java.util.stream.Collectors;
 
+import YASL.CEstimatedItems;
+import YASL.CEstimationFor;
+import YASL.IEstimationCollector;
+
 public class CExactCollector<T> implements IEstimationCollector<T> {
 
 	private Map<T, Long> _storage = new HashMap<>();

src/YASL/KTopCollector.java renamed to src/YASL/Collectors/KTopCollector.java

@@ -1,9 +1,13 @@
-package YASL;
+package YASL.Collectors;
 
 import java.util.Comparator;
 import java.util.PriorityQueue;
 import java.util.stream.Collectors;
 
+import YASL.CEstimatedItems;
+import YASL.CEstimationFor;
+import YASL.IEstimationCollector;
+
 public class KTopCollector<T> implements IEstimationCollector<T> {
 	private final long															_K;
 	private final PriorityQueue<CEstimationFor<T>>	_queue;

src/YASL/Counters/CCMCounter.java

@@ -0,0 +1,61 @@
+package YASL.Counters;
+
+import YASL.IItemsCounter;
+import YASL.Hashing.IHasher;
+import YASL.Hashing.IHashingGenerator;
+
+public class CCMCounter<T> implements IItemsCounter<T> {
+	private final IHasher<T>		_bucketsProvider;
+	protected final long[][]	_sketch;
+
+	public CCMCounter( //
+	    int width, int buckets, //
+	    IHasher<T> bucketsProvider//
+	) {
+		_bucketsProvider = bucketsProvider;
+		_sketch = new long[width][buckets];
+	}
+
+	public CCMCounter( //
+	    CCountMinParams params, //
+	    IHashingGenerator<T> hGen //
+	) {
+		this( //
+		    params.width, params.depth, //
+		    hGen.generate(params.width, params.depth) //
+		);
+	}
+
+	@Override
+	public long put(T item, long count) {
+		final int[] positions = _bucketsProvider.apply(item);
+		count += count(positions);
+		update(positions, count);
+		return count;
+	}
+
+	public void update(T item, long count) {
+		update(_bucketsProvider.apply(item), count);
+	}
+
+	protected void update(int[] positions, long count) {
+		for (int i = 0; i < positions.length; i++) {
+			final int pos = positions[i];
+			if (_sketch[pos][i] < count)
+				_sketch[pos][i] = count;
+		}
+	}
+
+	public long count(T item) {
+		return count(_bucketsProvider.apply(item));
+	}
+
+	private long count(int[] positions) {
+		long res = Long.MAX_VALUE;
+		for (int i = 0; i < positions.length; i++) {
+			final long cnt = _sketch[positions[i]][i];
+			res = (cnt < res) ? cnt : res;
+		}
+		return res;
+	}
+}

src/YASL/Counters/CCountMinParams.java

@@ -0,0 +1,15 @@
+package YASL.Counters;
+
+public class CCountMinParams {
+	public final int	width;
+	public final int	depth;
+
+	public CCountMinParams(double err, double confidence) {
+		// http://dimacs.rutgers.edu/~graham/pubs/papers/cmencyc.pdf
+		width = (int) Math.ceil(Math.E / err);
+		depth = (int) Math.ceil(Math.log(1 / (1 - confidence)));
+		// alternative
+		// width = (int) Math.ceil(2 * Math.E / err);
+		// depth = (int) Math.ceil(-Math.log(1 - confidence) / Math.log(2));
+	}
+}

src/YASL/Hashing/ChgBasic.java

@@ -0,0 +1,23 @@
+package YASL.Hashing;
+
+import java.util.Random;
+
+public class ChgBasic<T> implements IHashingGenerator<T> {
+
+	@Override
+	public IHasher<T> generate(int range, int levels) {
+		return x -> {
+			long hash = x.hashCode();
+			final Random r = new Random(hash);
+			final int[] res = new int[levels];
+			for (int i = 0; i < levels; i++) {
+				hash *= r.nextInt(range);
+				hash += r.nextLong();
+
+				res[i] = (int) (hash % range);
+				res[i] = (res[i] < 0) ? -res[i] : res[i];
+			}
+			return res;
+		};
+	}
+}

src/YASL/Hashing/ChgCombined.java

@@ -0,0 +1,53 @@
+package YASL.Hashing;
+
+public class ChgCombined<T> implements IHashingGenerator<T> {
+	public interface ISalter<T> {
+		public T apply(T value, int round);
+	}
+
+	private final IHashingAlgorithm<T>[]	_hashes;
+	private final ISalter<T>							_salting;
+
+	@SafeVarargs
+	public ChgCombined(ISalter<T> salter, IHashingAlgorithm<T>... hashes) {
+		_hashes = hashes;
+		_salting = salter;
+	}
+
+	@SafeVarargs
+	public ChgCombined(IHashingAlgorithm<T>... hashes) {
+		this( //
+		    (x, r) -> {
+			    throw new RuntimeException("No salter.");
+		    }, //
+		    hashes //
+		);
+	}
+
+	@Override
+	public IHasher<T> generate(int range, int levels) {
+		return x -> {
+			final int[] res = new int[levels];
+			int pos = populate(x, res, 0);
+
+			int round = 0;
+			while (pos < levels) {
+				round++;
+				pos = populate(_salting.apply(x, round), res, pos);
+			}
+
+			for (int i = 0; i < res.length; i++)
+				res[i] = Math.abs(res[i] % range);
+			return res;
+		};
+	}
+
+	private int populate(T x, int[] res, int pos) {
+		final int N = Math.min(_hashes.length, res.length - pos);
+		for (int i = 0; i < N; i++) {
+			res[pos] = _hashes[i].hash(x);
+			pos++;
+		}
+		return pos;
+	}
+}

src/YASL/Hashing/IHasher.java

@@ -0,0 +1,6 @@
+package YASL.Hashing;
+
+import java.util.function.Function;
+
+public interface IHasher<T> extends Function<T, int[]> {
+}

src/YASL/Hashing/IHashingAlgorithm.java

@@ -0,0 +1,5 @@
+package YASL.Hashing;
+
+public interface IHashingAlgorithm<T> {
+	public int hash(T value);
+}

src/YASL/Hashing/IHashingGenerator.java

@@ -0,0 +1,5 @@
+package YASL.Hashing;
+
+public interface IHashingGenerator<T> {
+	public IHasher<T> generate(int range, int levels);
+}