Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Merge branch 'master' of github.com:boundary/high-scale-lib

  • Loading branch information...
commit 695eea9c583b8d61de2cc038509a7027079ae43e 2 parents 4300aca + 0503e7e
@moonpolysoft moonpolysoft authored
Showing with 1,126 additions and 319 deletions.
  1. +0 −6 Testing/NBHS_Tester/nbhs_tester.log
  2. +0 −6 Testing/NBHS_Tester/nbsi_tester.log
  3. +78 −53 build.xml
  4. +3 −2 ivy.xml
  5. +23 −19 ivysettings.xml
  6. +2 −1  publish.properties
  7. +28 −0 src/main/java/com/boundary/BitPrint.java
  8. 0  src/{ → main/java}/java/util/Hashtable.java
  9. 0  src/{ → main/java}/java/util/concurrent/ConcurrentHashMap.java
  10. 0  src/{ → main/java}/org/cliffc/high_scale_lib/AbstractEntry.java
  11. 0  src/{ → main/java}/org/cliffc/high_scale_lib/ConcurrentAutoTable.java
  12. 0  src/{ → main/java}/org/cliffc/high_scale_lib/Counter.java
  13. +6 −0 src/main/java/org/cliffc/high_scale_lib/IntIterator.java
  14. 0  src/{ → main/java}/org/cliffc/high_scale_lib/NonBlockingHashMap.java
  15. 0  src/{ → main/java}/org/cliffc/high_scale_lib/NonBlockingHashMapLong.java
  16. 0  src/{ → main/java}/org/cliffc/high_scale_lib/NonBlockingHashSet.java
  17. 0  src/{ → main/java}/org/cliffc/high_scale_lib/NonBlockingHashtable.java
  18. 0  src/{ → main/java}/org/cliffc/high_scale_lib/NonBlockingIdentityHashMap.java
  19. +162 −26 src/{ → main/java}/org/cliffc/high_scale_lib/NonBlockingSetInt.java
  20. 0  src/{ → main/java}/org/cliffc/high_scale_lib/UtilUnsafe.java
  21. +13 −0 src/test/java/com/boundary/BitPrintTest.java
  22. +20 −0 src/test/java/org/cliffc/high_scale_lib/HighScaleLibTestSuite.java
  23. +3 −5 ...HM_Tester/NBHML_Tester2.java → src/test/java/org/cliffc/high_scale_lib/NonBlockingHashMapLongTest.java
  24. +3 −5 ...ng/NBHM_Tester/NBHM_Tester2.java → src/test/java/org/cliffc/high_scale_lib/NonBlockingHashMapTest.java
  25. +3 −5 ...ing/NBHS_Tester/nbhs_tester.java → src/test/java/org/cliffc/high_scale_lib/NonBlockingHashSetTest.java
  26. +551 −0 src/test/java/org/cliffc/high_scale_lib/NonBlockingIdentityHashMapTest.java
  27. +231 −191 Testing/NBHS_Tester/nbsi_tester.java → src/test/java/org/cliffc/high_scale_lib/NonBlockingSetIntTest.java
View
6 Testing/NBHS_Tester/nbhs_tester.log
@@ -1,6 +0,0 @@
-JUnit version 4.4
-....
-Time: 0.094
-
-OK (4 tests)
-
View
6 Testing/NBHS_Tester/nbsi_tester.log
@@ -1,6 +0,0 @@
-JUnit version 4.4
-.....
-Time: 0.312
-
-OK (5 tests)
-
View
131 build.xml
@@ -1,86 +1,111 @@
<?xml version="1.0" encoding="UTF-8"?>
<project default="main" name="high-scale-lib" xmlns:ivy="antlib:org.apache.ivy.ant">
- <property name="ivy.version" value="2.2.0" />
- <property name="ivy.url"
- value="http://repo2.maven.org/maven2/org/apache/ivy/ivy" />
+ <property name="ivy.version" value="2.2.0" />
+ <property name="ivy.url"
+ value="http://repo2.maven.org/maven2/org/apache/ivy/ivy" />
<property file="publish.properties"/>
- <property name="basedir" value="."/>
- <property name="build.src" value="${basedir}/src"/>
- <property name="build.src.java" value="${basedir}/src/java"/>
- <property name="build.dir" value="${basedir}/build"/>
- <property name="build.dir.lib" value="${build.dir}/lib"/>
- <property name="build.classes" value="${build.dir}/classes"/>
- <property name="dist.dir" value="${build.dir}/dist"/>
-
- <path id="autoivy.classpath">
- <fileset dir="${build.dir.lib}">
- <include name="**/*.jar" />
- </fileset>
- <pathelement location="${build.dir}/ivy-${ivy.version}.jar"/>
- </path>
-
+ <property name="basedir" value="."/>
+ <property name="build.src" value="${basedir}/src"/>
+ <property name="build.src.java" value="${basedir}/src/main/java"/>
+ <property name="build.src.test.java" value="${basedir}/src/test/java"/>
+ <property name="build.dir" value="${basedir}/build"/>
+ <property name="build.dir.lib" value="${build.dir}/lib"/>
+ <property name="build.classes" value="${build.dir}/classes"/>
+ <property name="build.testclasses" value="${build.dir}/test-classes"/>
+ <property name="dist.dir" value="${build.dir}/dist"/>
+
+ <path id="autoivy.classpath">
+ <fileset dir="${build.dir.lib}">
+ <include name="**/*.jar" />
+ </fileset>
+ <pathelement location="${build.dir}/ivy-${ivy.version}.jar"/>
+ </path>
+
+ <path id="test.compile.classpath">
+ <path refid="autoivy.classpath"/>
+ <pathelement location="${build.classes}"/>
+ </path>
+
+ <path id="test.classpath">
+ <path refid="test.compile.classpath"/>
+ <pathelement location="${build.classes}"/>
+ <pathelement location="${build.testclasses}"/>
+ </path>
+
<target name="init">
<mkdir dir="${build.dir}"/>
<mkdir dir="${build.dir.lib}"/>
<mkdir dir="${build.classes}"/>
+ <mkdir dir="${build.testclasses}"/>
<mkdir dir="${dist.dir}"/>
</target>
- <target name="ivy-download" depends="init" unless="ivy.jar.exists">
- <echo>Downloading Ivy...</echo>
- <get src="${ivy.url}/${ivy.version}/ivy-${ivy.version}.jar"
- dest="${build.dir.lib}/ivy-${ivy.version}.jar" usetimestamp="true" />
- </target>
-
- <target name="ivy-init" depends="ivy-download" unless="ivy.initialized">
- <taskdef resource="org/apache/ivy/ant/antlib.xml"
- uri="antlib:org.apache.ivy.ant"
- classpathref="autoivy.classpath"/>
- <property name="ivy.initialized" value="true"/>
- </target>
+ <target name="ivy-download" depends="init" unless="ivy.jar.exists">
+ <echo>Downloading Ivy...</echo>
+ <get src="${ivy.url}/${ivy.version}/ivy-${ivy.version}.jar"
+ dest="${build.dir.lib}/ivy-${ivy.version}.jar" usetimestamp="true" />
+ </target>
- <target name="ivy-retrieve-build" depends="ivy-init, resolve">
- <ivy:retrieve type="jar,source" sync="true"
- pattern="${build.dir.lib}/[type]s/[artifact]-[revision].[ext]" />
- </target>
+ <target name="ivy-init" depends="ivy-download" unless="ivy.initialized">
+ <taskdef resource="org/apache/ivy/ant/antlib.xml"
+ uri="antlib:org.apache.ivy.ant"
+ classpathref="autoivy.classpath"/>
+ <property name="ivy.initialized" value="true"/>
+ </target>
+
+ <target name="ivy-retrieve-build" depends="ivy-init, resolve">
+ <ivy:retrieve type="jar,source" pattern="${build.dir.lib}/[type]s/[artifact].[ext]"/>
+ </target>
<target name="resolve">
<ivy:resolve file="ivy.xml" revision="${version}"/>
</target>
- <target name="main" depends="compile, jar" description="Main target">
- <echo>
- Building the .jar file.
- </echo>
- </target>
-
- <target name="compile" depends="ivy-retrieve-build" description="Compilation target">
- <javac srcdir="src" destdir="${build.classes}"/>
- </target>
+ <target name="main" depends="compile, jar" description="Main target">
+ <echo>
+ Building the .jar file.
+ </echo>
+ </target>
+
+ <target name="compile" depends="ivy-retrieve-build" description="Compilation target">
+ <javac srcdir="${build.src.java}" destdir="${build.classes}"/>
+ </target>
<target name="gen-pom" depends="ivy-init">
- <ivy:makepom ivyfile="ivy.xml" pomfile="${dist.dir}/${ant.project.name}.pom">
- <mapping conf="default" scope="compile"/>
+ <ivy:makepom ivyfile="ivy.xml" pomfile="${dist.dir}/${ant.project.name}.pom">
+ <mapping conf="default" scope="compile"/>
</ivy:makepom>
</target>
<target name="jar" description="jar target" depends="compile">
- <jar jarfile="${dist.dir}/high-scale-lib.jar" basedir="${build.classes}" includes="org/**/*.class" />
+ <jar jarfile="${dist.dir}/high-scale-lib.jar" basedir="${build.classes}" includes="org/**/*.class" />
</target>
-
+
+ <target name="compile-tests" depends="compile">
+ <javac srcdir="${build.src.test.java}" destdir="${build.testclasses}">
+ <classpath refid="test.compile.classpath"/>
+ </javac>
+ </target>
+
+ <target name="test" depends="compile-tests" description="run unit tests">
+ <junit>
+ <classpath refid="test.classpath"/>
+ <test name="org.cliffc.high_scale_lib.HighScaleLibTestSuite"/>
+ </junit>
+ </target>
+
<target name="publish" description="publish jar to artifactory" depends="jar, gen-pom">
- <ivy:publish resolver="artifactory-publish" overwrite="true" organisation="${organization}" module="${module}" revision="${version}">
+ <ivy:publish resolver="artifactory-publish" overwrite="true" organisation="${organization}" module="${module}" revision="${version}">
<artifacts pattern="${dist.dir}/[artifact].[ext]"/>
-
- <artifact name="${ivy.module}" ext="pom" type="pom"/>
+ <artifact name="${ivy.module}" ext="pom" type="pom"/>
</ivy:publish>
</target>
<target name="publish-local" description="publish jar to the local repo" depends="jar">
- <ivy:publish resolver="local" overwrite="true" organisation="${organization}" module="${module}" revision="${version}">
+ <ivy:publish resolver="local" overwrite="true" organisation="${organization}" module="${module}" revision="${version}">
<artifacts pattern="${dist.dir}/[artifact].[ext]"/>
</ivy:publish>
</target>
@@ -88,10 +113,10 @@
<target name="clean" description="get rid of any build artifacts">
<delete dir="${dist.dir}" />
<delete dir="${build.dir}" />
- <delete dir="doc" />
+ <delete dir="doc"/>
</target>
<target name="javadoc" description="generate the docs">
- <javadoc sourcepath="src" destdir="doc"/>
+ <javadoc sourcepath="src" destdir="doc"/>
</target>
</project>
View
5 ivy.xml
@@ -1,7 +1,8 @@
<ivy-module version="2.0">
<info organisation="${organization}" module="${module}" revision="${version}"/>
-
<dependencies>
-
+ <dependency org="junit" name="junit" rev="4.8.2" >
+ <artifact name="junit" type="jar" conf="test"/>
+ </dependency>
</dependencies>
</ivy-module>
View
42 ivysettings.xml
@@ -1,21 +1,25 @@
<ivysettings>
- <properties file="credentials.properties"/>
- <properties file="publish.properties"/>
-
- <settings defaultResolver="main"/>
- <credentials host="maven.boundary.com" realm="Artifactory Realm" username="${username}" passwd="${password}"/>
- <resolvers >
-
- <chain name="main">
- <filesystem name="local">
- <ivy pattern="${ivy.default.ivy.user.dir}/local/[organisation]/[module]/[revision]/[type]s/[artifact].[ext]" />
- <artifact pattern="${ivy.default.ivy.user.dir}/local/[organisation]/[module]/[revision]/[type]s/[artifact].[ext]" />
- </filesystem>
- <url name="artifactory-publish" m2compatible="true">
- <artifact pattern="http://maven.boundary.com/artifactory/external/[organization]/[module]/[revision]/[module]-[revision](-[classifier]).[ext]" />
- <ivy pattern="http://maven.boundary.com/artifactory/external/[organization]/[module]/[revision]/ivy-[module].xml" />
- </url>
-
- </chain>
- </resolvers>
+ <properties file="credentials.properties"/>
+ <properties file="publish.properties"/>
+ <settings defaultResolver="main"/>
+ <credentials host="maven.boundary.com" realm="Artifactory Realm" username="${username}" passwd="${password}"/>
+ <resolvers>
+ <chain name="main">
+ <filesystem name="local">
+ <ivy pattern="${ivy.default.ivy.user.dir}/local/[organisation]/[module]/[revision]/[type]s/[artifact].[ext]"/>
+ <artifact
+ pattern="${ivy.default.ivy.user.dir}/local/[organisation]/[module]/[revision]/[type]s/[artifact].[ext]"/>
+ </filesystem>
+ <url name="artifactory-external" m2compatible="true">
+ <artifact
+ pattern="http://maven.boundary.com/artifactory/repo/[organization]/[module]/[revision]/[module]-[revision](-[classifier]).[ext]"/>
+ <ivy pattern="http://maven.boundary.com/artifactory/repo/[organization]/[module]/[revision]/ivy-[module].xml"/>
+ </url>
+ <url name="artifactory-publish" m2compatible="true">
+ <artifact
+ pattern="http://maven.boundary.com/artifactory/external/[organization]/[module]/[revision]/[module]-[revision](-[classifier]).[ext]"/>
+ <ivy pattern="http://maven.boundary.com/artifactory/external/[organization]/[module]/[revision]/ivy-[module].xml"/>
+ </url>
+ </chain>
+ </resolvers>
</ivysettings>
View
3  publish.properties
@@ -1,3 +1,4 @@
organization=com.boundary
module=high-scale-lib
-version=1.0.2
+version=1.0.3
+
View
28 src/main/java/com/boundary/BitPrint.java
@@ -0,0 +1,28 @@
+package com.boundary;
+
+/**
+ * utility for printing bit patterns
+ */
+public class BitPrint {
+ public static String fmt(long bits) {
+ StringBuilder sb = new StringBuilder();
+ long mask = 1L<<63;
+ for(int i = 1; i <= 64; i++) {
+ if((mask & bits) == mask)
+ sb.append("1");
+ else
+ sb.append("0");
+ if(i%8 == 0)
+ sb.append("|");
+ mask >>>= 1;
+ }
+ return sb.toString();
+ }
+ public static String fmt(long ... buffer) {
+ StringBuilder sb = new StringBuilder();
+ for(long bits : buffer) {
+ sb.append(fmt(bits)).append("\n");
+ }
+ return sb.toString();
+ }
+}
View
0  src/java/util/Hashtable.java → src/main/java/java/util/Hashtable.java
File renamed without changes
View
0  ...va/util/concurrent/ConcurrentHashMap.java → ...va/util/concurrent/ConcurrentHashMap.java
File renamed without changes
View
0  .../cliffc/high_scale_lib/AbstractEntry.java → .../cliffc/high_scale_lib/AbstractEntry.java
File renamed without changes
View
0  ...c/high_scale_lib/ConcurrentAutoTable.java → ...c/high_scale_lib/ConcurrentAutoTable.java
File renamed without changes
View
0  src/org/cliffc/high_scale_lib/Counter.java → ...va/org/cliffc/high_scale_lib/Counter.java
File renamed without changes
View
6 src/main/java/org/cliffc/high_scale_lib/IntIterator.java
@@ -0,0 +1,6 @@
+package org.cliffc.high_scale_lib;
+
+public interface IntIterator {
+ public int next();
+ public boolean hasNext();
+}
View
0  ...fc/high_scale_lib/NonBlockingHashMap.java → ...fc/high_scale_lib/NonBlockingHashMap.java
File renamed without changes
View
0  ...igh_scale_lib/NonBlockingHashMapLong.java → ...igh_scale_lib/NonBlockingHashMapLong.java
File renamed without changes
View
0  ...fc/high_scale_lib/NonBlockingHashSet.java → ...fc/high_scale_lib/NonBlockingHashSet.java
File renamed without changes
View
0  .../high_scale_lib/NonBlockingHashtable.java → .../high_scale_lib/NonBlockingHashtable.java
File renamed without changes
View
0  ...scale_lib/NonBlockingIdentityHashMap.java → ...scale_lib/NonBlockingIdentityHashMap.java
File renamed without changes
View
188 ...ffc/high_scale_lib/NonBlockingSetInt.java → ...ffc/high_scale_lib/NonBlockingSetInt.java
@@ -8,7 +8,6 @@
import java.io.Serializable;
import java.lang.reflect.*;
import java.util.*;
-import java.util.concurrent.*;
import java.util.concurrent.atomic.*;
import sun.misc.Unsafe;
@@ -64,6 +63,10 @@ public NonBlockingSetInt( ) {
_nbsi = new NBSI(63, new Counter(), this); // The initial 1-word set
}
+ private NonBlockingSetInt(NonBlockingSetInt a, NonBlockingSetInt b) {
+ _nbsi = new NBSI(a._nbsi,b._nbsi,new Counter(),this);
+ }
+
/**
* Add {@code i} to the set. Uppercase {@link Integer} version of add,
* requires auto-unboxing. When possible use the {@code int} version of
@@ -133,6 +136,30 @@ public void clear ( ) {
;
}
+ public int sizeInBytes() { return _nbsi.sizeInBytes(); }
+
+ /*****************************************************************
+ *
+ * bitwise comparisons optimised for NBSI
+ *
+ *****************************************************************/
+
+ public NonBlockingSetInt intersect(final NonBlockingSetInt op) {
+ NonBlockingSetInt res = new NonBlockingSetInt(this,op);
+ res._nbsi.intersect(res._nbsi, this._nbsi, op._nbsi);
+ return res;
+ }
+
+ public NonBlockingSetInt union(final NonBlockingSetInt op) {
+ NonBlockingSetInt res = new NonBlockingSetInt(this,op);
+ res._nbsi.union(res._nbsi, this._nbsi, op._nbsi);
+ return res;
+ }
+
+// public NonBlockingSetInt not(final NonBlockingSetInt op) {
+//
+// }
+
/** Verbose printout of internal structure for debugging. */
public void print() { _nbsi.print(0); }
@@ -142,38 +169,60 @@ public void clear ( ) {
*/
public Iterator<Integer> iterator( ) { return new iter(); }
- private class iter implements Iterator<Integer> {
- NBSI _nbsi2;
- int _idx = -1;
- int _prev = -1;
- iter() { _nbsi2 = _nbsi; advance(); }
- public boolean hasNext() { return _idx != -2; }
- private void advance() {
+ public IntIterator intIterator() { return new NBSIIntIterator(); }
+
+ private class NBSIIntIterator implements IntIterator {
+
+ NBSI nbsi;
+ int index = -1;
+ int prev = -1;
+
+ NBSIIntIterator() {
+ nbsi = _nbsi;
+ advance();
+ }
+
+ private void advance() {
while( true ) {
- _idx++; // Next index
- while( (_idx>>6) >= _nbsi2._bits.length ) { // Index out of range?
- if( _nbsi2._new == null ) { // New table?
- _idx = -2; // No, so must be all done
- return; //
+ index++; // Next index
+ while( (index>>6) >= nbsi._bits.length ) { // Index out of range?
+ if( nbsi._new == null ) { // New table?
+ index = -2; // No, so must be all done
+ return; //
}
- _nbsi2 = _nbsi2._new; // Carry on, in the new table
+ nbsi = nbsi._new; // Carry on, in the new table
}
- if( _nbsi2.contains(_idx) ) return;
+ if( nbsi.contains(index) ) return;
}
}
- public Integer next() {
- if( _idx == -1 ) throw new NoSuchElementException();
- _prev = _idx;
+ @Override
+ public int next() {
+ if( index == -1 ) throw new NoSuchElementException();
+ prev = index;
advance();
- return _prev;
+ return prev;
}
- public void remove() {
- if( _prev == -1 ) throw new IllegalStateException();
- _nbsi2.remove(_prev);
- _prev = -1;
+
+ @Override
+ public boolean hasNext() {
+ return index != -2;
+ }
+
+ public void remove() {
+ if( prev == -1 ) throw new IllegalStateException();
+ nbsi.remove(prev);
+ prev = -1;
}
}
+ private class iter implements Iterator<Integer> {
+ NBSIIntIterator intIterator;
+ iter() { intIterator = new NBSIIntIterator(); }
+ public boolean hasNext() { return intIterator.hasNext(); }
+ public Integer next() { return intIterator.next(); }
+ public void remove() { intIterator.remove(); }
+ }
+
// --- writeObject -------------------------------------------------------
// Write a NBSI to a stream
private void writeObject(java.io.ObjectOutputStream s) throws IOException {
@@ -262,7 +311,42 @@ private NBSI( int max_elem, Counter ctr, NonBlockingSetInt nonb ) {
_nbsi64 = ((max_elem+1)>>>6) == 0 ? null : new NBSI((max_elem+1)>>>6, null, null);
_sum_bits_length = _bits.length + (_nbsi64==null ? 0 : _nbsi64._sum_bits_length);
}
-
+
+ /** built a new NBSI with buffers large enough to hold bitwise operations on the operands **/
+ private NBSI(NBSI a, NBSI b, Counter ctr, NonBlockingSetInt nonb) {
+ super();
+ _non_blocking_set_int = nonb;
+ _size = ctr;
+ _copyIdx = ctr == null ? null : new AtomicInteger();
+ _copyDone = ctr == null ? null : new AtomicInteger();
+
+ if(!has_bits(a) && !has_bits(b)) {
+ _bits = null;
+ _nbsi64 = null;
+ _sum_bits_length = 0;
+ return;
+ }
+
+ // todo - clean this nastiness up
+ // essentially just safely creates new empty buffers for each of the recursive bitsets
+ if(!has_bits(a)) {
+ _bits = new long[b._bits.length];
+ _nbsi64 = new NBSI(null,b._nbsi64,null,null);
+ } else if(!has_bits(b)) {
+ _bits = new long[a._bits.length];
+ _nbsi64 = new NBSI(null,a._nbsi64,null,null);
+ } else {
+ int bit_length = a._bits.length > b._bits.length ? a._bits.length : b._bits.length;
+ _bits = new long[bit_length];
+ _nbsi64 = new NBSI(a._nbsi64,b._nbsi64,null,null);
+ }
+ _sum_bits_length = _bits.length + _nbsi64._sum_bits_length;
+ }
+
+ private static boolean has_bits(NBSI n) {
+ return n != null && n._bits != null;
+ }
+
// Lower-case 'int' versions - no autoboxing, very fast.
// 'i' is known positive.
public boolean add( final int i ) {
@@ -338,7 +422,59 @@ public boolean contains( final int i ) {
// Yes mutable: test & return bit
return (old & mask) != 0;
}
-
+
+ /**
+ * Bitwise operations which store the result in this instance.
+ * Assumes that this instance contains ample buffer space to store the largest
+ * buffer from each NBSI in the recursive bitmap.
+ *
+ * Also assumes that this method is called during the construction process of
+ * the bitset before the instance could be leaked to multiple threads.
+ ***/
+ public boolean intersect(NBSI dest, NBSI a, NBSI b) {
+ // terminate recursion if one bitset is missing data
+ // since that word should be left as 0L anyway
+ if(!has_bits(a) || !has_bits(b))
+ return true;
+ for(int i = 0; i < dest._bits.length; i++) {
+ long left = a.safe_read_word(i,0L);
+ long right = b.safe_read_word(i,0L);
+ dest._bits[i] = (left & right) & Long.MAX_VALUE; // mask sign bit
+ }
+ // todo - recompute size
+ return intersect(dest._nbsi64, a._nbsi64, b._nbsi64);
+ }
+
+ public boolean union(NBSI dest, NBSI a, NBSI b) {
+ // terminate recursion if neiter bitset has data
+ if(!has_bits(a) && !has_bits(b))
+ return true;
+ if(has_bits(a) || has_bits(b)) {
+ for(int i = 0; i < dest._bits.length; i++) {
+ long left = a.safe_read_word(i,0);
+ long right = b.safe_read_word(i,0);
+ dest._bits[i] = (left | right) & Long.MAX_VALUE;
+ }
+ }
+ return union(dest._nbsi64, a._nbsi64, b._nbsi64);
+ }
+
+ /**************************************************************************/
+
+ private long safe_read_word(int i, long default_word) {
+ if(i >= _bits.length) {
+ // allow reading past the end of the buffer filling in a default word
+ return default_word;
+ }
+ long word = _bits[i];
+ if(word < 0) {
+ word = help_copy_impl(i).help_copy()._bits[i];
+ }
+ return word;
+ }
+
+ public int sizeInBytes() { return (int)_bits.length; }
+
public int size() { return (int)_size.get(); }
// Must grow the current array to hold an element of size i
@@ -474,5 +610,5 @@ private void print(int d) {
_new.print(d+1);
}
}
- }
+ }
}
View
0  ...org/cliffc/high_scale_lib/UtilUnsafe.java → ...org/cliffc/high_scale_lib/UtilUnsafe.java
File renamed without changes
View
13 src/test/java/com/boundary/BitPrintTest.java
@@ -0,0 +1,13 @@
+package com.boundary;
+
+import junit.framework.TestCase;
+
+public class BitPrintTest extends TestCase {
+ public void testBitPrinting() {
+ long[] buf = new long[256];
+ for(int i = 0; i < buf.length; i++) {
+ buf[i] = i;
+ }
+ System.out.println(BitPrint.fmt(buf));
+ }
+}
View
20 src/test/java/org/cliffc/high_scale_lib/HighScaleLibTestSuite.java
@@ -0,0 +1,20 @@
+package org.cliffc.high_scale_lib;
+
+import junit.framework.Test;
+import junit.framework.TestSuite;
+
+/**
+ * HighScaleLibTestSuite
+ * @author Dietrich Featherston
+ */
+public class HighScaleLibTestSuite extends TestSuite {
+ public static Test suite() {
+ TestSuite suite = new TestSuite();
+ suite.addTest(new TestSuite(NonBlockingSetIntTest.class));
+ suite.addTest(new TestSuite(NonBlockingHashSetTest.class));
+ suite.addTest(new TestSuite(NonBlockingHashMapTest.class));
+ suite.addTest(new TestSuite(NonBlockingIdentityHashMapTest.class));
+ suite.addTest(new TestSuite(NonBlockingHashMapLongTest.class));
+ return suite;
+ }
+}
View
8 Testing/NBHM_Tester/NBHML_Tester2.java → ...scale_lib/NonBlockingHashMapLongTest.java
@@ -1,3 +1,5 @@
+package org.cliffc.high_scale_lib;
+
/*
* Written by Cliff Click and released to the public domain, as explained at
* http://creativecommons.org/licenses/publicdomain
@@ -9,15 +11,11 @@
import java.util.Map.Entry;
import java.util.concurrent.*;
import junit.framework.TestCase;
-import org.cliffc.high_scale_lib.*;
import static org.hamcrest.CoreMatchers.*;
import static org.junit.Assert.*;
// Test NonBlockingHashMapLong via JUnit
-public class NBHML_Tester2 extends TestCase {
- public static void main(String args[]) {
- org.junit.runner.JUnitCore.main("Testing.NBHM_Tester.NBHML_Tester2");
- }
+public class NonBlockingHashMapLongTest extends TestCase {
private NonBlockingHashMapLong<String> _nbhml;
protected void setUp () { _nbhml = new NonBlockingHashMapLong<String>(); }
View
8 Testing/NBHM_Tester/NBHM_Tester2.java → ...igh_scale_lib/NonBlockingHashMapTest.java
@@ -1,3 +1,5 @@
+package org.cliffc.high_scale_lib;
+
/*
* Written by Cliff Click and released to the public domain, as explained at
* http://creativecommons.org/licenses/publicdomain
@@ -8,15 +10,11 @@
import java.util.Map.Entry;
import java.util.concurrent.*;
import junit.framework.TestCase;
-import org.cliffc.high_scale_lib.*;
import static org.hamcrest.CoreMatchers.*;
import static org.junit.Assert.*;
// Test NonBlockingHashMap via JUnit
-public class NBHM_Tester2 extends TestCase {
- public static void main(String args[]) {
- org.junit.runner.JUnitCore.main("Testing.NBHM_Tester.NBHM_Tester2");
- }
+public class NonBlockingHashMapTest extends TestCase {
private NonBlockingHashMap<String,String> _nbhm;
protected void setUp () { _nbhm = new NonBlockingHashMap<String,String>(); }
View
8 Testing/NBHS_Tester/nbhs_tester.java → ...igh_scale_lib/NonBlockingHashSetTest.java
@@ -1,3 +1,5 @@
+package org.cliffc.high_scale_lib;
+
/*
* Written by Cliff Click and released to the public domain, as explained at
* http://creativecommons.org/licenses/publicdomain
@@ -7,15 +9,11 @@
import java.io.*;
import java.util.*;
import junit.framework.TestCase;
-import org.cliffc.high_scale_lib.*;
import static org.hamcrest.CoreMatchers.*;
import static org.junit.Assert.*;
// Test NonBlockingHashSet via JUnit
-public class nbhs_tester extends TestCase {
- public static void main(String args[]) {
- org.junit.runner.JUnitCore.main("Testing.NBHS_Tester.nbhs_tester");
- }
+public class NonBlockingHashSetTest extends TestCase {
private NonBlockingHashSet<String> _nbhs;
protected void setUp () { _nbhs = new NonBlockingHashSet<String>(); }
View
551 src/test/java/org/cliffc/high_scale_lib/NonBlockingIdentityHashMapTest.java
@@ -0,0 +1,551 @@
+package org.cliffc.high_scale_lib;
+
+/*
+ * Written by Cliff Click and released to the public domain, as explained at
+ * http://creativecommons.org/licenses/publicdomain
+ */
+
+import java.io.*;
+import java.util.*;
+import java.util.Map.Entry;
+import java.util.concurrent.*;
+import junit.framework.TestCase;
+import org.cliffc.high_scale_lib.*;
+import static org.hamcrest.CoreMatchers.*;
+import static org.junit.Assert.*;
+
+// Test NonBlockingHashMap via JUnit
+public class NonBlockingIdentityHashMapTest extends TestCase {
+ public static void main(String args[]) {
+ org.junit.runner.JUnitCore.main("Testing.NBHM_Tester.NBHMID_Tester2");
+ }
+
+ private NonBlockingIdentityHashMap<String,String> _nbhm;
+ protected void setUp () { _nbhm = new NonBlockingIdentityHashMap<String,String>(); }
+ protected void tearDown() { _nbhm = null; }
+
+ // Throw a ClassCastException if I see a tombstone during key-compares
+ private static class KeyBonk {
+ final int _x;
+ KeyBonk( int i ) { _x=i; }
+ public boolean equals( Object o ) {
+ if( o == null ) return false;
+ return ((KeyBonk)o)._x // Throw CCE here
+ == this._x;
+ }
+ public int hashCode() { return (_x>>2); }
+ public String toString() { return "Bonk_"+Integer.toString(_x); }
+ }
+
+ // Test some basic stuff; add a few keys, remove a few keys
+ public void testBasic() {
+ assertTrue ( _nbhm.isEmpty() );
+ assertThat ( _nbhm.putIfAbsent("k1","v1"), nullValue() );
+ checkSizes (1);
+ assertThat ( _nbhm.putIfAbsent("k2","v2"), nullValue() );
+ checkSizes (2);
+ assertTrue ( _nbhm.containsKey("k2") );
+ assertThat ( _nbhm.put("k1","v1a"), is("v1") );
+ assertThat ( _nbhm.put("k2","v2a"), is("v2") );
+ checkSizes (2);
+ assertThat ( _nbhm.putIfAbsent("k2","v2b"), is("v2a") );
+ assertThat ( _nbhm.remove("k1"), is("v1a") );
+ assertFalse( _nbhm.containsKey("k1") );
+ checkSizes (1);
+ assertThat ( _nbhm.remove("k1"), nullValue() );
+ assertThat ( _nbhm.remove("k2"), is("v2a") );
+ checkSizes (0);
+ assertThat ( _nbhm.remove("k2"), nullValue() );
+ assertThat ( _nbhm.remove("k3"), nullValue() );
+ assertTrue ( _nbhm.isEmpty() );
+
+ assertThat ( _nbhm.put("k0","v0"), nullValue() );
+ assertTrue ( _nbhm.containsKey("k0") );
+ checkSizes (1);
+ assertThat ( _nbhm.remove("k0"), is("v0") );
+ assertFalse( _nbhm.containsKey("k0") );
+ checkSizes (0);
+
+ assertThat ( _nbhm.replace("k0","v0"), nullValue() );
+ assertFalse( _nbhm.containsKey("k0") );
+ assertThat ( _nbhm.put("k0","v0"), nullValue() );
+ assertEquals(_nbhm.replace("k0","v0a"), "v0" );
+ assertEquals(_nbhm.get("k0"), "v0a" );
+ assertThat ( _nbhm.remove("k0"), is("v0a") );
+ assertFalse( _nbhm.containsKey("k0") );
+ checkSizes (0);
+
+ assertThat ( _nbhm.replace("k1","v1"), nullValue() );
+ assertFalse( _nbhm.containsKey("k1") );
+ assertThat ( _nbhm.put("k1","v1"), nullValue() );
+ assertEquals(_nbhm.replace("k1","v1a"), "v1" );
+ assertEquals(_nbhm.get("k1"), "v1a" );
+ assertThat ( _nbhm.remove("k1"), is("v1a") );
+ assertFalse( _nbhm.containsKey("k1") );
+ checkSizes (0);
+
+ // Insert & Remove KeyBonks until the table resizes and we start
+ // finding Tombstone keys- and KeyBonk's equals-call with throw a
+ // ClassCastException if it sees a non-KeyBonk.
+ NonBlockingIdentityHashMap<KeyBonk,String> dumb = new NonBlockingIdentityHashMap<KeyBonk,String>();
+ for( int i=0; i<10000; i++ ) {
+ final KeyBonk happy1 = new KeyBonk(i);
+ assertThat( dumb.put(happy1,"and"), nullValue() );
+ if( (i&1)==0 ) dumb.remove(happy1);
+ final KeyBonk happy2 = new KeyBonk(i); // 'equals' but not '=='
+ dumb.get(happy2);
+ }
+ }
+
+ // Check all iterators for correct size counts
+ private void checkSizes(int expectedSize) {
+ assertEquals( "size()", _nbhm.size(), expectedSize );
+ Collection<String> vals = _nbhm.values();
+ checkSizes("values()",vals.size(),vals.iterator(),expectedSize);
+ Set<String> keys = _nbhm.keySet();
+ checkSizes("keySet()",keys.size(),keys.iterator(),expectedSize);
+ Set<Entry<String,String>> ents = _nbhm.entrySet();
+ checkSizes("entrySet()",ents.size(),ents.iterator(),expectedSize);
+ }
+
+ // Check that the iterator iterates the correct number of times
+ private void checkSizes(String msg, int sz, Iterator it, int expectedSize) {
+ assertEquals( msg, expectedSize, sz );
+ int result = 0;
+ while (it.hasNext()) {
+ result++;
+ it.next();
+ }
+ assertEquals( msg, expectedSize, result );
+ }
+
+
+ public void testIteration() {
+ assertTrue ( _nbhm.isEmpty() );
+ assertThat ( _nbhm.put("k1","v1"), nullValue() );
+ assertThat ( _nbhm.put("k2","v2"), nullValue() );
+
+ String str1 = "";
+ for( Iterator<Map.Entry<String,String>> i = _nbhm.entrySet().iterator(); i.hasNext(); ) {
+ Map.Entry<String,String> e = i.next();
+ str1 += e.getKey();
+ }
+ assertThat("found all entries",str1,anyOf(is("k1k2"),is("k2k1")));
+
+ String str2 = "";
+ for( Iterator<String> i = _nbhm.keySet().iterator(); i.hasNext(); ) {
+ String key = i.next();
+ str2 += key;
+ }
+ assertThat("found all keys",str2,anyOf(is("k1k2"),is("k2k1")));
+
+ String str3 = "";
+ for( Iterator<String> i = _nbhm.values().iterator(); i.hasNext(); ) {
+ String val = i.next();
+ str3 += val;
+ }
+ assertThat("found all vals",str3,anyOf(is("v1v2"),is("v2v1")));
+
+ assertThat("toString works",_nbhm.toString(), anyOf(is("{k1=v1, k2=v2}"),is("{k2=v2, k1=v1}")));
+ }
+
+ public void testSerial() {
+ assertTrue ( _nbhm.isEmpty() );
+ final String k1 = "k1";
+ final String k2 = "k2";
+ assertThat ( _nbhm.put(k1,"v1"), nullValue() );
+ assertThat ( _nbhm.put(k2,"v2"), nullValue() );
+
+ // Serialize it out
+ try {
+ FileOutputStream fos = new FileOutputStream("NBHM_test.txt");
+ ObjectOutputStream out = new ObjectOutputStream(fos);
+ out.writeObject(_nbhm);
+ out.close();
+ } catch(IOException ex) {
+ ex.printStackTrace();
+ }
+
+ // Read it back
+ try {
+ File f = new File("NBHM_test.txt");
+ FileInputStream fis = new FileInputStream(f);
+ ObjectInputStream in = new ObjectInputStream(fis);
+ NonBlockingIdentityHashMap nbhm = (NonBlockingIdentityHashMap)in.readObject();
+ in.close();
+ assertThat("serialization works",nbhm.toString(), anyOf(is("{k1=v1, k2=v2}"),is("{k2=v2, k1=v1}")));
+ if( !f.delete() ) throw new IOException("delete failed");
+ } catch(IOException ex) {
+ ex.printStackTrace();
+ } catch(ClassNotFoundException ex) {
+ ex.printStackTrace();
+ }
+ }
+
+ public void testIterationBig2() {
+ final int CNT = 10000;
+ NonBlockingIdentityHashMap<Integer,String> nbhm = new NonBlockingIdentityHashMap<Integer,String>();
+ final String v = "v";
+ for( int i=0; i<CNT; i++ ) {
+ final Integer z = new Integer(i);
+ String s0 = nbhm.get(z);
+ assertThat( s0, nullValue() );
+ nbhm.put(z,v);
+ String s1 = nbhm.get(z);
+ assertThat( s1, is(v) );
+ }
+ assertThat( nbhm.size(), is(CNT) );
+ }
+
+ public void testIterationBig() {
+ final int CNT = 10000;
+ String [] keys = new String[CNT];
+ String [] vals = new String[CNT];
+ assertThat( _nbhm.size(), is(0) );
+ for( int i=0; i<CNT; i++ )
+ _nbhm.put(keys[i]=("k"+i),vals[i]=("v"+i));
+ assertThat( _nbhm.size(), is(CNT) );
+
+ int sz =0;
+ int sum = 0;
+ for( String s : _nbhm.keySet() ) {
+ sz++;
+ assertThat("",s.charAt(0),is('k'));
+ int x = Integer.parseInt(s.substring(1));
+ sum += x;
+ assertTrue(x>=0 && x<=(CNT-1));
+ }
+ assertThat("Found 10000 ints",sz,is(CNT));
+ assertThat("Found all integers in list",sum,is(CNT*(CNT-1)/2));
+
+ assertThat( "can remove 3", _nbhm.remove(keys[3]), is(vals[3]) );
+ assertThat( "can remove 4", _nbhm.remove(keys[4]), is(vals[4]) );
+ sz =0;
+ sum = 0;
+ for( String s : _nbhm.keySet() ) {
+ sz++;
+ assertThat("",s.charAt(0),is('k'));
+ int x = Integer.parseInt(s.substring(1));
+ sum += x;
+ assertTrue(x>=0 && x<=(CNT-1));
+ String v = _nbhm.get(s);
+ assertThat("",v.charAt(0),is('v'));
+ assertThat("",s.substring(1),is(v.substring(1)));
+ }
+ assertThat("Found "+(CNT-2)+" ints",sz,is(CNT-2));
+ assertThat("Found all integers in list",sum,is(CNT*(CNT-1)/2 - (3+4)));
+ }
+
+ // Do some simple concurrent testing
+ public void testConcurrentSimple() throws InterruptedException {
+ final NonBlockingIdentityHashMap<String,String> nbhm = new NonBlockingIdentityHashMap<String,String>();
+ final String [] keys = new String[20000];
+ for( int i=0; i<20000; i++ )
+ keys[i]="k"+i;
+
+ // In 2 threads, add & remove even & odd elements concurrently
+ Thread t1 = new Thread() { public void run() { work_helper(nbhm,"T1",1,keys); } };
+ t1.start();
+ work_helper(nbhm,"T0",0,keys);
+ t1.join();
+
+ // In the end, all members should be removed
+ StringBuffer buf = new StringBuffer();
+ buf.append("Should be emptyset but has these elements: {");
+ boolean found = false;
+ for( String x : nbhm.keySet() ) {
+ buf.append(" ").append(x);
+ found = true;
+ }
+ if( found ) System.out.println(buf+" }");
+ assertThat( "concurrent size=0", nbhm.size(), is(0) );
+ for( String x : nbhm.keySet() ) {
+ assertTrue("No elements so never get here",false);
+ }
+ }
+
+ void work_helper(NonBlockingIdentityHashMap<String,String> nbhm, String thrd, int d, String[] keys) {
+ final int ITERS = 20000;
+ for( int j=0; j<10; j++ ) {
+ long start = System.nanoTime();
+ for( int i=d; i<ITERS; i+=2 )
+ assertThat( "this key not in there, so putIfAbsent must work",
+ nbhm.putIfAbsent(keys[i],thrd), is((String)null) );
+ for( int i=d; i<ITERS; i+=2 )
+ assertTrue( nbhm.remove(keys[i],thrd) );
+ double delta_nanos = System.nanoTime()-start;
+ double delta_secs = delta_nanos/1000000000.0;
+ double ops = ITERS*2;
+ //System.out.println("Thrd"+thrd+" "+(ops/delta_secs)+" ops/sec size="+nbhm.size());
+ }
+ }
+
+ public final void testNonBlockingIdentityHashMapSize() {
+ NonBlockingIdentityHashMap<Long,String> items = new NonBlockingIdentityHashMap<Long,String>();
+ items.put(Long.valueOf(100), "100");
+ items.put(Long.valueOf(101), "101");
+
+ assertEquals("keySet().size()", 2, items.keySet().size());
+ assertTrue("keySet().contains(100)", items.keySet().contains(Long.valueOf(100)));
+ assertTrue("keySet().contains(101)", items.keySet().contains(Long.valueOf(101)));
+
+ assertEquals("values().size()", 2, items.values().size());
+ assertTrue("values().contains(\"100\")", items.values().contains("100"));
+ assertTrue("values().contains(\"101\")", items.values().contains("101"));
+
+ assertEquals("entrySet().size()", 2, items.entrySet().size());
+ boolean found100 = false;
+ boolean found101 = false;
+ for (Entry<Long, String> entry : items.entrySet()) {
+ if (entry.getKey().equals(Long.valueOf(100))) {
+ assertEquals("entry[100].getValue()==\"100\"", "100", entry.getValue());
+ found100 = true;
+ } else if (entry.getKey().equals(Long.valueOf(101))) {
+ assertEquals("entry[101].getValue()==\"101\"", "101", entry.getValue());
+ found101 = true;
+ }
+ }
+ assertTrue("entrySet().contains([100])", found100);
+ assertTrue("entrySet().contains([101])", found101);
+ }
+
+ // Concurrent insertion & then iterator test.
+ static public void testNonBlockingIdentityHashMapIterator() throws InterruptedException {
+ final int ITEM_COUNT1 = 1000;
+ final int THREAD_COUNT = 5;
+ final int PER_CNT = ITEM_COUNT1/THREAD_COUNT;
+ final int ITEM_COUNT = PER_CNT*THREAD_COUNT; // fix roundoff for odd thread counts
+
+ NonBlockingIdentityHashMap<Long,TestKey> nbhml = new NonBlockingIdentityHashMap<Long,TestKey>();
+ // use a barrier to open the gate for all threads at once to avoid rolling
+ // start and no actual concurrency
+ final CyclicBarrier barrier = new CyclicBarrier(THREAD_COUNT);
+ final ExecutorService ex = Executors.newFixedThreadPool(THREAD_COUNT);
+ final CompletionService<Object> co = new ExecutorCompletionService<Object>(ex);
+ for( int i=0; i<THREAD_COUNT; i++ ) {
+ co.submit(new NBHMLFeeder(nbhml, PER_CNT, barrier, i*PER_CNT));
+ }
+ for( int retCount = 0; retCount < THREAD_COUNT; retCount++ ) {
+ co.take();
+ }
+ ex.shutdown();
+
+ assertEquals("values().size()", ITEM_COUNT, nbhml.values().size());
+ assertEquals("entrySet().size()", ITEM_COUNT, nbhml.entrySet().size());
+ int itemCount = 0;
+ for( TestKey K : nbhml.values() )
+ itemCount++;
+ assertEquals("values().iterator() count", ITEM_COUNT, itemCount);
+ }
+
+ // --- NBHMLFeeder ---
+ // Class to be called from another thread, to get concurrent installs into
+ // the table.
+ static private class NBHMLFeeder implements Callable<Object> {
+ static private final Random _rand = new Random(System.currentTimeMillis());
+ private final NonBlockingIdentityHashMap<Long,TestKey> _map;
+ private final int _count;
+ private final CyclicBarrier _barrier;
+ private final long _offset;
+ public NBHMLFeeder(final NonBlockingIdentityHashMap<Long,TestKey> map, final int count, final CyclicBarrier barrier, final long offset) {
+ _map = map;
+ _count = count;
+ _barrier = barrier;
+ _offset = offset;
+ }
+ public Object call() throws Exception {
+ _barrier.await(); // barrier, to force racing start
+ for( long j=0; j<_count; j++ )
+ _map.put(j+_offset, new TestKey(_rand.nextLong(),_rand.nextInt (), (short) _rand.nextInt(Short.MAX_VALUE)));
+ return null;
+ }
+ }
+
+ // --- TestKey ---
+ // Funny key tests all sorts of things, has a pre-wired hashCode & equals.
+ static private final class TestKey {
+ public final int _type;
+ public final long _id;
+ public final int _hash;
+ public TestKey(final long id, final int type, int hash) {
+ _id = id;
+ _type = type;
+ _hash = hash;
+ }
+ public int hashCode() { return _hash; }
+ public boolean equals(Object object) {
+ if (null == object) return false;
+ if (object == this) return true;
+ if (object.getClass() != this.getClass()) return false;
+ final TestKey other = (TestKey) object;
+ return (this._type == other._type && this._id == other._id);
+ }
+ public String toString() { return String.format("%s:%d,%d,%d", getClass().getSimpleName(), _id, _type, _hash); }
+ }
+
+ // --- Customer Test Case 3 ------------------------------------------------
+ private TestKeyFeeder getTestKeyFeeder() {
+ final TestKeyFeeder feeder = new TestKeyFeeder();
+ feeder.checkedPut(10401000001844L, 657829272, 680293140); // section 12
+ feeder.checkedPut(10401000000614L, 657829272, 401326994); // section 12
+ feeder.checkedPut(10400345749304L, 2095121916, -9852212); // section 12
+ feeder.checkedPut(10401000002204L, 657829272, 14438460); // section 12
+ feeder.checkedPut(10400345749234L, 1186831289, -894006017); // section 12
+ feeder.checkedPut(10401000500234L, 969314784, -2112018706); // section 12
+ feeder.checkedPut(10401000000284L, 657829272, 521425852); // section 12
+ feeder.checkedPut(10401000002134L, 657829272, 208406306); // section 12
+ feeder.checkedPut(10400345749254L, 2095121916, -341939818); // section 12
+ feeder.checkedPut(10401000500384L, 969314784, -2136811544); // section 12
+ feeder.checkedPut(10401000001944L, 657829272, 935194952); // section 12
+ feeder.checkedPut(10400345749224L, 1186831289, -828214183); // section 12
+ feeder.checkedPut(10400345749244L, 2095121916, -351234120); // section 12
+ feeder.checkedPut(10400333128994L, 2095121916, -496909430); // section 12
+ feeder.checkedPut(10400333197934L, 2095121916, 2147144926); // section 12
+ feeder.checkedPut(10400333197944L, 2095121916, -2082366964); // section 12
+ feeder.checkedPut(10400336947684L, 2095121916, -1404212288); // section 12
+ feeder.checkedPut(10401000000594L, 657829272, 124369790); // section 12
+ feeder.checkedPut(10400331896264L, 2095121916, -1028383492); // section 12
+ feeder.checkedPut(10400332415044L, 2095121916, 1629436704); // section 12
+ feeder.checkedPut(10400345749614L, 1186831289, 1027996827); // section 12
+ feeder.checkedPut(10401000500424L, 969314784, -1871616544); // section 12
+ feeder.checkedPut(10400336947694L, 2095121916, -1468802722); // section 12
+ feeder.checkedPut(10410002672481L, 2154973, 1515288586); // section 12
+ feeder.checkedPut(10410345749171L, 2154973, 2084791828); // section 12
+ feeder.checkedPut(10400004960671L, 2154973, 1554754674); // section 12
+ feeder.checkedPut(10410009983601L, 2154973, -2049707334); // section 12
+ feeder.checkedPut(10410335811601L, 2154973, 1547385114); // section 12
+ feeder.checkedPut(10410000005951L, 2154973, -1136117016); // section 12
+ feeder.checkedPut(10400004938331L, 2154973, -1361373018); // section 12
+ feeder.checkedPut(10410001490421L, 2154973, -818792874); // section 12
+ feeder.checkedPut(10400001187131L, 2154973, 649763142); // section 12
+ feeder.checkedPut(10410000409071L, 2154973, -614460616); // section 12
+ feeder.checkedPut(10410333717391L, 2154973, 1343531416); // section 12
+ feeder.checkedPut(10410336680071L, 2154973, -914544144); // section 12
+ feeder.checkedPut(10410002068511L, 2154973, -746995576); // section 12
+ feeder.checkedPut(10410336207851L, 2154973, 863146156); // section 12
+ feeder.checkedPut(10410002365251L, 2154973, 542724164); // section 12
+ feeder.checkedPut(10400335812581L, 2154973, 2146284796); // section 12
+ feeder.checkedPut(10410337345361L, 2154973, -384625318); // section 12
+ feeder.checkedPut(10410000409091L, 2154973, -528258556); // section 12
+ return feeder;
+ }
+
+ // ---
+ static private class TestKeyFeeder {
+ private final Hashtable<Integer, List<TestKey>> _items = new Hashtable<Integer, List<TestKey>>();
+ private int _size = 0;
+ public int size() { return _size; }
+ // Put items into the hashtable, sorted by 'type' into LinkedLists.
+ public void checkedPut(final long id, final int type, final int hash) {
+ _size++;
+ final TestKey item = new TestKey(id, type, hash);
+ if( !_items.containsKey(type) )
+ _items.put(type, new LinkedList<TestKey>());
+ _items.get(type).add(item);
+ }
+
+ public NonBlockingIdentityHashMap<Long,TestKey> getMapMultithreaded() throws InterruptedException, ExecutionException {
+ final int threadCount = _items.keySet().size();
+ final NonBlockingIdentityHashMap<Long,TestKey> map = new NonBlockingIdentityHashMap<Long,TestKey>();
+
+ // use a barrier to open the gate for all threads at once to avoid rolling start and no actual concurrency
+ final CyclicBarrier barrier = new CyclicBarrier(threadCount);
+ final ExecutorService ex = Executors.newFixedThreadPool(threadCount);
+ final CompletionService<Integer> co = new ExecutorCompletionService<Integer>(ex);
+ for( Integer type : _items.keySet() ) {
+ // A linked-list of things to insert
+ List<TestKey> items = _items.get(type);
+ TestKeyFeederThread feeder = new TestKeyFeederThread(type, items, map, barrier);
+ co.submit(feeder);
+ }
+
+ // wait for all threads to return
+ int itemCount = 0;
+ for( int retCount = 0; retCount < threadCount; retCount++ ) {
+ final Future<Integer> result = co.take();
+ itemCount += result.get();
+ }
+ ex.shutdown();
+ return map;
+ }
+ }
+
+ // --- TestKeyFeederThread
+ static private class TestKeyFeederThread implements Callable<Integer> {
+ private final int _type;
+ private final NonBlockingIdentityHashMap<Long,TestKey> _map;
+ private final List<TestKey> _items;
+ private final CyclicBarrier _barrier;
+ public TestKeyFeederThread(final int type, final List<TestKey> items, final NonBlockingIdentityHashMap<Long,TestKey> map, final CyclicBarrier barrier) {
+ _type = type;
+ _map = map;
+ _items = items;
+ _barrier = barrier;
+ }
+
+ public Integer call() throws Exception {
+ _barrier.await();
+ int count = 0;
+ for( TestKey item : _items ) {
+ if (_map.contains(item._id)) {
+ System.err.printf("COLLISION DETECTED: %s exists\n", item.toString());
+ }
+ final TestKey exists = _map.putIfAbsent(item._id, item);
+ if (exists == null) {
+ count++;
+ } else {
+ System.err.printf("COLLISION DETECTED: %s exists as %s\n", item.toString(), exists.toString());
+ }
+ }
+ return count;
+ }
+ }
+
+ // ---
+ public void testNonBlockingIdentityHashMapIteratorMultithreaded() throws InterruptedException, ExecutionException {
+ TestKeyFeeder feeder = getTestKeyFeeder();
+ final int itemCount = feeder.size();
+
+ // validate results
+ final NonBlockingIdentityHashMap<Long,TestKey> items = feeder.getMapMultithreaded();
+ assertEquals("size()", itemCount, items.size());
+
+ assertEquals("values().size()", itemCount, items.values().size());
+
+ assertEquals("entrySet().size()", itemCount, items.entrySet().size());
+
+ int iteratorCount = 0;
+ for( TestKey m : items.values() )
+ iteratorCount++;
+ // sometimes a different result comes back the second time
+ int iteratorCount2 = 0;
+ for( Iterator<TestKey> it = items.values().iterator(); it.hasNext(); ) {
+ iteratorCount2++;
+ it.next();
+ }
+ assertEquals("iterator counts differ", iteratorCount, iteratorCount2);
+ assertEquals("values().iterator() count", itemCount, iteratorCount);
+ }
+
+ // This test is a copy of the JCK test Hashtable2027, which is incorrect.
+ // The test requires a particular order of values to appear in the esa
+ // array - but this is not part of the spec. A different implementation
+ // might put the same values into the array but in a different order.
+ //public void testToArray() {
+ // NonBlockingIdentityHashMap ht = new NonBlockingIdentityHashMap();
+ //
+ // ht.put("Nine", new Integer(9));
+ // ht.put("Ten", new Integer(10));
+ // ht.put("Ten1", new Integer(100));
+ //
+ // Collection es = ht.values();
+ //
+ // Object [] esa = es.toArray();
+ //
+ // ht.remove("Ten1");
+ //
+ // assertEquals( "size check", es.size(), 2 );
+ // assertEquals( "iterator_order[0]", new Integer( 9), esa[0] );
+ // assertEquals( "iterator_order[1]", new Integer(10), esa[1] );
+ //}
+}
View
422 Testing/NBHS_Tester/nbsi_tester.java → ...high_scale_lib/NonBlockingSetIntTest.java
@@ -1,191 +1,231 @@
-/*
- * Written by Cliff Click and released to the public domain, as explained at
- * http://creativecommons.org/licenses/publicdomain
- */
-
-import java.io.*;
-import java.util.*;
-import junit.framework.TestCase;
-import org.cliffc.high_scale_lib.*;
-import static org.hamcrest.CoreMatchers.*;
-import static org.junit.Assert.*;
-
-// Test NonBlockingSetInt via JUnit
-public class nbsi_tester extends TestCase {
- public static void main(String args[]) {
- org.junit.runner.JUnitCore.main("Testing.NBHS_Tester.nbsi_tester");
- }
-
- private NonBlockingSetInt _nbsi;
- protected void setUp () { _nbsi = new NonBlockingSetInt(); }
- protected void tearDown() { _nbsi = null; }
-
- // Test some basic stuff; add a few keys, remove a few keys
- public void testBasic() {
- assertTrue ( _nbsi.isEmpty() );
- assertTrue ( _nbsi.add(1) );
- checkSizes (1);
- assertTrue ( _nbsi.add(2) );
- checkSizes (2);
- assertFalse( _nbsi.add(1) );
- assertFalse( _nbsi.add(2) );
- checkSizes (2);
- assertThat ( _nbsi.remove(1), is(true ) );
- checkSizes (1);
- assertThat ( _nbsi.remove(1), is(false) );
- assertTrue ( _nbsi.remove(2) );
- checkSizes (0);
- assertFalse( _nbsi.remove(2) );
- assertFalse( _nbsi.remove(3) );
- assertTrue ( _nbsi.isEmpty() );
- assertTrue ( _nbsi.add(63) );
- checkSizes (1);
- assertTrue ( _nbsi.remove(63) );
- assertFalse( _nbsi.remove(63) );
-
-
- assertTrue ( _nbsi.isEmpty() );
- assertTrue ( _nbsi.add(10000) );
- checkSizes (1);
- assertTrue ( _nbsi.add(20000) );
- checkSizes (2);
- assertFalse( _nbsi.add(10000) );
- assertFalse( _nbsi.add(20000) );
- checkSizes (2);
- assertThat ( _nbsi.remove(10000), is(true ) );
- checkSizes (1);
- assertThat ( _nbsi.remove(10000), is(false) );
- assertTrue ( _nbsi.remove(20000) );
- checkSizes (0);
- assertFalse( _nbsi.remove(20000) );
- }
-
- // Check all iterators for correct size counts
- private void checkSizes(int expectedSize) {
- assertEquals( "size()", _nbsi.size(), expectedSize );
- Iterator it = _nbsi.iterator();
- int result = 0;
- while (it.hasNext()) {
- result++;
- it.next();
- }
- assertEquals( "iterator missed", expectedSize, result );
- }
-
-
- public void testIteration() {
- assertTrue ( _nbsi.isEmpty() );
- assertTrue ( _nbsi.add(1) );
- assertTrue ( _nbsi.add(2) );
-
- StringBuffer buf = new StringBuffer();
- for( Iterator<Integer> i = _nbsi.iterator(); i.hasNext(); ) {
- Integer val = i.next();
- buf.append(val);
- }
- assertThat("found all vals",buf.toString(),anyOf(is("12"),is("21")));
-
- assertThat("toString works",_nbsi.toString(), anyOf(is("[1, 2]"),is("[2, 1]")));
- }
-
- public void testIterationBig() {
- for( int i=0; i<100; i++ )
- _nbsi.add(i);
- assertThat( _nbsi.size(), is(100) );
-
- int sz =0;
- int sum = 0;
- for( Integer x : _nbsi ) {
- sz++;
- sum += x;
- assertTrue(x>=0 && x<=99);
- }
- assertThat("Found 100 ints",sz,is(100));
- assertThat("Found all integers in list",sum,is(100*99/2));
-
- assertThat( "can remove 3", _nbsi.remove(3), is(true) );
- assertThat( "can remove 4", _nbsi.remove(4), is(true) );
- sz =0;
- sum = 0;
- for( Integer x : _nbsi ) {
- sz++;
- sum += x;
- assertTrue(x>=0 && x<=99);
- }
- assertThat("Found 98 ints",sz,is(98));
- assertThat("Found all integers in list",sum,is(100*99/2 - (3+4)));
-
- }
-
- public void testSerial() {
- assertTrue ( _nbsi.isEmpty() );
- assertTrue ( _nbsi.add(1) );
- assertTrue ( _nbsi.add(2) );
-
- // Serialize it out
- try {
- FileOutputStream fos = new FileOutputStream("NBSI_test.txt");
- ObjectOutputStream out = new ObjectOutputStream(fos);
- out.writeObject(_nbsi);
- out.close();
- } catch(IOException ex) {
- ex.printStackTrace();
- }
-
- // Read it back
- try {
- File f = new File("NBSI_test.txt");
- FileInputStream fis = new FileInputStream(f);
- ObjectInputStream in = new ObjectInputStream(fis);
- NonBlockingSetInt nbsi = (NonBlockingSetInt)in.readObject();
- in.close();
- assertEquals(_nbsi.toString(),nbsi.toString());
- if( !f.delete() ) throw new IOException("delete failed");
- } catch(IOException ex) {
- ex.printStackTrace();
- } catch(ClassNotFoundException ex) {
- ex.printStackTrace();
- }
- }
-
- // Do some simple concurrent testing
- public void testConcurrentSimple() throws InterruptedException {
- final NonBlockingSetInt nbsi = new NonBlockingSetInt();
-
- // In 2 threads, add & remove even & odd elements concurrently
- Thread t1 = new Thread() { public void run() { work_helper(nbsi,"T1",1); } };
- t1.start();
- work_helper(nbsi,"T0",1);
- t1.join();
-
- // In the end, all members should be removed
- StringBuffer buf = new StringBuffer();
- buf.append("Should be emptyset but has these elements: {");
- boolean found = false;
- for( Integer x : nbsi ) {
- buf.append(" ").append(x);
- found = true;
- }
- if( found ) System.out.println(buf);
- assertThat( "concurrent size=0", nbsi.size(), is(0) );
- for( Integer x : nbsi ) {
- assertTrue("No elements so never get here",false);
- }
-
- }
-
- void work_helper(NonBlockingSetInt nbsi, String thrd, int d) {
- final int ITERS = 100000;
- for( int j=0; j<10; j++ ) {
- long start = System.nanoTime();
- for( int i=d; i<ITERS; i+=2 )
- nbsi.add(i);
- for( int i=d; i<ITERS; i+=2 )
- nbsi.remove(i);
- double delta_nanos = System.nanoTime()-start;
- double delta_secs = delta_nanos/1000000000.0;
- double ops = ITERS*2;
- //System.out.println("Thrd"+thrd+" "+(ops/delta_secs)+" ops/sec size="+nbsi.size());
- }
- }
-}
+package org.cliffc.high_scale_lib;
+
+import junit.framework.TestCase;
+
+import java.io.*;
+import java.util.Iterator;
+
+import static org.hamcrest.CoreMatchers.anyOf;
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertThat;
+
+public class NonBlockingSetIntTest extends TestCase {
+
+ public void testSetOperations() {
+ // add values to the following bitsets
+ // 'a' will get all values divisible by 63
+ // 'b' will get the rest
+ // this ensures that one NBSI will have the largest bit array, while the other
+ // will have the largest recursive bitset used to store values for every 64th bit
+ // this exercises that the constructor building a blank NBSI to hold bitwise operations
+ // on two others is sized appropriately initially
+ NonBlockingSetInt a = new NonBlockingSetInt();
+ NonBlockingSetInt b = new NonBlockingSetInt();
+ int max = 10000;
+
+ for(int i = 0; i < max; i++) {
+ NonBlockingSetInt t = (i&63) == 63 ? a : b;
+ t.add(i);
+ assertTrue(t.contains(i));
+ }
+
+ // c should contain the empty set since a and b are disjoint
+ NonBlockingSetInt c = a.intersect(b);
+ NonBlockingSetInt d = b.intersect(a);
+ for(int i = 0; i < max; i++) {
+ assertFalse(c.contains(i));
+ assertFalse(d.contains(i));
+ }
+
+ c = a.union(b);
+ d = b.union(a);
+ for(int i = 0; i < max; i++) {
+ assertTrue(c.contains(i));
+ assertTrue(d.contains(i));
+ }
+
+ // just make sure the bitset is usable after building out an instance with an ample internal buffer
+ for(int i = 0; i < max; i++) {
+ c.add(i);
+ assertTrue(c.contains(i));
+ d.add(i);
+ assertTrue(d.contains(i));
+ }
+
+ c.intersect(new NonBlockingSetInt());
+ new NonBlockingSetInt().intersect(c);
+ }
+
+ private NonBlockingSetInt _nbsi;
+ protected void setUp () { _nbsi = new NonBlockingSetInt(); }
+ protected void tearDown() { _nbsi = null; }
+
+ // Test some basic stuff; add a few keys, remove a few keys
+ public void testBasic() {
+ assertTrue ( _nbsi.isEmpty() );
+ assertTrue ( _nbsi.add(1) );
+ checkSizes (1);
+ assertTrue ( _nbsi.add(2) );
+ checkSizes (2);
+ assertFalse( _nbsi.add(1) );
+ assertFalse( _nbsi.add(2) );
+ checkSizes (2);
+ assertThat ( _nbsi.remove(1), is(true ) );
+ checkSizes (1);
+ assertThat ( _nbsi.remove(1), is(false) );
+ assertTrue ( _nbsi.remove(2) );
+ checkSizes (0);
+ assertFalse( _nbsi.remove(2) );
+ assertFalse( _nbsi.remove(3) );
+ assertTrue ( _nbsi.isEmpty() );
+ assertTrue ( _nbsi.add(63) );
+ checkSizes (1);
+ assertTrue ( _nbsi.remove(63) );
+ assertFalse( _nbsi.remove(63) );
+
+
+ assertTrue ( _nbsi.isEmpty() );
+ assertTrue ( _nbsi.add(10000) );
+ checkSizes (1);
+ assertTrue ( _nbsi.add(20000) );
+ checkSizes (2);
+ assertFalse( _nbsi.add(10000) );
+ assertFalse( _nbsi.add(20000) );
+ checkSizes (2);
+ assertThat ( _nbsi.remove(10000), is(true ) );
+ checkSizes (1);
+ assertThat ( _nbsi.remove(10000), is(false) );
+ assertTrue ( _nbsi.remove(20000) );
+ checkSizes (0);
+ assertFalse( _nbsi.remove(20000) );
+ }
+
+ // Check all iterators for correct size counts
+ private void checkSizes(int expectedSize) {
+ assertEquals( "size()", _nbsi.size(), expectedSize );
+ Iterator it = _nbsi.iterator();
+ int result = 0;
+ while (it.hasNext()) {
+ result++;
+ it.next();
+ }
+ assertEquals( "iterator missed", expectedSize, result );
+ }
+
+
+ public void testIteration() {
+ assertTrue ( _nbsi.isEmpty() );
+ assertTrue ( _nbsi.add(1) );
+ assertTrue ( _nbsi.add(2) );
+
+ StringBuffer buf = new StringBuffer();
+ for( Iterator<Integer> i = _nbsi.iterator(); i.hasNext(); ) {
+ Integer val = i.next();
+ buf.append(val);
+ }
+ assertThat("found all vals",buf.toString(),anyOf(is("12"),is("21")));
+
+ assertThat("toString works",_nbsi.toString(), anyOf(is("[1, 2]"),is("[2, 1]")));
+ }
+
+ public void testIterationBig() {
+ for( int i=0; i<100; i++ )
+ _nbsi.add(i);
+ assertThat( _nbsi.size(), is(100) );
+
+ int sz =0;
+ int sum = 0;
+ for( Integer x : _nbsi ) {
+ sz++;
+ sum += x;
+ assertTrue(x>=0 && x<=99);
+ }
+ assertThat("Found 100 ints",sz,is(100));
+ assertThat("Found all integers in list",sum,is(100*99/2));
+
+ assertThat( "can remove 3", _nbsi.remove(3), is(true) );
+ assertThat( "can remove 4", _nbsi.remove(4), is(true) );
+ sz =0;
+ sum = 0;
+ for( Integer x : _nbsi ) {
+ sz++;
+ sum += x;
+ assertTrue(x>=0 && x<=99);
+ }
+ assertThat("Found 98 ints",sz,is(98));
+ assertThat("Found all integers in list",sum,is(100*99/2 - (3+4)));
+
+ }
+
+ public void testSerial() {
+ assertTrue ( _nbsi.isEmpty() );
+ assertTrue ( _nbsi.add(1) );
+ assertTrue ( _nbsi.add(2) );
+
+ // Serialize it out
+ try {
+ FileOutputStream fos = new FileOutputStream("NBSI_test.txt");
+ ObjectOutputStream out = new ObjectOutputStream(fos);
+ out.writeObject(_nbsi);
+ out.close();
+ } catch(IOException ex) {
+ ex.printStackTrace();
+ }
+
+ // Read it back
+ try {
+ File f = new File("NBSI_test.txt");
+ FileInputStream fis = new FileInputStream(f);
+ ObjectInputStream in = new ObjectInputStream(fis);
+ NonBlockingSetInt nbsi = (NonBlockingSetInt)in.readObject();
+ in.close();
+ assertEquals(_nbsi.toString(),nbsi.toString());
+ if( !f.delete() ) throw new IOException("delete failed");
+ } catch(IOException ex) {
+ ex.printStackTrace();
+ } catch(ClassNotFoundException ex) {
+ ex.printStackTrace();
+ }
+ }
+
+ // Do some simple concurrent testing
+ public void testConcurrentSimple() throws InterruptedException {
+ final NonBlockingSetInt nbsi = new NonBlockingSetInt();
+
+ // In 2 threads, add & remove even & odd elements concurrently
+ Thread t1 = new Thread() { public void run() { work_helper(nbsi,"T1",1); } };
+ t1.start();
+ work_helper(nbsi,"T0",1);
+ t1.join();
+
+ // In the end, all members should be removed
+ StringBuffer buf = new StringBuffer();
+ buf.append("Should be emptyset but has these elements: {");
+ boolean found = false;
+ for( Integer x : nbsi ) {
+ buf.append(" ").append(x);
+ found = true;
+ }
+ if( found ) System.out.println(buf);
+ assertThat( "concurrent size=0", nbsi.size(), is(0) );
+ for( Integer x : nbsi ) {
+ assertTrue("No elements so never get here",false);
+ }
+
+ }
+
+ void work_helper(NonBlockingSetInt nbsi, String thrd, int d) {
+ final int ITERS = 100000;
+ for( int j=0; j<10; j++ ) {
+ long start = System.nanoTime();
+ for( int i=d; i<ITERS; i+=2 )
+ nbsi.add(i);
+ for( int i=d; i<ITERS; i+=2 )
+ nbsi.remove(i);
+ double delta_nanos = System.nanoTime()-start;
+ double delta_secs = delta_nanos/1000000000.0;
+ double ops = ITERS*2;
+ //System.out.println("Thrd"+thrd+" "+(ops/delta_secs)+" ops/sec size="+nbsi.size());
+ }
+ }
+}
Please sign in to comment.
Something went wrong with that request. Please try again.