From 913dbbe17d8e06da69decd275aedcab3064380a9 Mon Sep 17 00:00:00 2001 From: Nick Tolomiczenko Date: Wed, 1 Aug 2012 20:30:02 -0400 Subject: [PATCH 01/39] Created HyperClient aware ByteArrayKeyedMap java class Signed-off-by: Nick Tolomiczenko --- .../java/extra_src/ByteArrayKeyedMap.java | 68 +++++++++++++++++++ hyperclient/java/proxies/HyperClient.i | 16 +---- 2 files changed, 71 insertions(+), 13 deletions(-) create mode 100644 hyperclient/java/extra_src/ByteArrayKeyedMap.java diff --git a/hyperclient/java/extra_src/ByteArrayKeyedMap.java b/hyperclient/java/extra_src/ByteArrayKeyedMap.java new file mode 100644 index 00000000..f671d125 --- /dev/null +++ b/hyperclient/java/extra_src/ByteArrayKeyedMap.java @@ -0,0 +1,68 @@ +package hyperclient; + +import java.util.*; +import java.io.UnsupportedEncodingException; + +public class ByteArrayKeyedMap extends HashMap +{ + private String defaultEncoding = "UTF-8"; + + public ByteArrayKeyedMap() + { + super(); + } + + public ByteArrayKeyedMap(String defaultEncoding) + { + super(); + this.defaultEncoding = defaultEncoding; + } + + private Object getByteArrayObjectFromObject(Object o) + { + + try + { + if ( o instanceof byte[] ) + { + return new ByteArray((byte[])o); + } + else if ( o instanceof ByteArray ) + { + return (ByteArray)o; + } + else if ( o instanceof String ) + { + return new ByteArray((String)o,defaultEncoding); + } + else + { + return o; + } + } + catch(TypeError e) + { + return o; + } + } + + public boolean containsKey(Object key) + { + return super.containsKey(getByteArrayObjectFromObject(key)); + } + + public Object get(Object key) + { + return super.get(getByteArrayObjectFromObject(key)); + } + + public void put(byte[] keyBytes, Object val) + { + super.put(new ByteArray(keyBytes),val); + } + + public Object remove(Object key) + { + return super.remove(getByteArrayObjectFromObject(key)); + } +} diff --git a/hyperclient/java/proxies/HyperClient.i b/hyperclient/java/proxies/HyperClient.i index e28782b5..779a110f 100644 --- a/hyperclient/java/proxies/HyperClient.i +++ b/hyperclient/java/proxies/HyperClient.i @@ -290,7 +290,7 @@ java.util.Map attrs_to_dict(hyperclient_attribute attrs, long attrs_sz) throws ValueError { - java.util.HashMap map = new java.util.HashMap(); + ByteArrayKeyedMap map = new ByteArrayKeyedMap(defaultStringEncoding); int sz = HyperClient.size_t_to_int(attrs_sz); @@ -298,21 +298,11 @@ { hyperclient_attribute ha = get_attr(attrs,i); - String attrName = null; - - try - { - attrName = new String(ha.getAttrNameBytes(),defaultStringEncoding); - } - catch(java.io.UnsupportedEncodingException usee) - { - throw new ValueError("Could not decode bytes using encoding '" - + defaultStringEncoding +"'"); - } + byte[] attrBytes = ha.getAttrNameBytes(); Object attrValue = ha.getAttrValue(defaultStringEncoding); - map.put(attrName, attrValue); + map.put(attrBytes, attrValue); } return map; From f65bd005a02bd4dd552769b253d4b70644746c45 Mon Sep 17 00:00:00 2001 From: Nick Tolomiczenko Date: Thu, 2 Aug 2012 16:11:23 -0400 Subject: [PATCH 02/39] More user-friendly string comparisons Retieved ByteArray map keys and ByteArray collection elements can now also be key-looked-up and compared with byte[]'s and java String's. You don't have to turn them into a ByteArray object beforehand anymore. Signed-off-by: Nick Tolomiczenko --- .../java/extra_src/ByteArrayKeyedMap.java | 13 +++- .../extra_src/ByteArrayKeyedSortedMap.java | 73 +++++++++++++++++++ .../java/extra_src/ByteArraySortedSet.java | 63 ++++++++++++++++ .../java/extra_src/ByteArrayVector.java | 63 ++++++++++++++++ hyperclient/java/proxies/HyperClient.i | 2 +- .../java/proxies/hyperclient_attribute.i | 15 ++-- 6 files changed, 217 insertions(+), 12 deletions(-) create mode 100644 hyperclient/java/extra_src/ByteArrayKeyedSortedMap.java create mode 100644 hyperclient/java/extra_src/ByteArraySortedSet.java create mode 100644 hyperclient/java/extra_src/ByteArrayVector.java diff --git a/hyperclient/java/extra_src/ByteArrayKeyedMap.java b/hyperclient/java/extra_src/ByteArrayKeyedMap.java index f671d125..908d2add 100644 --- a/hyperclient/java/extra_src/ByteArrayKeyedMap.java +++ b/hyperclient/java/extra_src/ByteArrayKeyedMap.java @@ -3,7 +3,7 @@ import java.util.*; import java.io.UnsupportedEncodingException; -public class ByteArrayKeyedMap extends HashMap +public class ByteArrayKeyedMap extends HashMap { private String defaultEncoding = "UTF-8"; @@ -51,17 +51,22 @@ public boolean containsKey(Object key) return super.containsKey(getByteArrayObjectFromObject(key)); } - public Object get(Object key) + public V get(Object key) { return super.get(getByteArrayObjectFromObject(key)); } - public void put(byte[] keyBytes, Object val) + public void put(byte[] keyBytes, V val) { super.put(new ByteArray(keyBytes),val); } - public Object remove(Object key) + public void put(String keyStr, V val) throws TypeError + { + super.put(new ByteArray(keyStr,defaultEncoding),val); + } + + public V remove(Object key) { return super.remove(getByteArrayObjectFromObject(key)); } diff --git a/hyperclient/java/extra_src/ByteArrayKeyedSortedMap.java b/hyperclient/java/extra_src/ByteArrayKeyedSortedMap.java new file mode 100644 index 00000000..ae3eaea2 --- /dev/null +++ b/hyperclient/java/extra_src/ByteArrayKeyedSortedMap.java @@ -0,0 +1,73 @@ +package hyperclient; + +import java.util.*; +import java.io.UnsupportedEncodingException; + +public class ByteArrayKeyedSortedMap extends TreeMap +{ + private String defaultEncoding = "UTF-8"; + + public ByteArrayKeyedSortedMap() + { + super(); + } + + public ByteArrayKeyedSortedMap(String defaultEncoding) + { + super(); + this.defaultEncoding = defaultEncoding; + } + + private Object getByteArrayObjectFromObject(Object o) + { + + try + { + if ( o instanceof byte[] ) + { + return new ByteArray((byte[])o); + } + else if ( o instanceof ByteArray ) + { + return (ByteArray)o; + } + else if ( o instanceof String ) + { + return new ByteArray((String)o,defaultEncoding); + } + else + { + return o; + } + } + catch(TypeError e) + { + return o; + } + } + + public boolean containsKey(Object key) + { + return super.containsKey(getByteArrayObjectFromObject(key)); + } + + public V get(Object key) + { + return super.get(getByteArrayObjectFromObject(key)); + } + + public void put(byte[] keyBytes, V val) + { + super.put(new ByteArray(keyBytes),val); + } + + public void put(String keyStr, V val) throws TypeError + { + super.put(new ByteArray(keyStr,defaultEncoding),val); + } + + public V remove(Object key) + { + return super.remove(getByteArrayObjectFromObject(key)); + } +} diff --git a/hyperclient/java/extra_src/ByteArraySortedSet.java b/hyperclient/java/extra_src/ByteArraySortedSet.java new file mode 100644 index 00000000..43a5734c --- /dev/null +++ b/hyperclient/java/extra_src/ByteArraySortedSet.java @@ -0,0 +1,63 @@ +package hyperclient; + +import java.util.*; +import java.io.UnsupportedEncodingException; + +public class ByteArraySortedSet extends TreeSet +{ + private String defaultEncoding = "UTF-8"; + + public ByteArraySortedSet() + { + super(); + } + + public ByteArraySortedSet(String defaultEncoding) + { + super(); + this.defaultEncoding = defaultEncoding; + } + + private Object getByteArrayObjectFromObject(Object o) + { + + try + { + if ( o instanceof byte[] ) + { + return new ByteArray((byte[])o); + } + else if ( o instanceof ByteArray ) + { + return (ByteArray)o; + } + else if ( o instanceof String ) + { + return new ByteArray((String)o,defaultEncoding); + } + else + { + return o; + } + } + catch(TypeError e) + { + return o; + } + } + + public void add(byte[] elementBytes) + { + super.add(new ByteArray(elementBytes)); + } + + public void add(String elementStr) throws TypeError + { + super.add(new ByteArray(elementStr,defaultEncoding)); + } + + public boolean contains(Object element) + { + return super.contains(getByteArrayObjectFromObject(element)); + } +} diff --git a/hyperclient/java/extra_src/ByteArrayVector.java b/hyperclient/java/extra_src/ByteArrayVector.java new file mode 100644 index 00000000..6d5e10fc --- /dev/null +++ b/hyperclient/java/extra_src/ByteArrayVector.java @@ -0,0 +1,63 @@ +package hyperclient; + +import java.util.*; +import java.io.UnsupportedEncodingException; + +public class ByteArrayVector extends Vector +{ + private String defaultEncoding = "UTF-8"; + + public ByteArrayVector() + { + super(); + } + + public ByteArrayVector(String defaultEncoding) + { + super(); + this.defaultEncoding = defaultEncoding; + } + + private Object getByteArrayObjectFromObject(Object o) + { + + try + { + if ( o instanceof byte[] ) + { + return new ByteArray((byte[])o); + } + else if ( o instanceof ByteArray ) + { + return (ByteArray)o; + } + else if ( o instanceof String ) + { + return new ByteArray((String)o,defaultEncoding); + } + else + { + return o; + } + } + catch(TypeError e) + { + return o; + } + } + + public void add(byte[] elementBytes) + { + super.add(new ByteArray(elementBytes)); + } + + public void add(String elementStr) throws TypeError + { + super.add(new ByteArray(elementStr,defaultEncoding)); + } + + public boolean contains(Object element) + { + return super.contains(getByteArrayObjectFromObject(element)); + } +} diff --git a/hyperclient/java/proxies/HyperClient.i b/hyperclient/java/proxies/HyperClient.i index 779a110f..2a7d39b5 100644 --- a/hyperclient/java/proxies/HyperClient.i +++ b/hyperclient/java/proxies/HyperClient.i @@ -290,7 +290,7 @@ java.util.Map attrs_to_dict(hyperclient_attribute attrs, long attrs_sz) throws ValueError { - ByteArrayKeyedMap map = new ByteArrayKeyedMap(defaultStringEncoding); + ByteArrayKeyedMap map = new ByteArrayKeyedMap(defaultStringEncoding); int sz = HyperClient.size_t_to_int(attrs_sz); diff --git a/hyperclient/java/proxies/hyperclient_attribute.i b/hyperclient/java/proxies/hyperclient_attribute.i index f8666942..7b9c0456 100644 --- a/hyperclient/java/proxies/hyperclient_attribute.i +++ b/hyperclient/java/proxies/hyperclient_attribute.i @@ -205,8 +205,8 @@ private java.lang.Object getAttrMapStringStringValue(String defaultStringEncoding) throws ValueError { - java.util.HashMap map - = new java.util.HashMap(); + ByteArrayKeyedSortedMap map + = new ByteArrayKeyedSortedMap(); // Interpret return value of getValue_sz() as unsigned // @@ -275,7 +275,8 @@ private java.lang.Object getAttrMapStringLongValue(String defaultStringEncoding) throws ValueError { - java.util.HashMap map = new java.util.HashMap(); + ByteArrayKeyedSortedMap map + = new ByteArrayKeyedSortedMap(); // Interpret return value of getValue_sz() as unsigned // @@ -340,8 +341,8 @@ private java.lang.Object getAttrMapStringDoubleValue(String defaultStringEncoding) throws ValueError { - java.util.HashMap - map = new java.util.HashMap(); + ByteArrayKeyedSortedMap + map = new ByteArrayKeyedSortedMap(); // Interpret return value of getValue_sz() as unsigned // @@ -715,7 +716,7 @@ return getAttrDoubleValue(); case HYPERDATATYPE_LIST_STRING: - java.util.Vector ls = new java.util.Vector(); + ByteArrayVector ls = new ByteArrayVector(); getAttrCollectionStringValue(ls,defaultStringEncoding); return ls; @@ -730,7 +731,7 @@ return lf; case HYPERDATATYPE_SET_STRING: - java.util.HashSet ss = new java.util.HashSet(); + ByteArraySortedSet ss = new ByteArraySortedSet(); getAttrCollectionStringValue(ss,defaultStringEncoding); return ss; From a5fa6b6899ed36584d262734c56da0f6c3f4343d Mon Sep 17 00:00:00 2001 From: Nick Tolomiczenko Date: Fri, 5 Oct 2012 16:51:58 -0400 Subject: [PATCH 03/39] Java bindings are now up to date with range search using floats Signed-off-by: Nick Tolomiczenko --- .../java/examples/HyperSearchFloatRange.java | 90 +++++++++++++++++++ hyperclient/java/examples/README.txt | 1 + hyperclient/java/proxies/HyperClient.i | 87 +++++++++++++----- .../java/proxies/hyperclient_range_query.i | 5 ++ 4 files changed, 159 insertions(+), 24 deletions(-) create mode 100644 hyperclient/java/examples/HyperSearchFloatRange.java diff --git a/hyperclient/java/examples/HyperSearchFloatRange.java b/hyperclient/java/examples/HyperSearchFloatRange.java new file mode 100644 index 00000000..308f4bcc --- /dev/null +++ b/hyperclient/java/examples/HyperSearchFloatRange.java @@ -0,0 +1,90 @@ +/* This example uses space definition */ +/* +space weightbook +dimensions username, first, last, weight (float) +key username auto 1 3 +subspace first, last, weight auto 2 3 +*/ + + +import hyperclient.*; +import java.util.*; + +public class HyperSearchFloatRange +{ + public static void main(String[] args) throws Exception + { + HashMap values = new HashMap(); + + HyperClient c = new HyperClient("127.0.0.1",1234); + + values.put("first","Nick"); + values.put("last","Tolomiczenko"); + values.put("weight",175.2); + + System.out.println("ntolomic put: " + c.put("weightbook","ntolomic",values)); + + values.put("first","George"); + values.put("last","Tolomiczenko"); + values.put("weight",160.7); + + System.out.println("gtolomic put: " + c.put("weightbook","gtolomic",values)); + + values.put("first","Paul"); + values.put("last","Tolomiczenko"); + values.put("weight",162.3); + + System.out.println("ptolomic put: " + c.put("weightbook","ptolomic",values)); + + System.out.println("\nAbout retrieve key ntolomic:\n"); + + Map row = c.get("weightbook","ntolomic"); + + System.out.println("Got back: " + row); + + System.out.println("\nSearching for last name of 'Tolomiczenko':\n"); + + // We'll use the same 'values' map for our search predicate + + // This leaves us with only the last name entry + values.remove("first"); + values.remove("weight"); + + SearchBase s = c.search("weightbook",values); + + while(s.hasNext()) + { + System.out.println(s.next()); + } + + // Now add a range stipulation on the weight: [160.1,165) + values.put("weight", + new AbstractMap.SimpleEntry(160.1D,165D)); + + System.out.println("\nSearching for last name of 'Tolomiczenko'\n\n AND\n\nweight in the range [160.1,165):\n"); + + // Do the search again + s = c.search("weightbook",values); + + while(s.hasNext()) + { + System.out.println(s.next()); + } + + // Stipulate the range with a List of size 2 instead. + // In particular, use in Vector. + Vector vrange = new Vector(2); + vrange.add(160.1D); vrange.add(165D); + values.put("weight",vrange); + + System.out.println("\nDo the search again using a List of size 2 for the range stipulation:\n"); + // Do the search again + s = c.search("weightbook",values); + + while(s.hasNext()) + { + System.out.println(s.next()); + } + + } +} diff --git a/hyperclient/java/examples/README.txt b/hyperclient/java/examples/README.txt index f0f647b1..92c2ede2 100644 --- a/hyperclient/java/examples/README.txt +++ b/hyperclient/java/examples/README.txt @@ -4,3 +4,4 @@ INSTRUCTIONS: javac HyperTest.java javac HyperAllTypes.java javac HyperBinaryTest.java +javac HyperSearchFloatRange.java diff --git a/hyperclient/java/proxies/HyperClient.i b/hyperclient/java/proxies/HyperClient.i index 2a7d39b5..d6315350 100644 --- a/hyperclient/java/proxies/HyperClient.i +++ b/hyperclient/java/proxies/HyperClient.i @@ -202,8 +202,23 @@ if ((buf = (char *)calloc(attr_sz+1,sizeof(char))) == NULL) return 0; memcpy(buf,attr,attr_sz); rq->attr = buf; - rq->lower = lower; - rq->upper = upper; + (rq->lower_t).i = lower; + (rq->upper_t).i = upper; + return 1; + } + + static int write_range_query(hyperclient_range_query *rq, + const char *attr, size_t attr_sz, + double lower, + double upper) + { + char *buf; + + if ((buf = (char *)calloc(attr_sz+1,sizeof(char))) == NULL) return 0; + memcpy(buf,attr,attr_sz); + rq->attr = buf; + (rq->lower_t).d = lower; + (rq->upper_t).d = upper; return 1; } @@ -403,6 +418,30 @@ return retType; } + // write_range_query private static method expects (lower,upper) to be of types + // (Long, Long) or (Double,Double) + // + private static void write_range_query(hyperclient_range_query rq, byte[] attr, + Object lower, Object upper) throws MemoryError + { + if ( lower instanceof Long ) // then upper MUST be Long at this point + { + if ( HyperClient.write_range_query(rq,attr, + ((Long)lower).longValue(),((Long)upper).longValue()) == 0 ) + { + throw new MemoryError(); + } + } + else // lower and upper must both be Double at this point + { + if ( HyperClient.write_range_query(rq,attr, + ((Double)lower).doubleValue(),((Double)upper).doubleValue()) == 0 ) + { + throw new MemoryError(); + } + } + } + // Using a Vector retvals to return multiple values. // // retvals at 0 - eq @@ -443,7 +482,7 @@ throw new TypeError("Cannot search with a null criteria"); - String errStr = "Attribute '" + attrStr + "' has incorrect type ( expected Long, Double, String, Map.Entry or List (of size 2), but got %s"; + String errStr = "Attribute '" + attrStr + "' has incorrect type ( expected Long, Double, String, Map.Entry, Map.Entry, List or List (List being of size 2), but got %s"; if ( isBytes(params) || params instanceof Long || params instanceof Double ) { @@ -453,19 +492,15 @@ { if ( params instanceof java.util.Map.Entry ) { - try - { - long lower - = ((Long)((java.util.Map.Entry)params).getKey()).longValue(); + Object lower = ((java.util.Map.Entry)params).getKey(); + Object upper = ((java.util.Map.Entry)params).getValue(); - long upper - = ((Long)((java.util.Map.Entry)params).getValue()).longValue(); - } - catch(Exception e) + if ( ! ( lower instanceof Long && upper instanceof Long ) && + ! ( lower instanceof Double && upper instanceof Double ) ) { throw new TypeError( - String.format(errStr,params.getClass().getName())); + String.format(errStr,params.getClass().getName())); } } else if ( params instanceof java.util.List ) @@ -481,7 +516,12 @@ { throw te; } - catch (Exception e) + + Object lower = ((java.util.List)params).get(0); + Object upper = ((java.util.List)params).get(1); + + if ( ! ( lower instanceof Long && upper instanceof Long ) && + ! ( lower instanceof Double && upper instanceof Double ) ) { throw new TypeError( @@ -524,24 +564,23 @@ Object params = ranges.get(attr); - long lower = 0; - long upper = 0; + hyperclient_range_query rq = HyperClient.get_range_query(rn,i); if ( params instanceof java.util.Map.Entry ) { - lower = (Long)(((java.util.Map.Entry)params).getKey()); - upper = (Long)(((java.util.Map.Entry)params).getValue()); + Object lower = ((java.util.Map.Entry)params).getKey(); + Object upper = ((java.util.Map.Entry)params).getValue(); + + write_range_query(rq,attr.getBytes(),lower,upper); + } else // Must be a List of Longs of size = 2 { - lower = (Long)(((java.util.List)params).get(0)); - upper = (Long)(((java.util.List)params).get(1)); - } - - hyperclient_range_query rq = HyperClient.get_range_query(rn,i); + Object lower = ((java.util.List)params).get(0); + Object upper = ((java.util.List)params).get(1); - if ( HyperClient.write_range_query(rq,attr.getBytes(),lower,upper) == 0 ) - throw new MemoryError(); + write_range_query(rq,attr.getBytes(),lower,upper); + } i++; } diff --git a/hyperclient/java/proxies/hyperclient_range_query.i b/hyperclient/java/proxies/hyperclient_range_query.i index fca74ec5..95b76c83 100644 --- a/hyperclient/java/proxies/hyperclient_range_query.i +++ b/hyperclient/java/proxies/hyperclient_range_query.i @@ -3,6 +3,11 @@ // %ignore hyperclient_range_query::attr; +// No need to expose upper_t and lower_t in java +// +%ignore hyperclient_range_query::upper_t; +%ignore hyperclient_range_query::lower_t; + %typemap(javacode) hyperclient_range_query %{ byte[] getRangeQueryAttrNameBytes() From fc45a5c467673cbb75e232c34a5bec73e46583c7 Mon Sep 17 00:00:00 2001 From: Nick Tolomiczenko Date: Tue, 1 Jan 2013 18:19:24 -0500 Subject: [PATCH 04/39] Began fixing broken java bindings Signed-off-by: Nick Tolomiczenko --- .../java/examples/HyperAllTypes.java | 0 .../java/examples/HyperBinaryTest.java | 0 .../java/examples/HyperSearchFloatRange.java | 0 .../java/examples/HyperTest.java | 0 .../java/examples/README.txt | 0 .../java/examples/source_me | 0 .../java/extra_src/ByteArray.java | 0 .../java/extra_src/ByteArrayKeyedMap.java | 0 .../extra_src/ByteArrayKeyedSortedMap.java | 0 .../java/extra_src/ByteArraySortedSet.java | 0 .../java/extra_src/ByteArrayVector.java | 0 .../java/extra_src/Deferred.java | 0 .../java/extra_src/DeferredCondPut.java | 0 .../java/extra_src/DeferredCount.java | 0 .../java/extra_src/DeferredDelete.java | 0 .../java/extra_src/DeferredFromAttrs.java | 0 .../java/extra_src/DeferredGet.java | 0 .../java/extra_src/DeferredGroupDel.java | 0 .../java/extra_src/DeferredMapOp.java | 0 .../java/extra_src/HyperClientException.java | 0 .../java/extra_src/MapOp.java | 0 .../java/extra_src/MemoryError.java | 0 .../java/extra_src/Pending.java | 0 .../java/extra_src/Search.java | 0 .../java/extra_src/SearchBase.java | 0 .../java/extra_src/SimpleOp.java | 0 .../java/extra_src/SortedSearch.java | 0 .../java/extra_src/TypeError.java | 0 .../java/extra_src/ValueError.java | 0 .../java/gen_code/MapOp.template.java | 0 .../java/gen_code/SimpleOp.template.java | 0 .../java/gen_code/gen_op_files | 0 .../java/gen_code/op_list.txt | 0 {hyperclient => client}/java/hyperclient.i | 6 +- .../java/proxies/HyperClient.i | 77 +- .../java/proxies/hyperclient_attribute.i | 0 .../proxies/hyperclient_attribute_check.i | 779 ++++++++++++++++++ .../java/proxies/hyperclient_map_attribute.i | 0 .../java/proxies/hyperclient_range_query.i | 22 - 39 files changed, 828 insertions(+), 56 deletions(-) rename {hyperclient => client}/java/examples/HyperAllTypes.java (100%) rename {hyperclient => client}/java/examples/HyperBinaryTest.java (100%) rename {hyperclient => client}/java/examples/HyperSearchFloatRange.java (100%) rename {hyperclient => client}/java/examples/HyperTest.java (100%) rename {hyperclient => client}/java/examples/README.txt (100%) rename {hyperclient => client}/java/examples/source_me (100%) rename {hyperclient => client}/java/extra_src/ByteArray.java (100%) rename {hyperclient => client}/java/extra_src/ByteArrayKeyedMap.java (100%) rename {hyperclient => client}/java/extra_src/ByteArrayKeyedSortedMap.java (100%) rename {hyperclient => client}/java/extra_src/ByteArraySortedSet.java (100%) rename {hyperclient => client}/java/extra_src/ByteArrayVector.java (100%) rename {hyperclient => client}/java/extra_src/Deferred.java (100%) rename {hyperclient => client}/java/extra_src/DeferredCondPut.java (100%) rename {hyperclient => client}/java/extra_src/DeferredCount.java (100%) rename {hyperclient => client}/java/extra_src/DeferredDelete.java (100%) rename {hyperclient => client}/java/extra_src/DeferredFromAttrs.java (100%) rename {hyperclient => client}/java/extra_src/DeferredGet.java (100%) rename {hyperclient => client}/java/extra_src/DeferredGroupDel.java (100%) rename {hyperclient => client}/java/extra_src/DeferredMapOp.java (100%) rename {hyperclient => client}/java/extra_src/HyperClientException.java (100%) rename {hyperclient => client}/java/extra_src/MapOp.java (100%) rename {hyperclient => client}/java/extra_src/MemoryError.java (100%) rename {hyperclient => client}/java/extra_src/Pending.java (100%) rename {hyperclient => client}/java/extra_src/Search.java (100%) rename {hyperclient => client}/java/extra_src/SearchBase.java (100%) rename {hyperclient => client}/java/extra_src/SimpleOp.java (100%) rename {hyperclient => client}/java/extra_src/SortedSearch.java (100%) rename {hyperclient => client}/java/extra_src/TypeError.java (100%) rename {hyperclient => client}/java/extra_src/ValueError.java (100%) rename {hyperclient => client}/java/gen_code/MapOp.template.java (100%) rename {hyperclient => client}/java/gen_code/SimpleOp.template.java (100%) rename {hyperclient => client}/java/gen_code/gen_op_files (100%) rename {hyperclient => client}/java/gen_code/op_list.txt (100%) rename {hyperclient => client}/java/hyperclient.i (96%) rename {hyperclient => client}/java/proxies/HyperClient.i (97%) rename {hyperclient => client}/java/proxies/hyperclient_attribute.i (100%) create mode 100644 client/java/proxies/hyperclient_attribute_check.i rename {hyperclient => client}/java/proxies/hyperclient_map_attribute.i (100%) delete mode 100644 hyperclient/java/proxies/hyperclient_range_query.i diff --git a/hyperclient/java/examples/HyperAllTypes.java b/client/java/examples/HyperAllTypes.java similarity index 100% rename from hyperclient/java/examples/HyperAllTypes.java rename to client/java/examples/HyperAllTypes.java diff --git a/hyperclient/java/examples/HyperBinaryTest.java b/client/java/examples/HyperBinaryTest.java similarity index 100% rename from hyperclient/java/examples/HyperBinaryTest.java rename to client/java/examples/HyperBinaryTest.java diff --git a/hyperclient/java/examples/HyperSearchFloatRange.java b/client/java/examples/HyperSearchFloatRange.java similarity index 100% rename from hyperclient/java/examples/HyperSearchFloatRange.java rename to client/java/examples/HyperSearchFloatRange.java diff --git a/hyperclient/java/examples/HyperTest.java b/client/java/examples/HyperTest.java similarity index 100% rename from hyperclient/java/examples/HyperTest.java rename to client/java/examples/HyperTest.java diff --git a/hyperclient/java/examples/README.txt b/client/java/examples/README.txt similarity index 100% rename from hyperclient/java/examples/README.txt rename to client/java/examples/README.txt diff --git a/hyperclient/java/examples/source_me b/client/java/examples/source_me similarity index 100% rename from hyperclient/java/examples/source_me rename to client/java/examples/source_me diff --git a/hyperclient/java/extra_src/ByteArray.java b/client/java/extra_src/ByteArray.java similarity index 100% rename from hyperclient/java/extra_src/ByteArray.java rename to client/java/extra_src/ByteArray.java diff --git a/hyperclient/java/extra_src/ByteArrayKeyedMap.java b/client/java/extra_src/ByteArrayKeyedMap.java similarity index 100% rename from hyperclient/java/extra_src/ByteArrayKeyedMap.java rename to client/java/extra_src/ByteArrayKeyedMap.java diff --git a/hyperclient/java/extra_src/ByteArrayKeyedSortedMap.java b/client/java/extra_src/ByteArrayKeyedSortedMap.java similarity index 100% rename from hyperclient/java/extra_src/ByteArrayKeyedSortedMap.java rename to client/java/extra_src/ByteArrayKeyedSortedMap.java diff --git a/hyperclient/java/extra_src/ByteArraySortedSet.java b/client/java/extra_src/ByteArraySortedSet.java similarity index 100% rename from hyperclient/java/extra_src/ByteArraySortedSet.java rename to client/java/extra_src/ByteArraySortedSet.java diff --git a/hyperclient/java/extra_src/ByteArrayVector.java b/client/java/extra_src/ByteArrayVector.java similarity index 100% rename from hyperclient/java/extra_src/ByteArrayVector.java rename to client/java/extra_src/ByteArrayVector.java diff --git a/hyperclient/java/extra_src/Deferred.java b/client/java/extra_src/Deferred.java similarity index 100% rename from hyperclient/java/extra_src/Deferred.java rename to client/java/extra_src/Deferred.java diff --git a/hyperclient/java/extra_src/DeferredCondPut.java b/client/java/extra_src/DeferredCondPut.java similarity index 100% rename from hyperclient/java/extra_src/DeferredCondPut.java rename to client/java/extra_src/DeferredCondPut.java diff --git a/hyperclient/java/extra_src/DeferredCount.java b/client/java/extra_src/DeferredCount.java similarity index 100% rename from hyperclient/java/extra_src/DeferredCount.java rename to client/java/extra_src/DeferredCount.java diff --git a/hyperclient/java/extra_src/DeferredDelete.java b/client/java/extra_src/DeferredDelete.java similarity index 100% rename from hyperclient/java/extra_src/DeferredDelete.java rename to client/java/extra_src/DeferredDelete.java diff --git a/hyperclient/java/extra_src/DeferredFromAttrs.java b/client/java/extra_src/DeferredFromAttrs.java similarity index 100% rename from hyperclient/java/extra_src/DeferredFromAttrs.java rename to client/java/extra_src/DeferredFromAttrs.java diff --git a/hyperclient/java/extra_src/DeferredGet.java b/client/java/extra_src/DeferredGet.java similarity index 100% rename from hyperclient/java/extra_src/DeferredGet.java rename to client/java/extra_src/DeferredGet.java diff --git a/hyperclient/java/extra_src/DeferredGroupDel.java b/client/java/extra_src/DeferredGroupDel.java similarity index 100% rename from hyperclient/java/extra_src/DeferredGroupDel.java rename to client/java/extra_src/DeferredGroupDel.java diff --git a/hyperclient/java/extra_src/DeferredMapOp.java b/client/java/extra_src/DeferredMapOp.java similarity index 100% rename from hyperclient/java/extra_src/DeferredMapOp.java rename to client/java/extra_src/DeferredMapOp.java diff --git a/hyperclient/java/extra_src/HyperClientException.java b/client/java/extra_src/HyperClientException.java similarity index 100% rename from hyperclient/java/extra_src/HyperClientException.java rename to client/java/extra_src/HyperClientException.java diff --git a/hyperclient/java/extra_src/MapOp.java b/client/java/extra_src/MapOp.java similarity index 100% rename from hyperclient/java/extra_src/MapOp.java rename to client/java/extra_src/MapOp.java diff --git a/hyperclient/java/extra_src/MemoryError.java b/client/java/extra_src/MemoryError.java similarity index 100% rename from hyperclient/java/extra_src/MemoryError.java rename to client/java/extra_src/MemoryError.java diff --git a/hyperclient/java/extra_src/Pending.java b/client/java/extra_src/Pending.java similarity index 100% rename from hyperclient/java/extra_src/Pending.java rename to client/java/extra_src/Pending.java diff --git a/hyperclient/java/extra_src/Search.java b/client/java/extra_src/Search.java similarity index 100% rename from hyperclient/java/extra_src/Search.java rename to client/java/extra_src/Search.java diff --git a/hyperclient/java/extra_src/SearchBase.java b/client/java/extra_src/SearchBase.java similarity index 100% rename from hyperclient/java/extra_src/SearchBase.java rename to client/java/extra_src/SearchBase.java diff --git a/hyperclient/java/extra_src/SimpleOp.java b/client/java/extra_src/SimpleOp.java similarity index 100% rename from hyperclient/java/extra_src/SimpleOp.java rename to client/java/extra_src/SimpleOp.java diff --git a/hyperclient/java/extra_src/SortedSearch.java b/client/java/extra_src/SortedSearch.java similarity index 100% rename from hyperclient/java/extra_src/SortedSearch.java rename to client/java/extra_src/SortedSearch.java diff --git a/hyperclient/java/extra_src/TypeError.java b/client/java/extra_src/TypeError.java similarity index 100% rename from hyperclient/java/extra_src/TypeError.java rename to client/java/extra_src/TypeError.java diff --git a/hyperclient/java/extra_src/ValueError.java b/client/java/extra_src/ValueError.java similarity index 100% rename from hyperclient/java/extra_src/ValueError.java rename to client/java/extra_src/ValueError.java diff --git a/hyperclient/java/gen_code/MapOp.template.java b/client/java/gen_code/MapOp.template.java similarity index 100% rename from hyperclient/java/gen_code/MapOp.template.java rename to client/java/gen_code/MapOp.template.java diff --git a/hyperclient/java/gen_code/SimpleOp.template.java b/client/java/gen_code/SimpleOp.template.java similarity index 100% rename from hyperclient/java/gen_code/SimpleOp.template.java rename to client/java/gen_code/SimpleOp.template.java diff --git a/hyperclient/java/gen_code/gen_op_files b/client/java/gen_code/gen_op_files similarity index 100% rename from hyperclient/java/gen_code/gen_op_files rename to client/java/gen_code/gen_op_files diff --git a/hyperclient/java/gen_code/op_list.txt b/client/java/gen_code/op_list.txt similarity index 100% rename from hyperclient/java/gen_code/op_list.txt rename to client/java/gen_code/op_list.txt diff --git a/hyperclient/java/hyperclient.i b/client/java/hyperclient.i similarity index 96% rename from hyperclient/java/hyperclient.i rename to client/java/hyperclient.i index 5e8197e7..14025ff0 100644 --- a/hyperclient/java/hyperclient.i +++ b/client/java/hyperclient.i @@ -38,7 +38,7 @@ %{ #include #include -#include "hyperclient/hyperclient.h" +#include "client/hyperclient.h" typedef hyperclient_attribute* hyperclient_attribute_asterisk; typedef hyperclient_range_query* hyperclient_range_query_asterisk; %} @@ -93,7 +93,7 @@ typedef hyperclient_range_query* hyperclient_range_query_asterisk; %apply (char *STRING, int LENGTH) { (const char *value, size_t value_sz) } -// Pertaining to the include of hyperdex.h and hyperclient/hyperclient.h below: +// Pertaining to the include of hyperdex.h and client/hyperclient.h below: // Ignore everything %ignore ""; @@ -116,4 +116,4 @@ typedef hyperclient_range_query* hyperclient_range_query_asterisk; %rename("%s", %$ismember) ""; %include "hyperdex.h" -%include "hyperclient/hyperclient.h" +%include "client/hyperclient.h" diff --git a/hyperclient/java/proxies/HyperClient.i b/client/java/proxies/HyperClient.i similarity index 97% rename from hyperclient/java/proxies/HyperClient.i rename to client/java/proxies/HyperClient.i index d6315350..beaac614 100644 --- a/hyperclient/java/proxies/HyperClient.i +++ b/client/java/proxies/HyperClient.i @@ -36,20 +36,28 @@ memcpy(name, str.data(), name_sz); } - static int get_range_query_attr_name_sz(hyperclient_range_query *rq) + static int get_attr_check_name_sz(hyperclient_attribute_check *hac) { - std::string str = std::string(rq->attr); + std::string str = std::string(hac->attr); size_t name_sz = str.length(); return name_sz > INT_MAX ? (int)INT_MAX : (int)name_sz; } - static std::string read_range_query_attr_name(hyperclient_range_query *rq, - char *name, size_t name_sz) + static void read_attr_check_name(hyperclient_attribute_check *hac, + char *name, size_t name_sz) { - std::string str = std::string(rq->attr); + std::string str = std::string(hac->attr); memcpy(name, str.data(), name_sz); } + static void read_attr_check_value(hyperclient_attribute_check *hac, + char *value, size_t value_sz, + size_t pos) + { + size_t available = hac->value_sz - pos; + memcpy(value, hac->value+pos, value_szattr will point to allocated memory @@ -192,33 +201,39 @@ return 1; } - static int write_range_query(hyperclient_range_query *rq, + // Returns 1 on success. hac->attr will point to allocated memory + // Returns 0 on failure. hac->attr will be NULL + static int write_attr_check_name(hyperclient_attribute_check *hac, const char *attr, size_t attr_sz, - int64_t lower, - int64_t upper) + hyperdatatype type, hyperpredicate pred) { char *buf; if ((buf = (char *)calloc(attr_sz+1,sizeof(char))) == NULL) return 0; memcpy(buf,attr,attr_sz); - rq->attr = buf; - (rq->lower_t).i = lower; - (rq->upper_t).i = upper; + hac->attr = buf; + hac->datatype = type; + hac->predicate = pred; return 1; } - static int write_range_query(hyperclient_range_query *rq, - const char *attr, size_t attr_sz, - double lower, - double upper) + // Returns 1 on success. hac->value will point to allocated memory + // hac->value_sz will hold the size of this memory + // Returns 0 on failure. hac->value will be NULL + // hac->value_sz will be 0 + // + // If hac->value is already non-NULL, then we are appending to it. + static int write_attr_check_value(hyperclient_attribute_check *hac, + const char *value, size_t value_sz) { - char *buf; - - if ((buf = (char *)calloc(attr_sz+1,sizeof(char))) == NULL) return 0; - memcpy(buf,attr,attr_sz); - rq->attr = buf; - (rq->lower_t).d = lower; - (rq->upper_t).d = upper; + char *buf = NULL; + // Note: Since hyperclient_attribute_check array was calloced + // hac->value = NULL and hac->value_sz = 0 initially + if ((buf = (char *)realloc((void *)(hac->value), hac->value_sz + value_sz)) + == NULL) return 0; + memcpy(buf + hac->value_sz, value, value_sz); + hac->value = buf; + hac->value_sz += value_sz; return 1; } @@ -232,9 +247,9 @@ return hma + i; } - static hyperclient_range_query *get_range_query(hyperclient_range_query *rqs, size_t i) + static hyperclient_attribute_check *get_attr_check(hyperclient_attribute *hac, size_t i) { - return rqs + i; + return hac + i; } }; diff --git a/hyperclient/java/proxies/hyperclient_attribute.i b/client/java/proxies/hyperclient_attribute.i similarity index 100% rename from hyperclient/java/proxies/hyperclient_attribute.i rename to client/java/proxies/hyperclient_attribute.i diff --git a/client/java/proxies/hyperclient_attribute_check.i b/client/java/proxies/hyperclient_attribute_check.i new file mode 100644 index 00000000..c92799a6 --- /dev/null +++ b/client/java/proxies/hyperclient_attribute_check.i @@ -0,0 +1,779 @@ +// Write our own (safe) getters and setters for attr and value +// in order to avoid warnings +// +%ignore hyperclient_attribute_check::attr; +%ignore hyperclient_attribute_check::value; + +%typemap(javacode) hyperclient_attribute_check +%{ + byte[] getAttrNameBytes() + { + int name_sz = HyperClient.get_attr_name_sz(this); + byte[] bytes = new byte[name_sz]; + + HyperClient.read_attr_name(this,bytes); + + return bytes; + } + + private byte[] getAttrValueBytes() + { + int bytes_sz = HyperClient.size_t_to_int(getValue_sz()); + + return getAttrValueBytes(0,bytes_sz); + } + + private byte[] getAttrValueBytes(long pos, int size) + { + byte[] bytes = new byte[size]; + HyperClient.read_attr_value(this,bytes,pos); + return bytes; + } + + private java.lang.Object getAttrStringValue(String defaultStringEncoding) + { + ByteArray attrValue = new ByteArray(getAttrValueBytes()); + attrValue.setDefaultEncoding(defaultStringEncoding); + return attrValue; + } + + private byte[] zerofill(byte[] inbytes) + { + byte[] outbytes = new byte[8]; + + // Make sure outbytes is initialized to 0 + for (int i=0; i<8; i++ ) outbytes[i] = 0; + + // Copy little-endingly + for (int i=0; i coll, + String defaultStringEncoding) throws ValueError + { + String collType = coll instanceof java.util.List?"list":"set"; + + // Interpret return value of getValue_sz() as unsigned + // + java.math.BigInteger value_sz + = new java.math.BigInteger(1,java.nio.ByteBuffer.allocate(8).order( + java.nio.ByteOrder.BIG_ENDIAN).putLong(getValue_sz()).array()); + + java.math.BigInteger rem = new java.math.BigInteger(value_sz.toString()); + long pos = 0; + + java.math.BigInteger four = new java.math.BigInteger("4"); + + int coll_sz = 0; + + while ( rem.compareTo(four) >= 0 && coll_sz <= Integer.MAX_VALUE ) + { + int sz + = java.nio.ByteBuffer.wrap(getAttrValueBytes(pos,4)).order( + java.nio.ByteOrder.LITTLE_ENDIAN).getInt(); + + java.math.BigInteger sz_bi + = new java.math.BigInteger(new Integer(sz).toString()); + + if ( rem.subtract(four).compareTo(sz_bi) < 0 ) + { + throw new ValueError(collType + + "(string) is improperly structured (file a bug)"); + } + + ByteArray collElement = new ByteArray(getAttrValueBytes(pos+4,sz)); + collElement.setDefaultEncoding(defaultStringEncoding); + coll.add(collElement); + + rem = rem.subtract(four).subtract(sz_bi); + pos = value_sz.subtract(rem).longValue(); + + coll_sz += 1; + } + + if ( coll_sz < Integer.MAX_VALUE && rem.compareTo(java.math.BigInteger.ZERO) > 0 ) + { + throw new ValueError(collType + "(string) contains excess data (file a bug)"); + } + + return coll; + } + + private java.lang.Object getAttrCollectionLongValue( + java.util.AbstractCollection coll) throws ValueError + { + String collType = coll instanceof java.util.List?"list":"set"; + + // Interpret return value of getValue_sz() as unsigned + // + java.math.BigInteger value_sz + = new java.math.BigInteger(1,java.nio.ByteBuffer.allocate(8).order( + java.nio.ByteOrder.BIG_ENDIAN).putLong(getValue_sz()).array()); + + java.math.BigInteger rem = new java.math.BigInteger(value_sz.toString()); + long pos = 0; + + java.math.BigInteger eight = new java.math.BigInteger("8"); + + int coll_sz = 0; + + while ( rem.compareTo(eight) >= 0 && coll_sz <= Integer.MAX_VALUE ) + { + coll.add(new Long(java.nio.ByteBuffer.wrap(getAttrValueBytes(pos,8)).order( + java.nio.ByteOrder.LITTLE_ENDIAN).getLong())); + + rem = rem.subtract(eight); + pos = value_sz.subtract(rem).longValue(); + + coll_sz += 1; + } + + if ( coll_sz < Integer.MAX_VALUE && rem.compareTo(java.math.BigInteger.ZERO) > 0 ) + { + throw new ValueError(collType + "(int64) contains excess data (file a bug)"); + } + + return coll; + } + + private java.lang.Object getAttrCollectionDoubleValue( + java.util.AbstractCollection coll) throws ValueError + { + String collType = coll instanceof java.util.List?"list":"set"; + + // Interpret return value of getValue_sz() as unsigned + // + java.math.BigInteger value_sz + = new java.math.BigInteger(1,java.nio.ByteBuffer.allocate(8).order( + java.nio.ByteOrder.BIG_ENDIAN).putLong(getValue_sz()).array()); + + java.math.BigInteger rem = new java.math.BigInteger(value_sz.toString()); + long pos = 0; + + java.math.BigInteger eight = new java.math.BigInteger("8"); + + int coll_sz = 0; + + while ( rem.compareTo(eight) >= 0 && coll_sz <= Integer.MAX_VALUE ) + { + coll.add(new Double(java.nio.ByteBuffer.wrap(getAttrValueBytes(pos,8)).order( + java.nio.ByteOrder.LITTLE_ENDIAN).getDouble())); + + rem = rem.subtract(eight); + pos = value_sz.subtract(rem).longValue(); + + coll_sz += 1; + } + + if ( coll_sz < Integer.MAX_VALUE && rem.compareTo(java.math.BigInteger.ZERO) > 0 ) + { + throw new ValueError(collType + "(float) contains excess data (file a bug)"); + } + + return coll; + } + + private java.lang.Object getAttrMapStringStringValue(String defaultStringEncoding) + throws ValueError + { + ByteArrayKeyedSortedMap map + = new ByteArrayKeyedSortedMap(); + + // Interpret return value of getValue_sz() as unsigned + // + java.math.BigInteger value_sz + = new java.math.BigInteger(1,java.nio.ByteBuffer.allocate(8).order( + java.nio.ByteOrder.BIG_ENDIAN).putLong(getValue_sz()).array()); + + java.math.BigInteger rem = new java.math.BigInteger(value_sz.toString()); + long pos = 0; + + java.math.BigInteger four = new java.math.BigInteger("4"); + + int map_sz = 0; + + while ( rem.compareTo(four) >= 0 && map_sz <= Integer.MAX_VALUE ) + { + int sz + = java.nio.ByteBuffer.wrap(getAttrValueBytes(pos,4)).order( + java.nio.ByteOrder.LITTLE_ENDIAN).getInt(); + + java.math.BigInteger sz_bi + = new java.math.BigInteger(new Integer(sz).toString()); + + if ( rem.subtract(four).compareTo(sz_bi) < 0 ) + { + throw new ValueError( + "map(string,string) is improperly structured (file a bug)"); + } + + ByteArray key = new ByteArray(getAttrValueBytes(pos+4,sz)); + key.setDefaultEncoding(defaultStringEncoding); + + rem = rem.subtract(four).subtract(sz_bi); + pos = value_sz.subtract(rem).longValue(); + + sz = java.nio.ByteBuffer.wrap(getAttrValueBytes(pos,4)).order( + java.nio.ByteOrder.LITTLE_ENDIAN).getInt(); + + sz_bi = new java.math.BigInteger(new Integer(sz).toString()); + + if ( rem.subtract(four).compareTo(sz_bi) < 0 ) + { + throw new ValueError( + "map(string,string) is improperly structured (file a bug)"); + } + + ByteArray val = new ByteArray(getAttrValueBytes(pos+4,sz)); + val.setDefaultEncoding(defaultStringEncoding); + + rem = rem.subtract(four).subtract(sz_bi); + pos = value_sz.subtract(rem).longValue(); + + map.put(key,val); + + map_sz += 1; + } + + if ( map_sz < Integer.MAX_VALUE && rem.compareTo(java.math.BigInteger.ZERO) > 0 ) + { + throw new ValueError("map(string,string) contains excess data (file a bug)"); + } + + return map; + } + + private java.lang.Object getAttrMapStringLongValue(String defaultStringEncoding) + throws ValueError + { + ByteArrayKeyedSortedMap map + = new ByteArrayKeyedSortedMap(); + + // Interpret return value of getValue_sz() as unsigned + // + java.math.BigInteger value_sz + = new java.math.BigInteger(1,java.nio.ByteBuffer.allocate(8).order( + java.nio.ByteOrder.BIG_ENDIAN).putLong(getValue_sz()).array()); + + java.math.BigInteger rem = new java.math.BigInteger(value_sz.toString()); + long pos = 0; + + java.math.BigInteger four = new java.math.BigInteger("4"); + java.math.BigInteger eight = new java.math.BigInteger("8"); + + int map_sz = 0; + + while ( rem.compareTo(four) >= 0 && map_sz <= Integer.MAX_VALUE ) + { + int sz + = java.nio.ByteBuffer.wrap(getAttrValueBytes(pos,4)).order( + java.nio.ByteOrder.LITTLE_ENDIAN).getInt(); + + java.math.BigInteger sz_bi + = new java.math.BigInteger(new Integer(sz).toString()); + + if ( rem.subtract(four).compareTo(sz_bi) < 0 ) + { + throw new ValueError( + "map(string,int64) is improperly structured (file a bug)"); + } + + ByteArray key = new ByteArray(getAttrValueBytes(pos+4,sz)); + key.setDefaultEncoding(defaultStringEncoding); + + rem = rem.subtract(four).subtract(sz_bi); + pos = value_sz.subtract(rem).longValue(); + + if ( rem.compareTo(eight) < 0 ) + { + throw new ValueError( + "map(string,int64) is improperly structured (file a bug)"); + } + + Long val = new Long(java.nio.ByteBuffer.wrap(getAttrValueBytes(pos,8)).order( + java.nio.ByteOrder.LITTLE_ENDIAN).getLong()); + + rem = rem.subtract(eight); + pos = value_sz.subtract(rem).longValue(); + + map.put(key,val); + + map_sz += 1; + } + + if ( map_sz < Integer.MAX_VALUE && rem.compareTo(java.math.BigInteger.ZERO) > 0 ) + { + throw new ValueError("map(string,int64) contains excess data (file a bug)"); + } + + return map; + } + + private java.lang.Object getAttrMapStringDoubleValue(String defaultStringEncoding) + throws ValueError + { + ByteArrayKeyedSortedMap + map = new ByteArrayKeyedSortedMap(); + + // Interpret return value of getValue_sz() as unsigned + // + java.math.BigInteger value_sz + = new java.math.BigInteger(1,java.nio.ByteBuffer.allocate(8).order( + java.nio.ByteOrder.BIG_ENDIAN).putLong(getValue_sz()).array()); + + java.math.BigInteger rem = new java.math.BigInteger(value_sz.toString()); + long pos = 0; + + java.math.BigInteger four = new java.math.BigInteger("4"); + java.math.BigInteger eight = new java.math.BigInteger("8"); + + int map_sz = 0; + + while ( rem.compareTo(four) >= 0 && map_sz <= Integer.MAX_VALUE ) + { + int sz + = java.nio.ByteBuffer.wrap(getAttrValueBytes(pos,4)).order( + java.nio.ByteOrder.LITTLE_ENDIAN).getInt(); + + java.math.BigInteger sz_bi + = new java.math.BigInteger(new Integer(sz).toString()); + + if ( rem.subtract(four).compareTo(sz_bi) < 0 ) + { + throw new ValueError( + "map(string,float) is improperly structured (file a bug)"); + } + + ByteArray key = new ByteArray(getAttrValueBytes(pos+4,sz)); + key.setDefaultEncoding(defaultStringEncoding); + + rem = rem.subtract(four).subtract(sz_bi); + pos = value_sz.subtract(rem).longValue(); + + if ( rem.compareTo(eight) < 0 ) + { + throw new ValueError( + "map(string,float) is improperly structured (file a bug)"); + } + + Double val = new Double(java.nio.ByteBuffer.wrap(getAttrValueBytes(pos,8)).order( + java.nio.ByteOrder.LITTLE_ENDIAN).getDouble()); + + rem = rem.subtract(eight); + pos = value_sz.subtract(rem).longValue(); + + map.put(key,val); + + map_sz += 1; + } + + if ( map_sz < Integer.MAX_VALUE && rem.compareTo(java.math.BigInteger.ZERO) > 0 ) + { + throw new ValueError("map(string,float) contains excess data (file a bug)"); + } + + return map; + } + + private java.lang.Object getAttrMapLongStringValue(String defaultStringEncoding) + throws ValueError + { + java.util.HashMap map = new java.util.HashMap(); + + // Interpret return value of getValue_sz() as unsigned + // + java.math.BigInteger value_sz + = new java.math.BigInteger(1,java.nio.ByteBuffer.allocate(8).order( + java.nio.ByteOrder.BIG_ENDIAN).putLong(getValue_sz()).array()); + + java.math.BigInteger rem = new java.math.BigInteger(value_sz.toString()); + long pos = 0; + + java.math.BigInteger four = new java.math.BigInteger("4"); + java.math.BigInteger eight = new java.math.BigInteger("8"); + + int map_sz = 0; + + while ( rem.compareTo(eight) >= 0 && map_sz <= Integer.MAX_VALUE ) + { + Long key = new Long(java.nio.ByteBuffer.wrap(getAttrValueBytes(pos,8)).order( + java.nio.ByteOrder.LITTLE_ENDIAN).getLong()); + + rem = rem.subtract(eight); + pos = value_sz.subtract(rem).longValue(); + + int sz + = java.nio.ByteBuffer.wrap(getAttrValueBytes(pos,4)).order( + java.nio.ByteOrder.LITTLE_ENDIAN).getInt(); + + java.math.BigInteger sz_bi + = new java.math.BigInteger(new Integer(sz).toString()); + + if ( rem.subtract(four).compareTo(sz_bi) < 0 ) + { + throw new ValueError( + "map(int64,string) is improperly structured (file a bug)"); + } + + ByteArray val = new ByteArray(getAttrValueBytes(pos+4,sz)); + val.setDefaultEncoding(defaultStringEncoding); + + rem = rem.subtract(four).subtract(sz_bi); + pos = value_sz.subtract(rem).longValue(); + + map.put(key,val); + + map_sz += 1; + } + + if ( map_sz < Integer.MAX_VALUE && rem.compareTo(java.math.BigInteger.ZERO) > 0 ) + { + throw new ValueError("map(int64,string) contains excess data (file a bug)"); + } + + return map; + } + + private java.lang.Object getAttrMapLongLongValue() throws ValueError + { + java.util.HashMap map = new java.util.HashMap(); + + // Interpret return value of getValue_sz() as unsigned + // + java.math.BigInteger value_sz + = new java.math.BigInteger(1,java.nio.ByteBuffer.allocate(8).order( + java.nio.ByteOrder.BIG_ENDIAN).putLong(getValue_sz()).array()); + + java.math.BigInteger rem = new java.math.BigInteger(value_sz.toString()); + long pos = 0; + + java.math.BigInteger eight = new java.math.BigInteger("8"); + java.math.BigInteger sixteen = new java.math.BigInteger("16"); + + int map_sz = 0; + + while ( rem.compareTo(sixteen) >= 0 && map_sz <= Integer.MAX_VALUE ) + { + Long key = new Long(java.nio.ByteBuffer.wrap(getAttrValueBytes(pos,8)).order( + java.nio.ByteOrder.LITTLE_ENDIAN).getLong()); + + rem = rem.subtract(eight); + pos = value_sz.subtract(rem).longValue(); + + Long val = new Long(java.nio.ByteBuffer.wrap(getAttrValueBytes(pos,8)).order( + java.nio.ByteOrder.LITTLE_ENDIAN).getLong()); + + rem = rem.subtract(eight); + pos = value_sz.subtract(rem).longValue(); + + map.put(key,val); + + map_sz += 1; + } + + if ( map_sz < Integer.MAX_VALUE && rem.compareTo(java.math.BigInteger.ZERO) > 0 ) + { + throw new ValueError("map(int64,int64) contains excess data (file a bug)"); + } + + return map; + } + + private java.lang.Object getAttrMapLongDoubleValue() throws ValueError + { + java.util.HashMap map = new java.util.HashMap(); + + // Interpret return value of getValue_sz() as unsigned + // + java.math.BigInteger value_sz + = new java.math.BigInteger(1,java.nio.ByteBuffer.allocate(8).order( + java.nio.ByteOrder.BIG_ENDIAN).putLong(getValue_sz()).array()); + + java.math.BigInteger rem = new java.math.BigInteger(value_sz.toString()); + long pos = 0; + + java.math.BigInteger eight = new java.math.BigInteger("8"); + java.math.BigInteger sixteen = new java.math.BigInteger("16"); + + int map_sz = 0; + + while ( rem.compareTo(sixteen) >= 0 && map_sz <= Integer.MAX_VALUE ) + { + Long key = new Long(java.nio.ByteBuffer.wrap(getAttrValueBytes(pos,8)).order( + java.nio.ByteOrder.LITTLE_ENDIAN).getLong()); + + rem = rem.subtract(eight); + pos = value_sz.subtract(rem).longValue(); + + Double val = new Double(java.nio.ByteBuffer.wrap(getAttrValueBytes(pos,8)).order( + java.nio.ByteOrder.LITTLE_ENDIAN).getDouble()); + + rem = rem.subtract(eight); + pos = value_sz.subtract(rem).longValue(); + + map.put(key,val); + + map_sz += 1; + } + + if ( map_sz < Integer.MAX_VALUE && rem.compareTo(java.math.BigInteger.ZERO) > 0 ) + { + throw new ValueError("map(int64,float) contains excess data (file a bug)"); + } + + return map; + } + + private java.lang.Object getAttrMapDoubleStringValue(String defaultStringEncoding) + throws ValueError + { + java.util.HashMap map = new java.util.HashMap(); + + // Interpret return value of getValue_sz() as unsigned + // + java.math.BigInteger value_sz + = new java.math.BigInteger(1,java.nio.ByteBuffer.allocate(8).order( + java.nio.ByteOrder.BIG_ENDIAN).putLong(getValue_sz()).array()); + + java.math.BigInteger rem = new java.math.BigInteger(value_sz.toString()); + long pos = 0; + + java.math.BigInteger four = new java.math.BigInteger("4"); + java.math.BigInteger eight = new java.math.BigInteger("8"); + + int map_sz = 0; + + while ( rem.compareTo(eight) >= 0 && map_sz <= Integer.MAX_VALUE ) + { + Double key = new Double(java.nio.ByteBuffer.wrap(getAttrValueBytes(pos,8)).order( + java.nio.ByteOrder.LITTLE_ENDIAN).getDouble()); + + rem = rem.subtract(eight); + pos = value_sz.subtract(rem).longValue(); + + int sz + = java.nio.ByteBuffer.wrap(getAttrValueBytes(pos,4)).order( + java.nio.ByteOrder.LITTLE_ENDIAN).getInt(); + + java.math.BigInteger sz_bi + = new java.math.BigInteger(new Integer(sz).toString()); + + if ( rem.subtract(four).compareTo(sz_bi) < 0 ) + { + throw new ValueError( + "map(float,string) is improperly structured (file a bug)"); + } + + ByteArray val = new ByteArray(getAttrValueBytes(pos+4,sz)); + val.setDefaultEncoding(defaultStringEncoding); + + rem = rem.subtract(four).subtract(sz_bi); + pos = value_sz.subtract(rem).longValue(); + + map.put(key,val); + + map_sz += 1; + } + + if ( map_sz < Integer.MAX_VALUE && rem.compareTo(java.math.BigInteger.ZERO) > 0 ) + { + throw new ValueError("map(float,string) contains excess data (file a bug)"); + } + + return map; + } + + private java.lang.Object getAttrMapDoubleLongValue() throws ValueError + { + java.util.HashMap map = new java.util.HashMap(); + + // Interpret return value of getValue_sz() as unsigned + // + java.math.BigInteger value_sz + = new java.math.BigInteger(1,java.nio.ByteBuffer.allocate(8).order( + java.nio.ByteOrder.BIG_ENDIAN).putLong(getValue_sz()).array()); + + java.math.BigInteger rem = new java.math.BigInteger(value_sz.toString()); + long pos = 0; + + java.math.BigInteger eight = new java.math.BigInteger("8"); + java.math.BigInteger sixteen = new java.math.BigInteger("16"); + + int map_sz = 0; + + while ( rem.compareTo(sixteen) >= 0 && map_sz <= Integer.MAX_VALUE ) + { + Double key = new Double(java.nio.ByteBuffer.wrap(getAttrValueBytes(pos,8)).order( + java.nio.ByteOrder.LITTLE_ENDIAN).getDouble()); + + rem = rem.subtract(eight); + pos = value_sz.subtract(rem).longValue(); + + Long val = new Long(java.nio.ByteBuffer.wrap(getAttrValueBytes(pos,8)).order( + java.nio.ByteOrder.LITTLE_ENDIAN).getLong()); + + rem = rem.subtract(eight); + pos = value_sz.subtract(rem).longValue(); + + map.put(key,val); + + map_sz += 1; + } + + if ( map_sz < Integer.MAX_VALUE && rem.compareTo(java.math.BigInteger.ZERO) > 0 ) + { + throw new ValueError("map(float,int64) contains excess data (file a bug)"); + } + + return map; + } + + private java.lang.Object getAttrMapDoubleDoubleValue() throws ValueError + { + java.util.HashMap map = new java.util.HashMap(); + + // Interpret return value of getValue_sz() as unsigned + // + java.math.BigInteger value_sz + = new java.math.BigInteger(1,java.nio.ByteBuffer.allocate(8).order( + java.nio.ByteOrder.BIG_ENDIAN).putLong(getValue_sz()).array()); + + java.math.BigInteger rem = new java.math.BigInteger(value_sz.toString()); + long pos = 0; + + java.math.BigInteger eight = new java.math.BigInteger("8"); + java.math.BigInteger sixteen = new java.math.BigInteger("16"); + + int map_sz = 0; + + while ( rem.compareTo(sixteen) >= 0 && map_sz <= Integer.MAX_VALUE ) + { + Double key = new Double(java.nio.ByteBuffer.wrap(getAttrValueBytes(pos,8)).order( + java.nio.ByteOrder.LITTLE_ENDIAN).getDouble()); + + rem = rem.subtract(eight); + pos = value_sz.subtract(rem).longValue(); + + Double val = new Double(java.nio.ByteBuffer.wrap(getAttrValueBytes(pos,8)).order( + java.nio.ByteOrder.LITTLE_ENDIAN).getDouble()); + + rem = rem.subtract(eight); + pos = value_sz.subtract(rem).longValue(); + + map.put(key,val); + + map_sz += 1; + } + + if ( map_sz < Integer.MAX_VALUE && rem.compareTo(java.math.BigInteger.ZERO) > 0 ) + { + throw new ValueError("map(float,float) contains excess data (file a bug)"); + } + + return map; + } + + public java.lang.Object getAttrValue(String defaultStringEncoding) throws ValueError + { + switch(getDatatype()) + { + case HYPERDATATYPE_STRING: + return getAttrStringValue(defaultStringEncoding); + + case HYPERDATATYPE_INT64: + return getAttrLongValue(); + + case HYPERDATATYPE_FLOAT: + return getAttrDoubleValue(); + + case HYPERDATATYPE_LIST_STRING: + ByteArrayVector ls = new ByteArrayVector(); + getAttrCollectionStringValue(ls,defaultStringEncoding); + return ls; + + case HYPERDATATYPE_LIST_INT64: + java.util.Vector li = new java.util.Vector(); + getAttrCollectionLongValue(li); + return li; + + case HYPERDATATYPE_LIST_FLOAT: + java.util.Vector lf = new java.util.Vector(); + getAttrCollectionDoubleValue(lf); + return lf; + + case HYPERDATATYPE_SET_STRING: + ByteArraySortedSet ss = new ByteArraySortedSet(); + getAttrCollectionStringValue(ss,defaultStringEncoding); + return ss; + + case HYPERDATATYPE_SET_INT64: + java.util.HashSet si = new java.util.HashSet(); + getAttrCollectionLongValue(si); + return si; + + case HYPERDATATYPE_SET_FLOAT: + java.util.HashSet sf = new java.util.HashSet(); + getAttrCollectionDoubleValue(sf); + return sf; + + case HYPERDATATYPE_MAP_STRING_STRING: + return getAttrMapStringStringValue(defaultStringEncoding); + + case HYPERDATATYPE_MAP_STRING_INT64: + return getAttrMapStringLongValue(defaultStringEncoding); + + case HYPERDATATYPE_MAP_STRING_FLOAT: + return getAttrMapStringDoubleValue(defaultStringEncoding); + + case HYPERDATATYPE_MAP_INT64_STRING: + return getAttrMapLongStringValue(defaultStringEncoding); + + case HYPERDATATYPE_MAP_INT64_INT64: + return getAttrMapLongLongValue(); + + case HYPERDATATYPE_MAP_INT64_FLOAT: + return getAttrMapLongDoubleValue(); + + case HYPERDATATYPE_MAP_FLOAT_STRING: + return getAttrMapDoubleStringValue(defaultStringEncoding); + + case HYPERDATATYPE_MAP_FLOAT_INT64: + return getAttrMapDoubleLongValue(); + + case HYPERDATATYPE_MAP_FLOAT_FLOAT: + return getAttrMapDoubleDoubleValue(); + + default: + throw new ValueError("Server returned garbage value (file a bug)"); + } + } +%} diff --git a/hyperclient/java/proxies/hyperclient_map_attribute.i b/client/java/proxies/hyperclient_map_attribute.i similarity index 100% rename from hyperclient/java/proxies/hyperclient_map_attribute.i rename to client/java/proxies/hyperclient_map_attribute.i diff --git a/hyperclient/java/proxies/hyperclient_range_query.i b/hyperclient/java/proxies/hyperclient_range_query.i deleted file mode 100644 index 95b76c83..00000000 --- a/hyperclient/java/proxies/hyperclient_range_query.i +++ /dev/null @@ -1,22 +0,0 @@ -// Write our own (safe) setter for attr -// in order to avoid warnings -// -%ignore hyperclient_range_query::attr; - -// No need to expose upper_t and lower_t in java -// -%ignore hyperclient_range_query::upper_t; -%ignore hyperclient_range_query::lower_t; - -%typemap(javacode) hyperclient_range_query -%{ - byte[] getRangeQueryAttrNameBytes() - { - int name_sz = HyperClient.get_range_query_attr_name_sz(this); - byte[] bytes = new byte[name_sz]; - - HyperClient.read_range_query_attr_name(this,bytes); - - return bytes; - } -%} From 65bc3faff107742acfde5758f13316b155044c5a Mon Sep 17 00:00:00 2001 From: Nick Tolomiczenko Date: Sat, 5 Jan 2013 20:33:59 -0500 Subject: [PATCH 05/39] Refactoring predicate_to_c a little in order to update java bindings Signed-off-by: Nick Tolomiczenko --- client/java/proxies/HyperClient.i | 314 +++++++++++++++++------------- 1 file changed, 184 insertions(+), 130 deletions(-) diff --git a/client/java/proxies/HyperClient.i b/client/java/proxies/HyperClient.i index beaac614..082803d7 100644 --- a/client/java/proxies/HyperClient.i +++ b/client/java/proxies/HyperClient.i @@ -205,14 +205,13 @@ // Returns 0 on failure. hac->attr will be NULL static int write_attr_check_name(hyperclient_attribute_check *hac, const char *attr, size_t attr_sz, - hyperdatatype type, hyperpredicate pred) + hyperpredicate pred) { char *buf; if ((buf = (char *)calloc(attr_sz+1,sizeof(char))) == NULL) return 0; memcpy(buf,attr,attr_sz); hac->attr = buf; - hac->datatype = type; hac->predicate = pred; return 1; } @@ -237,6 +236,40 @@ return 1; } + // Returns 1 on success. hac->value will point to allocated memory + // hac->value_sz will hold the size of this memory + // Returns 0 on failure. hac->value will be NULL + // hac->value_sz will be 0 + // + static int write_attr_check_value(hyperclient_attribute_check *hac, + int64_t value) + { + char *buf = NULL; + if ((buf = (char *)malloc(sizeof(int64_t))) == NULL) return 0; + memcpy(buf, value, value_sz); + hac->value = buf; + hac->value_sz = sizeof(int64_t); + hac->datatype = hyperdatatype.HYPERDATATYPE_INT64; + return 1; + } + + // Returns 1 on success. hac->value will point to allocated memory + // hac->value_sz will hold the size of this memory + // Returns 0 on failure. hac->value will be NULL + // hac->value_sz will be 0 + // + static int write_attr_check_value(hyperclient_attribute_check *hac, + double value) + { + char *buf = NULL; + if ((buf = (char *)malloc(sizeof(double))) == NULL) return 0; + memcpy(buf, value, value_sz); + hac->value = buf; + hac->value_sz = sizeof(double); + hac->datatype = hyperdatatype.HYPERDATATYPE_FLOAT; + return 1; + } + static hyperclient_attribute *get_attr(hyperclient_attribute *ha, size_t i) { return ha + i; @@ -247,7 +280,7 @@ return hma + i; } - static hyperclient_attribute_check *get_attr_check(hyperclient_attribute *hac, size_t i) + static hyperclient_attribute_check *get_attr_check(hyperclient_attribute_check *hac, size_t i) { return hac + i; } @@ -433,178 +466,199 @@ return retType; } - // write_range_query private static method expects (lower,upper) to be of types - // (Long, Long) or (Double,Double) + // write_attr_check private static method expects value to satisfy + // either being instance of type Long or Double or isBytes(value) == true // - private static void write_range_query(hyperclient_range_query rq, byte[] attr, - Object lower, Object upper) throws MemoryError + private static void write_attr_check(hyperclient_attribute_check hac, + ByteArray attr, Object value, hyperpredicate pred) + throws MemoryError { - if ( lower instanceof Long ) // then upper MUST be Long at this point + if ( write_attr_check_name(hac,attr.getBytes,pred) == 0 ) + { + throw new MemoryError(); + } + + if ( value instanceof Long ) { - if ( HyperClient.write_range_query(rq,attr, - ((Long)lower).longValue(),((Long)upper).longValue()) == 0 ) + if ( write_attr_check_value(hac, + attr.getBytes(), ((Long)value).longValue()) == 0 ) { throw new MemoryError(); } } - else // lower and upper must both be Double at this point + else if ( value instanceof Double ) { - if ( HyperClient.write_range_query(rq,attr, - ((Double)lower).doubleValue(),((Double)upper).doubleValue()) == 0 ) + if ( write_attr_check_value(hac, + attr.getBytes(), ((Double)value).doubleValue()) == 0 ) + { + throw new MemoryError(); + } + } + else // isBytes(value) must be true + { + if ( write_attr_check_value(hac, + attr.getBytes(), getBytes(value)) == 0 ) { throw new MemoryError(); } } } - // Using a Vector retvals to return multiple values. - // - // retvals at 0 - eq - // retvals at 1 - eq_sz - // retvals at 2 - rn - // retvals at 3 - rn_sz java.util.Vector predicate_to_c(java.util.Map predicate) throws TypeError, - MemoryError, - ValueError + MemoryError, + ValueError { - java.util.Vector retvals = new java.util.Vector(4); - - retvals.add(null); - retvals.add(new Integer(0)); - retvals.add(null); - retvals.add(new Integer(0)); - java.util.HashMap equalities = new java.util.HashMap(); java.util.HashMap ranges = new java.util.HashMap(); - for (java.util.Iterator it=predicate.keySet().iterator(); it.hasNext();) - { - Object attrObject = it.next(); - - if ( attrObject == null ) - throw new TypeError("Cannot search on a null attribute"); - - byte[] attrBytes = getBytes(attrObject); - - String attrStr = ByteArray.decode(attrBytes,defaultStringEncoding); + hyperclient_attribute_check hacs = null; + long hacs_sz = 0; - Object params = predicate.get(attrObject); - - if ( params == null ) - throw new TypeError("Cannot search with a null criteria"); - - - String errStr = "Attribute '" + attrStr + "' has incorrect type ( expected Long, Double, String, Map.Entry, Map.Entry, List or List (List being of size 2), but got %s"; - - if ( isBytes(params) || params instanceof Long || params instanceof Double ) - { - equalities.put(new ByteArray(attrBytes), params); - } - else + try + { + for (java.util.Iterator it=predicate.keySet().iterator(); it.hasNext();) { - if ( params instanceof java.util.Map.Entry ) + Object attrObject = it.next(); + + if ( attrObject == null ) + throw new TypeError("Cannot search on a null attribute"); + + byte[] attrBytes = getBytes(attrObject); + + String attrStr = ByteArray.decode(attrBytes,defaultStringEncoding); + + Object params = predicate.get(attrObject); + + if ( params == null ) + throw new TypeError("Cannot search with a null criteria"); + + + String errStr = "Attribute '" + attrStr + "' has incorrect type ( expected Long, Double, String, Map.Entry, Map.Entry, List or List (List being of size 2), but got %s"; + + if ( isBytes(params) || params instanceof Long || params instanceof Double ) { - Object lower = ((java.util.Map.Entry)params).getKey(); - Object upper = ((java.util.Map.Entry)params).getValue(); - - if ( ! ( lower instanceof Long && upper instanceof Long ) && - ! ( lower instanceof Double && upper instanceof Double ) ) - { - throw - new TypeError( - String.format(errStr,params.getClass().getName())); - } + equalities.put(new ByteArray(attrBytes), params); } - else if ( params instanceof java.util.List ) + else { - try + if ( params instanceof java.util.Map.Entry ) { - java.util.List listParams = (java.util.List)params; - - if ( listParams.size() != 2 ) - throw new TypeError("Attribute '" + attrStr + "': using a List to specify a range requires its size to be 2, but got size " + listParams.size()); + Object lower = ((java.util.Map.Entry)params).getKey(); + Object upper = ((java.util.Map.Entry)params).getValue(); + + if ( ! ( lower instanceof Long && upper instanceof Long ) && + ! ( lower instanceof Double && upper instanceof Double ) ) + { + throw + new TypeError( + String.format(errStr,params.getClass().getName())); + } } - catch (TypeError te) + else if ( params instanceof java.util.List ) { - throw te; + try + { + java.util.List listParams = (java.util.List)params; + + if ( listParams.size() != 2 ) + throw new TypeError("Attribute '" + attrStr + "': using a List to specify a range requires its size to be 2, but got size " + listParams.size()); + } + catch (TypeError te) + { + throw te; + } + + Object lower = ((java.util.List)params).get(0); + Object upper = ((java.util.List)params).get(1); + + if ( ! ( lower instanceof Long && upper instanceof Long ) && + ! ( lower instanceof Double && upper instanceof Double ) ) + { + throw + new TypeError( + String.format(errStr,params.getClass().getName())); + } } - - Object lower = ((java.util.List)params).get(0); - Object upper = ((java.util.List)params).get(1); - - if ( ! ( lower instanceof Long && upper instanceof Long ) && - ! ( lower instanceof Double && upper instanceof Double ) ) + else { throw - new TypeError( - String.format(errStr,params.getClass().getName())); + new TypeError(String.format(errStr,params.getClass().getName())); } + + ranges.put(new ByteArray(attrBytes),params); } - else - { - throw - new TypeError(String.format(errStr,params.getClass().getName())); - } - - ranges.put(new ByteArray(attrBytes),params); } - } - - if ( equalities.size() > 0 ) - { - - retvals.set(0, dict_to_attrs(equalities)); - retvals.set(1, equalities.size()); - } - - if ( ranges.size() > 0 ) - { - int rn_sz = ranges.size(); - hyperclient_range_query rn = HyperClient.alloc_range_queries(rn_sz); - - if ( rn == null ) throw new MemoryError(); - - retvals.set(2, rn); - retvals.set(3, ranges.size()); - - int i = 0; - for (java.util.Iterator it=ranges.keySet().iterator(); it.hasNext();) + + if ( equalities.size() > 0 || ranges.size() > 0 ) { - ByteArray attr = it.next(); - - String attrStr = ByteArray.decode(attr,defaultStringEncoding); - - Object params = ranges.get(attr); - - hyperclient_range_query rq = HyperClient.get_range_query(rn,i); - - if ( params instanceof java.util.Map.Entry ) + hacs_sz = equalities.size() + (2*ranges.size()); + + hacs = alloc_attrs_check(hacs_sz); + + if ( hacs == null ) throw new MemoryError(); + } + + long i = 0; + + if ( equalities.size() > 0 ) + { + + for (java.util.Iterator it=equalities.keySet().iterator(); it.hasNext();) { - Object lower = ((java.util.Map.Entry)params).getKey(); - Object upper = ((java.util.Map.Entry)params).getValue(); - - write_range_query(rq,attr.getBytes(),lower,upper); - + ByteArray attr = it.next(); + hyperclient_attribute_check hac = HyperClient.get_attr_check(hacs,i); + write_attr_check(hac,attr, equalities.get(attr), + hyperpredicate.HYPERPREDICATE_EQUALS); + i++; } - else // Must be a List of Longs of size = 2 + } + + if ( ranges.size() > 0 ) + { + for (java.util.Iterator it=ranges.keySet().iterator(); it.hasNext();) { - Object lower = ((java.util.List)params).get(0); - Object upper = ((java.util.List)params).get(1); - - write_range_query(rq,attr.getBytes(),lower,upper); + ByteArray attr = it.next(); + + Object params = ranges.get(attr); + + hyperclient_range_query rq = HyperClient.get_range_query(rn,i); + + if ( params instanceof java.util.Map.Entry ) + { + Object lower = ((java.util.Map.Entry)params).getKey(); + Object upper = ((java.util.Map.Entry)params).getValue(); + + write_range_query(rq,attr.getBytes(),lower,upper); + + } + else // Must be a List of Longs of size = 2 + { + Object lower = ((java.util.List)params).get(0); + Object upper = ((java.util.List)params).get(1); + + write_range_query(rq,attr.getBytes(),lower,upper); + } + + i++; } - - i++; } + + if ( i == 0 ) throw new ValueError("Search criteria can't be empty"); } + catch(Exception e) + { + if ( hacs != null ) free_attrs_check(hacs, hacs_sz); - if ( retvals.get(0) == null && retvals.get(2) == null ) - throw new ValueError("Search criteria can't be empty"); + if ( e instanceof TypeError ) throw (TypeError)e; + if ( e instanceof ValueError ) throw (ValueError)e; + if ( e instanceof MemoryError ) throw (MemoryError)e; + + } - return retvals; + return hacs; } private static boolean isBytes(Object obj) From 28f056e4d9ade48234ac66df67e7c1f3ba4ac958 Mon Sep 17 00:00:00 2001 From: Nick Tolomiczenko Date: Sat, 5 Jan 2013 22:05:46 -0500 Subject: [PATCH 06/39] Finished coding the new java binding version of predicate_to_c Signed-off-by: Nick Tolomiczenko --- client/java/proxies/HyperClient.i | 37 ++++++++++++++++++------------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/client/java/proxies/HyperClient.i b/client/java/proxies/HyperClient.i index 082803d7..db2e5d26 100644 --- a/client/java/proxies/HyperClient.i +++ b/client/java/proxies/HyperClient.i @@ -233,6 +233,7 @@ memcpy(buf + hac->value_sz, value, value_sz); hac->value = buf; hac->value_sz += value_sz; + hac->datatype = hyperdatatype.HYPERDATATYPE_STRING; return 1; } @@ -606,43 +607,49 @@ if ( equalities.size() > 0 ) { - for (java.util.Iterator it=equalities.keySet().iterator(); it.hasNext();) + for (java.util.Iterator it=equalities.keySet().iterator(); + it.hasNext();) { ByteArray attr = it.next(); - hyperclient_attribute_check hac = HyperClient.get_attr_check(hacs,i); + hyperclient_attribute_check hac = HyperClient.get_attr_check(hacs,i++); write_attr_check(hac,attr, equalities.get(attr), hyperpredicate.HYPERPREDICATE_EQUALS); - i++; } } if ( ranges.size() > 0 ) { - for (java.util.Iterator it=ranges.keySet().iterator(); it.hasNext();) + for (java.util.Iterator it=ranges.keySet().iterator(); + it.hasNext();) { ByteArray attr = it.next(); Object params = ranges.get(attr); - hyperclient_range_query rq = HyperClient.get_range_query(rn,i); - + Object lower = null; + Object upper = null; + if ( params instanceof java.util.Map.Entry ) { - Object lower = ((java.util.Map.Entry)params).getKey(); - Object upper = ((java.util.Map.Entry)params).getValue(); - - write_range_query(rq,attr.getBytes(),lower,upper); + lower = ((java.util.Map.Entry)params).getKey(); + upper = ((java.util.Map.Entry)params).getValue(); } else // Must be a List of Longs of size = 2 { - Object lower = ((java.util.List)params).get(0); - Object upper = ((java.util.List)params).get(1); - - write_range_query(rq,attr.getBytes(),lower,upper); + lower = ((java.util.List)params).get(0); + upper = ((java.util.List)params).get(1); } + + hyperclient_attribute_check hac = HyperClient.get_attr_check(hacs,i++); - i++; + write_attr_check(hac,attr, lower, + hyperpredicate.HYPERPREDICATE_GREATER_EQUAL); + + hyperclient_attribute_check hac = HyperClient.get_attr_check(hacs,i++); + + write_attr_check(hac,attr, upper, + hyperpredicate.HYPERPREDICATE_LESS_EQUAL); } } From 47a4c6e9439e38178d4efa647b763a9c0574c6c4 Mon Sep 17 00:00:00 2001 From: Nick Tolomiczenko Date: Mon, 7 Jan 2013 23:03:02 -0500 Subject: [PATCH 07/39] Coded java bindings fix except new functions: add_space and rm_space Signed-off-by: Nick Tolomiczenko --- client/java/extra_src/DeferredCount.java | 21 ++++++---------- client/java/extra_src/DeferredGroupDel.java | 21 ++++++---------- .../java/extra_src/HyperClientException.java | 9 +++++-- client/java/extra_src/Pending.java | 24 +++++++------------ client/java/extra_src/Search.java | 14 ++++------- client/java/extra_src/SearchBase.java | 7 ++---- client/java/extra_src/SortedSearch.java | 14 ++++------- client/java/proxies/HyperClient.i | 12 +++++++++- 8 files changed, 53 insertions(+), 69 deletions(-) diff --git a/client/java/extra_src/DeferredCount.java b/client/java/extra_src/DeferredCount.java index dea38bdc..2004ae5e 100644 --- a/client/java/extra_src/DeferredCount.java +++ b/client/java/extra_src/DeferredCount.java @@ -22,37 +22,30 @@ public DeferredCount(HyperClient client, Object space, Map predicate, boolean un if ( predicate == null ) throw new ValueError("DeferredCount critera cannot be null"); - hyperclient_attribute eq = null; - int eq_sz = 0; - - hyperclient_range_query rn = null; - int rn_sz = 0; + hyperclient_attribute_check chks = null; + long chks_sz = 0; try { Vector retvals = client.predicate_to_c(predicate); - eq = (hyperclient_attribute)(retvals.get(0)); - eq_sz = ((Integer)(retvals.get(1))).intValue(); - rn = (hyperclient_range_query)(retvals.get(2)); - rn_sz = ((Integer)(retvals.get(3))).intValue(); + chks = (hyperclient_attribute_check)(retvals.get(0)); + chks_sz = ((Long)(retvals.get(1))).longValue(); res_ptr = hyperclient.new_uint64_t_ptr(); reqId = client.count(client.getBytes(space,true), - eq, eq_sz, - rn, rn_sz, + chks, chks_sz, rc_ptr, res_ptr); - checkReqIdSearch(reqId, status(), eq, eq_sz, rn, rn_sz); + checkReqIdSearch(reqId, status(), chks, chks_sz); client.ops.put(reqId,this); } finally { - if ( eq != null ) HyperClient.free_attrs(eq, eq_sz); - if ( rn != null ) HyperClient.free_range_queries(rn, rn_sz); + if ( chks != null ) HyperClient.free_attrs_check(chks, chks_sz); } } diff --git a/client/java/extra_src/DeferredGroupDel.java b/client/java/extra_src/DeferredGroupDel.java index 0254cac8..d1e2e7b2 100644 --- a/client/java/extra_src/DeferredGroupDel.java +++ b/client/java/extra_src/DeferredGroupDel.java @@ -16,35 +16,28 @@ public DeferredGroupDel(HyperClient client, Object space, Map predicate) if ( predicate == null ) throw new ValueError("DeferredGroupDel critera cannot be null"); - hyperclient_attribute eq = null; - int eq_sz = 0; - - hyperclient_range_query rn = null; - int rn_sz = 0; + hyperclient_attribute_check chks = null; + long chks_sz = 0; try { Vector retvals = client.predicate_to_c(predicate); - eq = (hyperclient_attribute)(retvals.get(0)); - eq_sz = ((Integer)(retvals.get(1))).intValue(); - rn = (hyperclient_range_query)(retvals.get(2)); - rn_sz = ((Integer)(retvals.get(3))).intValue(); + chks = (hyperclient_attribute_check)(retvals.get(0)); + chks_sz = ((Long)(retvals.get(1))).longValue(); reqId = client.group_del(client.getBytes(space,true), - eq, eq_sz, - rn, rn_sz, + chks, chks_sz, rc_ptr); - checkReqIdSearch(reqId, status(), eq, eq_sz, rn, rn_sz); + checkReqIdSearch(reqId, status(), chks, chks_sz); client.ops.put(reqId,this); } finally { - if ( eq != null ) HyperClient.free_attrs(eq, eq_sz); - if ( rn != null ) HyperClient.free_range_queries(rn, rn_sz); + if ( chks != null ) HyperClient.free_attrs_check(chks, chks_sz); } } diff --git a/client/java/extra_src/HyperClientException.java b/client/java/extra_src/HyperClientException.java index fbdd363f..8bcdb35f 100644 --- a/client/java/extra_src/HyperClientException.java +++ b/client/java/extra_src/HyperClientException.java @@ -27,10 +27,15 @@ public class HyperClientException extends Exception errorMap.put(hyperclient_returncode.HYPERCLIENT_DONTUSEKEY,"Do not specify the key in a search predicate and do not redundantly specify the key for an insert"); errorMap.put(hyperclient_returncode.HYPERCLIENT_WRONGTYPE,"Attribute '%s' has the wrong type"); errorMap.put(hyperclient_returncode.HYPERCLIENT_NOMEM,"Memory allocation failed"); - errorMap.put(hyperclient_returncode.HYPERCLIENT_EXCEPTION,"Internal Error (file a bug)"); + errorMap.put(hyperclient_returncode.HYPERCLIENT_BADCONFIG,"The coordinator provided a malformed configuration"); + errorMap.put(hyperclient_returncode.HYPERCLIENT_BADSPACE,"The space description does not parse"); + errorMap.put(hyperclient_returncode.HYPERCLIENT_DUPLICATE,"The space already exists"); + errorMap.put(hyperclient_returncode.HYPERCLIENT_INTERNAL,"Internal Error (file a bug)"); + errorMap.put(hyperclient_returncode.HYPERCLIENT_EXCEPTION,"Internal Exception (file a bug)"); + errorMap.put(hyperclient_returncode.HYPERCLIENT_GARBAGE,"Internal Corruption (file a bug)"); } - private hyperclient_returncode rc = hyperclient_returncode.HYPERCLIENT_ZERO; + private hyperclient_returncode rc = hyperclient_returncode.HYPERCLIENT_GARBAGE; public HyperClientException(hyperclient_returncode rc) { diff --git a/client/java/extra_src/Pending.java b/client/java/extra_src/Pending.java index 4780c91c..4759a9d8 100644 --- a/client/java/extra_src/Pending.java +++ b/client/java/extra_src/Pending.java @@ -15,7 +15,7 @@ public Pending(HyperClient client) { this.client = client; rc_ptr = hyperclient.new_rc_ptr(); - hyperclient.rc_ptr_assign(rc_ptr,hyperclient_returncode.HYPERCLIENT_ZERO); + hyperclient.rc_ptr_assign(rc_ptr,hyperclient_returncode.HYPERCLIENT_GARBAGE); } public void callback() @@ -124,28 +124,22 @@ protected void checkReqIdKeyMapAttrs(long reqid, hyperclient_returncode status, } protected void checkReqIdSearch(long reqId, hyperclient_returncode status, - hyperclient_attribute eq, int eq_sz, - hyperclient_range_query rn, int rn_sz) + hyperclient_attribute_check chks, long chks_sz) throws HyperClientException, TypeError { if (reqId < 0) { - int idx = (int)(-1 - reqId); - String attrName = null; + long idx = -1 - reqId; - if ( idx >= 0 && idx < eq_sz && eq != null ) - attrName = ByteArray.decode(HyperClient.get_attr(eq,idx).getAttrNameBytes(),client.getDefaultStringEncoding()); - - idx -= eq_sz; + String attrName = null; - if ( idx >= 0 && idx < rn_sz && rn != null ) - attrName - = ByteArray.decode( - HyperClient.get_range_query(rn,idx).getRangeQueryAttrNameBytes(), - client.getDefaultStringEncoding()); - if ( attrName != null ) + if ( chks != null && idx >= 0 && idx < chk_sz ) { + attrName = ByteArray.decode( + HyperClient.get_attr_chk(eq,idx).getAttrNameBytes(), + client.getDefaultStringEncoding()); + throw new HyperClientException(status,attrName); } else diff --git a/client/java/extra_src/Search.java b/client/java/extra_src/Search.java index fa3c890f..0a6929da 100644 --- a/client/java/extra_src/Search.java +++ b/client/java/extra_src/Search.java @@ -20,26 +20,22 @@ public Search(HyperClient client, Object space, Map predicate) { Vector retvals = client.predicate_to_c(predicate); - eq = (hyperclient_attribute)(retvals.get(0)); - eq_sz = ((Integer)(retvals.get(1))).intValue(); - rn = (hyperclient_range_query)(retvals.get(2)); - rn_sz = ((Integer)(retvals.get(3))).intValue(); + chks = (hyperclient_attribute_check)(retvals.get(0)); + chks_sz = ((Long)(retvals.get(1))).longValue(); reqId = client.search(client.getBytes(space,true), - eq, eq_sz, - rn, rn_sz, + chks, chks_sz, rc_ptr, attrs_ptr, attrs_sz_ptr); - checkReqIdSearch(reqId, status(), eq, eq_sz, rn, rn_sz); + checkReqIdSearch(reqId, status(), chks, chks_sz); client.ops.put(reqId,this); } finally { - if ( eq != null ) HyperClient.free_attrs(eq, eq_sz); - if ( rn != null ) HyperClient.free_range_queries(rn, rn_sz); + if ( chks != null ) HyperClient.free_attrs_check(chks, chks_sz); } } } diff --git a/client/java/extra_src/SearchBase.java b/client/java/extra_src/SearchBase.java index 115187c9..be7e6419 100644 --- a/client/java/extra_src/SearchBase.java +++ b/client/java/extra_src/SearchBase.java @@ -7,11 +7,8 @@ public class SearchBase extends Pending protected SWIGTYPE_p_p_hyperclient_attribute attrs_ptr = null; protected SWIGTYPE_p_size_t attrs_sz_ptr = null; - protected hyperclient_attribute eq = null; - protected int eq_sz = 0; - - protected hyperclient_range_query rn = null; - protected int rn_sz = 0; + protected hyperclient_attribute_check chks = null; + protected int chks_sz = 0; private Vector backlogged = new Vector(); diff --git a/client/java/extra_src/SortedSearch.java b/client/java/extra_src/SortedSearch.java index f71e0892..5abbd1d1 100644 --- a/client/java/extra_src/SortedSearch.java +++ b/client/java/extra_src/SortedSearch.java @@ -24,14 +24,11 @@ public SortedSearch(HyperClient client, Object space, Map predicate, { Vector retvals = client.predicate_to_c(predicate); - eq = (hyperclient_attribute)(retvals.get(0)); - eq_sz = ((Integer)(retvals.get(1))).intValue(); - rn = (hyperclient_range_query)(retvals.get(2)); - rn_sz = ((Integer)(retvals.get(3))).intValue(); + chks = (hyperclient_attribute_check)(retvals.get(0)); + chks_sz = ((Long)(retvals.get(1))).longValue(); reqId = client.sorted_search(client.getBytes(space,true), - eq, eq_sz, - rn, rn_sz, + chks, chks_sz, client.getBytes(sortBy,true), limit, descending, @@ -39,14 +36,13 @@ public SortedSearch(HyperClient client, Object space, Map predicate, attrs_ptr, attrs_sz_ptr); - checkReqIdSearch(reqId, status(), eq, eq_sz, rn, rn_sz); + checkReqIdSearch(reqId, status(), chks, chks_sz); client.ops.put(reqId,this); } finally { - if ( eq != null ) HyperClient.free_attrs(eq, eq_sz); - if ( rn != null ) HyperClient.free_range_queries(rn, rn_sz); + if ( chks != null ) HyperClient.free_attrs_check(chks, chks_sz); } } } diff --git a/client/java/proxies/HyperClient.i b/client/java/proxies/HyperClient.i index db2e5d26..664d95a2 100644 --- a/client/java/proxies/HyperClient.i +++ b/client/java/proxies/HyperClient.i @@ -505,6 +505,11 @@ } } + // Using a Vector retvals to return multiple values. + // + // retvals at 0 - hacs (hyperclient_attribute_check type) + // retvals at 1 - hacs_sz (long) + java.util.Vector predicate_to_c(java.util.Map predicate) throws TypeError, MemoryError, ValueError @@ -665,7 +670,12 @@ } - return hacs; + java.util.Vector retvals = new java.util.Vector(2); + + retvals.add(hacs); + retvals.add(hacs_sz); + + return retvals; } private static boolean isBytes(Object obj) From e262181fe09f766828be6c57bbe7acce0fa22a8b Mon Sep 17 00:00:00 2001 From: Nick Tolomiczenko Date: Wed, 9 Jan 2013 22:47:55 -0500 Subject: [PATCH 08/39] Added java bindings for add_space() and rm_space() Signed-off-by: Nick Tolomiczenko --- client/java/proxies/HyperClient.i | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/client/java/proxies/HyperClient.i b/client/java/proxies/HyperClient.i index 664d95a2..5d519e0f 100644 --- a/client/java/proxies/HyperClient.i +++ b/client/java/proxies/HyperClient.i @@ -1366,6 +1366,26 @@ return attrs; } + public void add_space(Object desc) throws HyperClientException + { + hyperclient_returncode rc = add_space(getBytes(desc,true)); + + if ( rc != hyperclient_returncode.HYPERCLIENT_SUCCESS ) + { + throw new HyperClientException(rc); + } + } + + public void rm_space(Object space) throws HyperClientException + { + hyperclient_returncode rc = rm_space(getBytes(space,true)); + + if ( rc != hyperclient_returncode.HYPERCLIENT_SUCCESS ) + { + throw new HyperClientException(rc); + } + } + // Synchronous methods // public java.util.Map get(Object space, Object key) throws HyperClientException, From 1cfcef91584f137a335957f4a687c57f640df10d Mon Sep 17 00:00:00 2001 From: Nick Tolomiczenko Date: Sun, 13 Jan 2013 23:00:22 -0500 Subject: [PATCH 09/39] Coded java version of new Predicate logic Signed-off-by: Nick Tolomiczenko --- client/java/extra_src/GreaterEqual.java | 19 ++ .../java/extra_src/HyperClientException.java | 3 + client/java/extra_src/LessEqual.java | 20 +++ client/java/extra_src/Predicate.java | 29 +++ client/java/extra_src/Range.java | 22 +++ client/java/proxies/HyperClient.i | 165 ++++++++---------- 6 files changed, 163 insertions(+), 95 deletions(-) create mode 100644 client/java/extra_src/GreaterEqual.java create mode 100644 client/java/extra_src/LessEqual.java create mode 100644 client/java/extra_src/Predicate.java create mode 100644 client/java/extra_src/Range.java diff --git a/client/java/extra_src/GreaterEqual.java b/client/java/extra_src/GreaterEqual.java new file mode 100644 index 00000000..63e1d5dc --- /dev/null +++ b/client/java/extra_src/GreaterEqual.java @@ -0,0 +1,19 @@ +public class GreaterEqual extends Predicate +{ + public GreaterEqual(Object lower) throws AttributeError + { + if ( ! HyperClient.isBytes(lower) + && ! lower instanceof Long + && ! lower instanceof Double ) + { + throw new AttributeError("GreaterEqual must be a byte[], ByteArray, String, Long, or Double") + } + + List> raw + = new Vector>(1); + + raw.put(hyperpredicate.HYPERPREDICATE_GREATER_EQUAL,lower); + + this.raw = raw; + } +} diff --git a/client/java/extra_src/HyperClientException.java b/client/java/extra_src/HyperClientException.java index 8bcdb35f..d25808f5 100644 --- a/client/java/extra_src/HyperClientException.java +++ b/client/java/extra_src/HyperClientException.java @@ -30,6 +30,9 @@ public class HyperClientException extends Exception errorMap.put(hyperclient_returncode.HYPERCLIENT_BADCONFIG,"The coordinator provided a malformed configuration"); errorMap.put(hyperclient_returncode.HYPERCLIENT_BADSPACE,"The space description does not parse"); errorMap.put(hyperclient_returncode.HYPERCLIENT_DUPLICATE,"The space already exists"); + errorMap.put(hyperclient_returncode.HYPERCLIENT_INTERRUPTED,"Interrupted by a signal"); + errorMap.put(hyperclient_returncode.HYPERCLIENT_CLUSTER_JUMP,"The cluster changed identities"); + errorMap.put(hyperclient_returncode.HYPERCLIENT_COORD_LOGGED,"HYPERCLIENT_COORD_LOGGED"); errorMap.put(hyperclient_returncode.HYPERCLIENT_INTERNAL,"Internal Error (file a bug)"); errorMap.put(hyperclient_returncode.HYPERCLIENT_EXCEPTION,"Internal Exception (file a bug)"); errorMap.put(hyperclient_returncode.HYPERCLIENT_GARBAGE,"Internal Corruption (file a bug)"); diff --git a/client/java/extra_src/LessEqual.java b/client/java/extra_src/LessEqual.java new file mode 100644 index 00000000..127c63da --- /dev/null +++ b/client/java/extra_src/LessEqual.java @@ -0,0 +1,20 @@ +public class LessEqual extends Predicate +{ + public LessEqual(Object upper) throws AttributeError + { + if ( ! HyperClient.isBytes(upper) + && ! upper instanceof Long + && ! upper instanceof Double ) + { + throw new AttributeError("LessEqual must be a byte[], ByteArray, String, Long, or Double") + } + + List> raw + = new Vector>(1); + + raw.put(hyperpredicate.HYPERPREDICATE_LESS_EQUAL,upper); + + this.raw = raw; + } +} + diff --git a/client/java/extra_src/Predicate.java b/client/java/extra_src/Predicate.java new file mode 100644 index 00000000..b81272c1 --- /dev/null +++ b/client/java/extra_src/Predicate.java @@ -0,0 +1,29 @@ +package hyperclient; + +class Predicate +{ + private List> raw; + + List>> + getRawChecks(ByteArray attribute) + { + if ( raw = null ) + { + return null; + } + else + { + Vector>> retval + = new Vector>>(); + + foreach ( Map.Entry entry: raw ) + { + retval.add(new Map.Entry>( + attribute,new Map.Entry( + raw.getKey(),raw.getValue())); + } + + return retval; + } + } +} diff --git a/client/java/extra_src/Range.java b/client/java/extra_src/Range.java new file mode 100644 index 00000000..478946c7 --- /dev/null +++ b/client/java/extra_src/Range.java @@ -0,0 +1,22 @@ +package hyperclient; + +public class Range extends Predicate +{ + public Range(Object lower, Object upper) throws AttributeError + { + if ( ! ( HyperClient.isBytes(lower) && HyperClient.isBytes(upper) ) + && ! ( lower instanceof Long && upper instanceof Long ) + && ! ( lower instanceof Double && upper instanceof Double ) ) + { + throw new AttributeError("Range search bounds must be of like types") + } + + List> raw + = new Vector>(2); + + raw.put(hyperpredicate.HYPERPREDICATE_GREATER_EQUAL,lower); + raw.put(hyperpredicate.HYPERPREDICATE_LESS_EQUAL,upper); + + this.raw = raw; + } +} diff --git a/client/java/proxies/HyperClient.i b/client/java/proxies/HyperClient.i index 5d519e0f..ea42204a 100644 --- a/client/java/proxies/HyperClient.i +++ b/client/java/proxies/HyperClient.i @@ -514,11 +514,12 @@ MemoryError, ValueError { - java.util.HashMap equalities - = new java.util.HashMap(); - - java.util.HashMap ranges - = new java.util.HashMap(); + java.util.Vector< + java.util.Map.Entry< + ByteArray,java.util.Map.Entry>> rawChecks + = new java.util.Vector< + java.util.Map.Entry< + ByteArray,java.util.Map.Entry>>(); hyperclient_attribute_check hacs = null; long hacs_sz = 0; @@ -542,123 +543,97 @@ throw new TypeError("Cannot search with a null criteria"); - String errStr = "Attribute '" + attrStr + "' has incorrect type ( expected Long, Double, String, Map.Entry, Map.Entry, List or List (List being of size 2), but got %s"; + String errStr = "Attribute '" + attrStr + "' has incorrect type ( expected Predicate, Long, Double, String, Map.Entry, Map.Entry, List or List (List being of size 2), but got %s"; if ( isBytes(params) || params instanceof Long || params instanceof Double ) { - equalities.put(new ByteArray(attrBytes), params); + rawChecks.add(new java.util.Map.Entry< + java.util.Map.Entry< + ByteArray,java.util.Map.Entry>>( + new ByteArray(attrBytes), + new java.util.Map.Entry( + hyperpredicate.HYPERPREDICATE_EQUALS, params))); } - else + else if ( params instanceof java.util.Map.Entry ) { - if ( params instanceof java.util.Map.Entry ) + Object lower = ((java.util.Map.Entry)params).getKey(); + Object upper = ((java.util.Map.Entry)params).getValue(); + + if ( ! ( lower instanceof Long && upper instanceof Long ) && + ! ( lower instanceof Double && upper instanceof Double ) ) { - Object lower = ((java.util.Map.Entry)params).getKey(); - Object upper = ((java.util.Map.Entry)params).getValue(); - - if ( ! ( lower instanceof Long && upper instanceof Long ) && - ! ( lower instanceof Double && upper instanceof Double ) ) - { - throw - new TypeError( - String.format(errStr,params.getClass().getName())); - } + throw + new TypeError( + String.format(errStr,params.getClass().getName())); } - else if ( params instanceof java.util.List ) + + rawChecks.addAll( + (new Range(lower,upper)).getRawChecks(ByteArray(attrBytes))); + } + else if ( params instanceof java.util.List ) + { + try { - try - { - java.util.List listParams = (java.util.List)params; - - if ( listParams.size() != 2 ) - throw new TypeError("Attribute '" + attrStr + "': using a List to specify a range requires its size to be 2, but got size " + listParams.size()); - } - catch (TypeError te) - { - throw te; - } - - Object lower = ((java.util.List)params).get(0); - Object upper = ((java.util.List)params).get(1); - - if ( ! ( lower instanceof Long && upper instanceof Long ) && - ! ( lower instanceof Double && upper instanceof Double ) ) - { - throw - new TypeError( - String.format(errStr,params.getClass().getName())); - } + java.util.List listParams = (java.util.List)params; + + if ( listParams.size() != 2 ) + throw new TypeError("Attribute '" + attrStr + "': using a List to specify a range requires its size to be 2, but got size " + listParams.size()); } - else + catch (TypeError te) + { + throw te; + } + + Object lower = ((java.util.List)params).get(0); + Object upper = ((java.util.List)params).get(1); + + if ( ! ( lower instanceof Long && upper instanceof Long ) && + ! ( lower instanceof Double && upper instanceof Double ) ) { throw - new TypeError(String.format(errStr,params.getClass().getName())); + new TypeError( + String.format(errStr,params.getClass().getName())); } - - ranges.put(new ByteArray(attrBytes),params); + + rawChecks.addAll( + (new Range(lower,upper)).getRawChecks(ByteArray(attrBytes))); + } + else if ( params instanceof Predicate ) + { + rawChecks.addAll( + ((Predicate)params).getRawChecks(ByteArray(attrBytes))); + } + else + { + throw + new TypeError(String.format(errStr,params.getClass().getName())); } } - if ( equalities.size() > 0 || ranges.size() > 0 ) + if ( rawChecks.size() > 0 ) { - hacs_sz = equalities.size() + (2*ranges.size()); + hacs_sz = rawChecks.size(); hacs = alloc_attrs_check(hacs_sz); if ( hacs == null ) throw new MemoryError(); - } - - long i = 0; - - if ( equalities.size() > 0 ) - { - for (java.util.Iterator it=equalities.keySet().iterator(); - it.hasNext();) + foreach ( java.util.Map.Entry< + ByteArray,java.util.Map.Entry< + hyperpredicate,Object>> rawCheck: rawChecks ) { - ByteArray attr = it.next(); + ByteArray attr = rawCheck.getKey(); + hyperpredicate p = rawCheck.getValue().getKey(); + Object value = rawCheck.getValue().getValue(); + hyperclient_attribute_check hac = HyperClient.get_attr_check(hacs,i++); - write_attr_check(hac,attr, equalities.get(attr), - hyperpredicate.HYPERPREDICATE_EQUALS); + write_attr_check(hac,attr, value, p); } } - - if ( ranges.size() > 0 ) + else { - for (java.util.Iterator it=ranges.keySet().iterator(); - it.hasNext();) - { - ByteArray attr = it.next(); - - Object params = ranges.get(attr); - - Object lower = null; - Object upper = null; - - if ( params instanceof java.util.Map.Entry ) - { - lower = ((java.util.Map.Entry)params).getKey(); - upper = ((java.util.Map.Entry)params).getValue(); - - } - else // Must be a List of Longs of size = 2 - { - lower = ((java.util.List)params).get(0); - upper = ((java.util.List)params).get(1); - } - - hyperclient_attribute_check hac = HyperClient.get_attr_check(hacs,i++); - - write_attr_check(hac,attr, lower, - hyperpredicate.HYPERPREDICATE_GREATER_EQUAL); - - hyperclient_attribute_check hac = HyperClient.get_attr_check(hacs,i++); - - write_attr_check(hac,attr, upper, - hyperpredicate.HYPERPREDICATE_LESS_EQUAL); - } + throw new ValueError("Search criteria can't be empty"); } - - if ( i == 0 ) throw new ValueError("Search criteria can't be empty"); } catch(Exception e) { From fa78f03ac22ab4390a0c8cf3a7ed89c7b5556206 Mon Sep 17 00:00:00 2001 From: Nick Tolomiczenko Date: Tue, 15 Jan 2013 00:30:43 -0500 Subject: [PATCH 10/39] 19 compilation errors left Signed-off-by: Nick Tolomiczenko --- .gitignore | 3 +++ Makefile.am | 24 +++++++++++----------- client/java/extra_src/AttributeError.java | 9 ++++++++ client/java/extra_src/DeferredCondPut.java | 14 +++++++++---- client/java/extra_src/GreaterEqual.java | 13 ++++++++---- client/java/extra_src/LessEqual.java | 13 ++++++++---- client/java/extra_src/Pending.java | 10 ++++----- client/java/extra_src/Predicate.java | 8 +++++--- client/java/extra_src/Range.java | 10 ++++++--- client/java/extra_src/SearchBase.java | 2 +- client/java/hyperclient.i | 10 +++------ client/java/proxies/HyperClient.i | 21 +++++++++++-------- 12 files changed, 85 insertions(+), 52 deletions(-) create mode 100644 client/java/extra_src/AttributeError.java diff --git a/.gitignore b/.gitignore index 84fc963b..b4faefda 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ client/python/hyperclient.c config.guess config.h config.h.in +config.h.in~ config.log config.status config.sub @@ -55,3 +56,5 @@ missing py-compile stamp-h1 ylwrap +*.cxx +client/java/hyperclient diff --git a/Makefile.am b/Makefile.am index 6ec0ca00..24e03024 100644 --- a/Makefile.am +++ b/Makefile.am @@ -299,12 +299,12 @@ client/keyop_info.cc: client/keyop_info.gperf java_bindings_cleanfiles = \ $(hyperclient_jarfile) \ - hyperclient/java/hyperclient_wrap.cxx \ - hyperclient/java/hyperclient/*.java \ - hyperclient/java/hyperclient/*.class \ + client/java/hyperclient_wrap.cxx \ + client/java/hyperclient/*.java \ + client/java/hyperclient/*.class \ hyperclient-java.stamp -nodist_libhyperclient_java_la_SOURCES = hyperclient/java/hyperclient_wrap.cxx +nodist_libhyperclient_java_la_SOURCES = client/java/hyperclient_wrap.cxx libhyperclient_java_la_CPPFLAGS = \ $(JNI_CPPFLAGS) \ $(CPPFLAGS) @@ -314,20 +314,20 @@ libhyperclient_java_la_LIBADD = \ libhyperclient.la \ $(PYTHON_LDFLAGS) -hyperclient/java/hyperclient_wrap.cxx: hyperclient/java/hyperclient.i hyperclient/hyperclient.h - mkdir -p hyperclient/java/hyperclient - ${SWIG} -java -package hyperclient -outdir $(abs_builddir)/hyperclient/java/hyperclient \ - -o $(abs_builddir)/hyperclient/java/hyperclient_wrap.cxx -w518 $(abs_top_srcdir)/hyperclient/java/hyperclient.i +client/java/hyperclient_wrap.cxx: client/java/hyperclient.i client/hyperclient.h + mkdir -p client/java/hyperclient + ${SWIG} -java -package hyperclient -outdir $(abs_builddir)/client/java/hyperclient \ + -o $(abs_builddir)/client/java/hyperclient_wrap.cxx -w518 $(abs_top_srcdir)/client/java/hyperclient.i hyperclient_jarfile = hyperclient-$(VERSION).jar -hyperclient-java.stamp: hyperclient/java/hyperclient_wrap.cxx - cp hyperclient/java/extra_src/*.java hyperclient/java/hyperclient/. - javac hyperclient/java/hyperclient/*.java +hyperclient-java.stamp: client/java/hyperclient_wrap.cxx + cp client/java/extra_src/*.java client/java/hyperclient/. + javac client/java/hyperclient/*.java date > hyperclient-java.stamp $(hyperclient_jarfile): hyperclient-java.stamp - cd $(abs_top_builddir)/hyperclient/java ; $(JAR) cvf $(JARFLAGS) $(abs_top_builddir)/$(hyperclient_jarfile) hyperclient/*.class + cd $(abs_top_builddir)/client/java ; $(JAR) cvf $(JARFLAGS) $(abs_top_builddir)/$(hyperclient_jarfile) hyperclient/*.class if ENABLE_JAVA_BINDINGS lib_LTLIBRARIES += libhyperclient-java.la diff --git a/client/java/extra_src/AttributeError.java b/client/java/extra_src/AttributeError.java new file mode 100644 index 00000000..8d3f6245 --- /dev/null +++ b/client/java/extra_src/AttributeError.java @@ -0,0 +1,9 @@ +package hyperclient; + +public class AttributeError extends Exception +{ + public AttributeError(String msg) + { + super(msg); + } +} diff --git a/client/java/extra_src/DeferredCondPut.java b/client/java/extra_src/DeferredCondPut.java index 001259f7..dc2e4d2c 100644 --- a/client/java/extra_src/DeferredCondPut.java +++ b/client/java/extra_src/DeferredCondPut.java @@ -9,20 +9,26 @@ public DeferredCondPut(HyperClient client, Object space, Object key, Map condition, Map value) throws HyperClientException, TypeError, + ValueError, MemoryError { super(client); - hyperclient_attribute condattrs = null; + hyperclient_attribute_check condattrs = null; long condattrs_sz = 0; hyperclient_attribute attrs = null; long attrs_sz = 0; + if ( condition == null ) + throw new ValueError("Condition critera cannot be null"); + try { - condattrs_sz = condition.size(); - condattrs = client.dict_to_attrs(condition); + Vector retvals = client.predicate_to_c(condition); + + condattrs = (hyperclient_attribute_check)(retvals.get(0)); + condattrs_sz = ((Long)(retvals.get(1))).longValue(); attrs_sz = value.size(); attrs = client.dict_to_attrs(value); @@ -41,7 +47,7 @@ public DeferredCondPut(HyperClient client, Object space, Object key, } finally { - if ( condattrs != null ) HyperClient.free_attrs(condattrs,condattrs_sz); + if ( condattrs != null ) HyperClient.free_attrs_check(condattrs,condattrs_sz); if ( attrs != null ) HyperClient.free_attrs(attrs,attrs_sz); } } diff --git a/client/java/extra_src/GreaterEqual.java b/client/java/extra_src/GreaterEqual.java index 63e1d5dc..43949786 100644 --- a/client/java/extra_src/GreaterEqual.java +++ b/client/java/extra_src/GreaterEqual.java @@ -1,18 +1,23 @@ +package hyperclient; + +import java.util.*; + public class GreaterEqual extends Predicate { public GreaterEqual(Object lower) throws AttributeError { if ( ! HyperClient.isBytes(lower) - && ! lower instanceof Long - && ! lower instanceof Double ) + && ! (lower instanceof Long) + && ! (lower instanceof Double) ) { - throw new AttributeError("GreaterEqual must be a byte[], ByteArray, String, Long, or Double") + throw new AttributeError("GreaterEqual must be a byte[], ByteArray, String, Long, or Double"); } List> raw = new Vector>(1); - raw.put(hyperpredicate.HYPERPREDICATE_GREATER_EQUAL,lower); + raw.add(new AbstractMap.SimpleEntry( + hyperpredicate.HYPERPREDICATE_GREATER_EQUAL,lower)); this.raw = raw; } diff --git a/client/java/extra_src/LessEqual.java b/client/java/extra_src/LessEqual.java index 127c63da..bbdda9e7 100644 --- a/client/java/extra_src/LessEqual.java +++ b/client/java/extra_src/LessEqual.java @@ -1,18 +1,23 @@ +package hyperclient; + +import java.util.*; + public class LessEqual extends Predicate { public LessEqual(Object upper) throws AttributeError { if ( ! HyperClient.isBytes(upper) - && ! upper instanceof Long - && ! upper instanceof Double ) + && ! (upper instanceof Long) + && ! (upper instanceof Double) ) { - throw new AttributeError("LessEqual must be a byte[], ByteArray, String, Long, or Double") + throw new AttributeError("LessEqual must be a byte[], ByteArray, String, Long, or Double"); } List> raw = new Vector>(1); - raw.put(hyperpredicate.HYPERPREDICATE_LESS_EQUAL,upper); + raw.add(new AbstractMap.SimpleEntry( + hyperpredicate.HYPERPREDICATE_LESS_EQUAL,upper)); this.raw = raw; } diff --git a/client/java/extra_src/Pending.java b/client/java/extra_src/Pending.java index 4759a9d8..0418d187 100644 --- a/client/java/extra_src/Pending.java +++ b/client/java/extra_src/Pending.java @@ -67,8 +67,8 @@ protected void checkReqIdKeyAttrs(long reqId, hyperclient_returncode status, } protected void checkReqIdKeyAttrs2(long reqId, hyperclient_returncode status, - hyperclient_attribute attrs1, long attrs_sz1, - hyperclient_attribute attrs2, long attrs_sz2) + hyperclient_attribute_check attrs1, long attrs_sz1, + hyperclient_attribute attrs2, long attrs_sz2) throws HyperClientException, TypeError { @@ -78,7 +78,7 @@ protected void checkReqIdKeyAttrs2(long reqId, hyperclient_returncode status, String attrName = null; if ( attrs1 != null && idx >= 0 && idx < attrs_sz1 ) - attrName = ByteArray.decode(HyperClient.get_attr(attrs1,idx).getAttrNameBytes(),client.getDefaultStringEncoding()); + attrName = ByteArray.decode(HyperClient.get_attr_check(attrs1,idx).getAttrNameBytes(),client.getDefaultStringEncoding()); idx -= attrs_sz1; @@ -134,10 +134,10 @@ protected void checkReqIdSearch(long reqId, hyperclient_returncode status, String attrName = null; - if ( chks != null && idx >= 0 && idx < chk_sz ) + if ( chks != null && idx >= 0 && idx < chks_sz ) { attrName = ByteArray.decode( - HyperClient.get_attr_chk(eq,idx).getAttrNameBytes(), + HyperClient.get_attr_check(chks,idx).getAttrNameBytes(), client.getDefaultStringEncoding()); throw new HyperClientException(status,attrName); diff --git a/client/java/extra_src/Predicate.java b/client/java/extra_src/Predicate.java index b81272c1..c6f1fbba 100644 --- a/client/java/extra_src/Predicate.java +++ b/client/java/extra_src/Predicate.java @@ -1,8 +1,10 @@ package hyperclient; +import java.util.*; + class Predicate { - private List> raw; + protected List> raw; List>> getRawChecks(ByteArray attribute) @@ -16,11 +18,11 @@ class Predicate Vector>> retval = new Vector>>(); - foreach ( Map.Entry entry: raw ) + for ( Map.Entry entry: raw ) { retval.add(new Map.Entry>( attribute,new Map.Entry( - raw.getKey(),raw.getValue())); + raw.getKey(),raw.getValue()))); } return retval; diff --git a/client/java/extra_src/Range.java b/client/java/extra_src/Range.java index 478946c7..7c09d0fd 100644 --- a/client/java/extra_src/Range.java +++ b/client/java/extra_src/Range.java @@ -1,5 +1,7 @@ package hyperclient; +import java.util.*; + public class Range extends Predicate { public Range(Object lower, Object upper) throws AttributeError @@ -8,14 +10,16 @@ public Range(Object lower, Object upper) throws AttributeError && ! ( lower instanceof Long && upper instanceof Long ) && ! ( lower instanceof Double && upper instanceof Double ) ) { - throw new AttributeError("Range search bounds must be of like types") + throw new AttributeError("Range search bounds must be of like types"); } List> raw = new Vector>(2); - raw.put(hyperpredicate.HYPERPREDICATE_GREATER_EQUAL,lower); - raw.put(hyperpredicate.HYPERPREDICATE_LESS_EQUAL,upper); + raw.add(new AbstractMap.SimpleEntry( + hyperpredicate.HYPERPREDICATE_GREATER_EQUAL,lower)); + raw.add(new AbstractMap.SimpleEntry( + hyperpredicate.HYPERPREDICATE_LESS_EQUAL,upper)); this.raw = raw; } diff --git a/client/java/extra_src/SearchBase.java b/client/java/extra_src/SearchBase.java index be7e6419..ae032596 100644 --- a/client/java/extra_src/SearchBase.java +++ b/client/java/extra_src/SearchBase.java @@ -8,7 +8,7 @@ public class SearchBase extends Pending protected SWIGTYPE_p_size_t attrs_sz_ptr = null; protected hyperclient_attribute_check chks = null; - protected int chks_sz = 0; + protected long chks_sz = 0; private Vector backlogged = new Vector(); diff --git a/client/java/hyperclient.i b/client/java/hyperclient.i index 14025ff0..bad35ee9 100644 --- a/client/java/hyperclient.i +++ b/client/java/hyperclient.i @@ -40,12 +40,10 @@ #include #include "client/hyperclient.h" typedef hyperclient_attribute* hyperclient_attribute_asterisk; -typedef hyperclient_range_query* hyperclient_range_query_asterisk; %} typedef uint16_t in_port_t; typedef hyperclient_attribute* hyperclient_attribute_asterisk; -typedef hyperclient_range_query* hyperclient_range_query_asterisk; %pragma(java) jniclasscode= %{ @@ -69,9 +67,6 @@ typedef hyperclient_range_query* hyperclient_range_query_asterisk; // essentially a pointer to this java-transparent pointer. %pointer_functions(hyperclient_attribute_asterisk,hyperclient_attribute_ptr); -// Likewise for hyperclient_range_query* -%pointer_functions(hyperclient_range_query_asterisk,hyperclient_range_query_ptr); - // A couple more c++ pointer handling macros I will need // %pointer_functions(size_t, size_t_ptr); @@ -80,7 +75,7 @@ typedef hyperclient_range_query* hyperclient_range_query_asterisk; %include "proxies/hyperclient_attribute.i" %include "proxies/hyperclient_map_attribute.i" -%include "proxies/hyperclient_range_query.i" +%include "proxies/hyperclient_attribute_check.i" %include "proxies/HyperClient.i" %apply (char *BYTE) { (const char *space) } @@ -101,12 +96,13 @@ typedef hyperclient_range_query* hyperclient_range_query_asterisk; // Un-ignore a couple of enums %rename("%s") "hyperclient_returncode"; %rename("%s") "hyperdatatype"; +%rename("%s") "hyperpredicate"; %rename("%s", %$isenumitem) ""; // Un-ignore some classes I want to proxy in java %rename("%s") "hyperclient_attribute"; %rename("%s") "hyperclient_map_attribute"; -%rename("%s") "hyperclient_range_query"; +%rename("%s") "hyperclient_attribute_check"; %rename("%s",%$isvariable) ""; // Un-ignore the only needed C function diff --git a/client/java/proxies/HyperClient.i b/client/java/proxies/HyperClient.i index ea42204a..1204b49b 100644 --- a/client/java/proxies/HyperClient.i +++ b/client/java/proxies/HyperClient.i @@ -233,7 +233,7 @@ memcpy(buf + hac->value_sz, value, value_sz); hac->value = buf; hac->value_sz += value_sz; - hac->datatype = hyperdatatype.HYPERDATATYPE_STRING; + hac->datatype = HYPERDATATYPE_STRING; return 1; } @@ -247,10 +247,10 @@ { char *buf = NULL; if ((buf = (char *)malloc(sizeof(int64_t))) == NULL) return 0; - memcpy(buf, value, value_sz); + memcpy(buf, &value, sizeof(int64_t)); hac->value = buf; hac->value_sz = sizeof(int64_t); - hac->datatype = hyperdatatype.HYPERDATATYPE_INT64; + hac->datatype = HYPERDATATYPE_INT64; return 1; } @@ -264,10 +264,10 @@ { char *buf = NULL; if ((buf = (char *)malloc(sizeof(double))) == NULL) return 0; - memcpy(buf, value, value_sz); + memcpy(buf, &value, sizeof(double)); hac->value = buf; hac->value_sz = sizeof(double); - hac->datatype = hyperdatatype.HYPERDATATYPE_FLOAT; + hac->datatype = HYPERDATATYPE_FLOAT; return 1; } @@ -547,11 +547,12 @@ if ( isBytes(params) || params instanceof Long || params instanceof Double ) { - rawChecks.add(new java.util.Map.Entry< + rawChecks.add(new java.util.AbstractMap.SimpleEntry< java.util.Map.Entry< ByteArray,java.util.Map.Entry>>( new ByteArray(attrBytes), - new java.util.Map.Entry( + new java.util.AbstractMap.SimpleEntry< + hyperpredicate,Object>( hyperpredicate.HYPERPREDICATE_EQUALS, params))); } else if ( params instanceof java.util.Map.Entry ) @@ -618,7 +619,9 @@ if ( hacs == null ) throw new MemoryError(); - foreach ( java.util.Map.Entry< + int i = 0; // Collections in java can only hold MAX_INT_SIZE elements + + for ( java.util.Map.Entry< ByteArray,java.util.Map.Entry< hyperpredicate,Object>> rawCheck: rawChecks ) { @@ -653,7 +656,7 @@ return retvals; } - private static boolean isBytes(Object obj) + static boolean isBytes(Object obj) { return obj instanceof byte[] || obj instanceof ByteArray || obj instanceof String; } From f9ab9833cc30eb81178c0a4c77d59b438ab11b4b Mon Sep 17 00:00:00 2001 From: Nick Tolomiczenko Date: Tue, 15 Jan 2013 13:06:08 -0500 Subject: [PATCH 11/39] java bindings with cleanslate successfully compiles Signed-off-by: Nick Tolomiczenko --- .gitignore | 2 + client/java/extra_src/Predicate.java | 13 +- client/java/hyperclient.i | 1 + client/java/proxies/HyperClient.i | 47 +- .../proxies/hyperclient_attribute_check.i | 765 +----------------- 5 files changed, 29 insertions(+), 799 deletions(-) diff --git a/.gitignore b/.gitignore index b4faefda..587d50c0 100644 --- a/.gitignore +++ b/.gitignore @@ -58,3 +58,5 @@ stamp-h1 ylwrap *.cxx client/java/hyperclient +*.jar +*.stamp diff --git a/client/java/extra_src/Predicate.java b/client/java/extra_src/Predicate.java index c6f1fbba..494cdd6d 100644 --- a/client/java/extra_src/Predicate.java +++ b/client/java/extra_src/Predicate.java @@ -9,20 +9,21 @@ class Predicate List>> getRawChecks(ByteArray attribute) { - if ( raw = null ) + if ( raw == null ) { return null; } else { - Vector>> retval - = new Vector>>(); + Vector>> retval + = new Vector>>(); for ( Map.Entry entry: raw ) { - retval.add(new Map.Entry>( - attribute,new Map.Entry( - raw.getKey(),raw.getValue()))); + retval.add(new AbstractMap.SimpleEntry< + ByteArray,Map.Entry>( + attribute, new AbstractMap.SimpleEntry( + entry.getKey(),entry.getValue()))); } return retval; diff --git a/client/java/hyperclient.i b/client/java/hyperclient.i index bad35ee9..1bf94ff8 100644 --- a/client/java/hyperclient.i +++ b/client/java/hyperclient.i @@ -78,6 +78,7 @@ typedef hyperclient_attribute* hyperclient_attribute_asterisk; %include "proxies/hyperclient_attribute_check.i" %include "proxies/HyperClient.i" +%apply (char *BYTE) { (const char *description) } %apply (char *BYTE) { (const char *space) } %apply (char *BYTE) { (const char *sort_by) } %apply (char *STRING, int LENGTH) { (const char *key, size_t key_sz) } diff --git a/client/java/proxies/HyperClient.i b/client/java/proxies/HyperClient.i index 1204b49b..f623af0a 100644 --- a/client/java/proxies/HyperClient.i +++ b/client/java/proxies/HyperClient.i @@ -50,14 +50,6 @@ memcpy(name, str.data(), name_sz); } - static void read_attr_check_value(hyperclient_attribute_check *hac, - char *value, size_t value_sz, - size_t pos) - { - size_t available = hac->value_sz - pos; - memcpy(value, hac->value+pos, value_szvalue is already non-NULL, then we are appending to it. static int write_attr_check_value(hyperclient_attribute_check *hac, - const char *value, size_t value_sz) + const char *value, size_t value_sz) { char *buf = NULL; // Note: Since hyperclient_attribute_check array was calloced @@ -242,8 +234,7 @@ // Returns 0 on failure. hac->value will be NULL // hac->value_sz will be 0 // - static int write_attr_check_value(hyperclient_attribute_check *hac, - int64_t value) + static int write_attr_check_value(hyperclient_attribute_check *hac, int64_t value) { char *buf = NULL; if ((buf = (char *)malloc(sizeof(int64_t))) == NULL) return 0; @@ -259,8 +250,7 @@ // Returns 0 on failure. hac->value will be NULL // hac->value_sz will be 0 // - static int write_attr_check_value(hyperclient_attribute_check *hac, - double value) + static int write_attr_check_value(hyperclient_attribute_check *hac, double value) { char *buf = NULL; if ((buf = (char *)malloc(sizeof(double))) == NULL) return 0; @@ -467,38 +457,36 @@ return retType; } - // write_attr_check private static method expects value to satisfy + // write_attr_check private method expects value to satisfy // either being instance of type Long or Double or isBytes(value) == true // - private static void write_attr_check(hyperclient_attribute_check hac, + private void write_attr_check(hyperclient_attribute_check hac, ByteArray attr, Object value, hyperpredicate pred) - throws MemoryError + throws MemoryError, + TypeError { - if ( write_attr_check_name(hac,attr.getBytes,pred) == 0 ) + if ( write_attr_check_name(hac,attr.getBytes(),pred) == 0 ) { throw new MemoryError(); } if ( value instanceof Long ) { - if ( write_attr_check_value(hac, - attr.getBytes(), ((Long)value).longValue()) == 0 ) + if ( write_attr_check_value(hac,((Long)value).longValue()) == 0 ) { throw new MemoryError(); } } else if ( value instanceof Double ) { - if ( write_attr_check_value(hac, - attr.getBytes(), ((Double)value).doubleValue()) == 0 ) + if ( write_attr_check_value(hac,((Double)value).doubleValue()) == 0 ) { throw new MemoryError(); } } else // isBytes(value) must be true { - if ( write_attr_check_value(hac, - attr.getBytes(), getBytes(value)) == 0 ) + if ( write_attr_check_value(hac,getBytes(value)) == 0 ) { throw new MemoryError(); } @@ -548,8 +536,7 @@ if ( isBytes(params) || params instanceof Long || params instanceof Double ) { rawChecks.add(new java.util.AbstractMap.SimpleEntry< - java.util.Map.Entry< - ByteArray,java.util.Map.Entry>>( + ByteArray,java.util.Map.Entry>( new ByteArray(attrBytes), new java.util.AbstractMap.SimpleEntry< hyperpredicate,Object>( @@ -569,7 +556,7 @@ } rawChecks.addAll( - (new Range(lower,upper)).getRawChecks(ByteArray(attrBytes))); + (new Range(lower,upper)).getRawChecks(new ByteArray(attrBytes))); } else if ( params instanceof java.util.List ) { @@ -597,12 +584,12 @@ } rawChecks.addAll( - (new Range(lower,upper)).getRawChecks(ByteArray(attrBytes))); + (new Range(lower,upper)).getRawChecks(new ByteArray(attrBytes))); } else if ( params instanceof Predicate ) { rawChecks.addAll( - ((Predicate)params).getRawChecks(ByteArray(attrBytes))); + ((Predicate)params).getRawChecks(new ByteArray(attrBytes))); } else { @@ -1344,7 +1331,7 @@ return attrs; } - public void add_space(Object desc) throws HyperClientException + public void add_space(Object desc) throws HyperClientException, TypeError { hyperclient_returncode rc = add_space(getBytes(desc,true)); @@ -1354,7 +1341,7 @@ } } - public void rm_space(Object space) throws HyperClientException + public void rm_space(Object space) throws HyperClientException, TypeError { hyperclient_returncode rc = rm_space(getBytes(space,true)); diff --git a/client/java/proxies/hyperclient_attribute_check.i b/client/java/proxies/hyperclient_attribute_check.i index c92799a6..2b7982f6 100644 --- a/client/java/proxies/hyperclient_attribute_check.i +++ b/client/java/proxies/hyperclient_attribute_check.i @@ -8,772 +8,11 @@ %{ byte[] getAttrNameBytes() { - int name_sz = HyperClient.get_attr_name_sz(this); + int name_sz = HyperClient.get_attr_check_name_sz(this); byte[] bytes = new byte[name_sz]; - HyperClient.read_attr_name(this,bytes); + HyperClient.read_attr_check_name(this,bytes); return bytes; } - - private byte[] getAttrValueBytes() - { - int bytes_sz = HyperClient.size_t_to_int(getValue_sz()); - - return getAttrValueBytes(0,bytes_sz); - } - - private byte[] getAttrValueBytes(long pos, int size) - { - byte[] bytes = new byte[size]; - HyperClient.read_attr_value(this,bytes,pos); - return bytes; - } - - private java.lang.Object getAttrStringValue(String defaultStringEncoding) - { - ByteArray attrValue = new ByteArray(getAttrValueBytes()); - attrValue.setDefaultEncoding(defaultStringEncoding); - return attrValue; - } - - private byte[] zerofill(byte[] inbytes) - { - byte[] outbytes = new byte[8]; - - // Make sure outbytes is initialized to 0 - for (int i=0; i<8; i++ ) outbytes[i] = 0; - - // Copy little-endingly - for (int i=0; i coll, - String defaultStringEncoding) throws ValueError - { - String collType = coll instanceof java.util.List?"list":"set"; - - // Interpret return value of getValue_sz() as unsigned - // - java.math.BigInteger value_sz - = new java.math.BigInteger(1,java.nio.ByteBuffer.allocate(8).order( - java.nio.ByteOrder.BIG_ENDIAN).putLong(getValue_sz()).array()); - - java.math.BigInteger rem = new java.math.BigInteger(value_sz.toString()); - long pos = 0; - - java.math.BigInteger four = new java.math.BigInteger("4"); - - int coll_sz = 0; - - while ( rem.compareTo(four) >= 0 && coll_sz <= Integer.MAX_VALUE ) - { - int sz - = java.nio.ByteBuffer.wrap(getAttrValueBytes(pos,4)).order( - java.nio.ByteOrder.LITTLE_ENDIAN).getInt(); - - java.math.BigInteger sz_bi - = new java.math.BigInteger(new Integer(sz).toString()); - - if ( rem.subtract(four).compareTo(sz_bi) < 0 ) - { - throw new ValueError(collType - + "(string) is improperly structured (file a bug)"); - } - - ByteArray collElement = new ByteArray(getAttrValueBytes(pos+4,sz)); - collElement.setDefaultEncoding(defaultStringEncoding); - coll.add(collElement); - - rem = rem.subtract(four).subtract(sz_bi); - pos = value_sz.subtract(rem).longValue(); - - coll_sz += 1; - } - - if ( coll_sz < Integer.MAX_VALUE && rem.compareTo(java.math.BigInteger.ZERO) > 0 ) - { - throw new ValueError(collType + "(string) contains excess data (file a bug)"); - } - - return coll; - } - - private java.lang.Object getAttrCollectionLongValue( - java.util.AbstractCollection coll) throws ValueError - { - String collType = coll instanceof java.util.List?"list":"set"; - - // Interpret return value of getValue_sz() as unsigned - // - java.math.BigInteger value_sz - = new java.math.BigInteger(1,java.nio.ByteBuffer.allocate(8).order( - java.nio.ByteOrder.BIG_ENDIAN).putLong(getValue_sz()).array()); - - java.math.BigInteger rem = new java.math.BigInteger(value_sz.toString()); - long pos = 0; - - java.math.BigInteger eight = new java.math.BigInteger("8"); - - int coll_sz = 0; - - while ( rem.compareTo(eight) >= 0 && coll_sz <= Integer.MAX_VALUE ) - { - coll.add(new Long(java.nio.ByteBuffer.wrap(getAttrValueBytes(pos,8)).order( - java.nio.ByteOrder.LITTLE_ENDIAN).getLong())); - - rem = rem.subtract(eight); - pos = value_sz.subtract(rem).longValue(); - - coll_sz += 1; - } - - if ( coll_sz < Integer.MAX_VALUE && rem.compareTo(java.math.BigInteger.ZERO) > 0 ) - { - throw new ValueError(collType + "(int64) contains excess data (file a bug)"); - } - - return coll; - } - - private java.lang.Object getAttrCollectionDoubleValue( - java.util.AbstractCollection coll) throws ValueError - { - String collType = coll instanceof java.util.List?"list":"set"; - - // Interpret return value of getValue_sz() as unsigned - // - java.math.BigInteger value_sz - = new java.math.BigInteger(1,java.nio.ByteBuffer.allocate(8).order( - java.nio.ByteOrder.BIG_ENDIAN).putLong(getValue_sz()).array()); - - java.math.BigInteger rem = new java.math.BigInteger(value_sz.toString()); - long pos = 0; - - java.math.BigInteger eight = new java.math.BigInteger("8"); - - int coll_sz = 0; - - while ( rem.compareTo(eight) >= 0 && coll_sz <= Integer.MAX_VALUE ) - { - coll.add(new Double(java.nio.ByteBuffer.wrap(getAttrValueBytes(pos,8)).order( - java.nio.ByteOrder.LITTLE_ENDIAN).getDouble())); - - rem = rem.subtract(eight); - pos = value_sz.subtract(rem).longValue(); - - coll_sz += 1; - } - - if ( coll_sz < Integer.MAX_VALUE && rem.compareTo(java.math.BigInteger.ZERO) > 0 ) - { - throw new ValueError(collType + "(float) contains excess data (file a bug)"); - } - - return coll; - } - - private java.lang.Object getAttrMapStringStringValue(String defaultStringEncoding) - throws ValueError - { - ByteArrayKeyedSortedMap map - = new ByteArrayKeyedSortedMap(); - - // Interpret return value of getValue_sz() as unsigned - // - java.math.BigInteger value_sz - = new java.math.BigInteger(1,java.nio.ByteBuffer.allocate(8).order( - java.nio.ByteOrder.BIG_ENDIAN).putLong(getValue_sz()).array()); - - java.math.BigInteger rem = new java.math.BigInteger(value_sz.toString()); - long pos = 0; - - java.math.BigInteger four = new java.math.BigInteger("4"); - - int map_sz = 0; - - while ( rem.compareTo(four) >= 0 && map_sz <= Integer.MAX_VALUE ) - { - int sz - = java.nio.ByteBuffer.wrap(getAttrValueBytes(pos,4)).order( - java.nio.ByteOrder.LITTLE_ENDIAN).getInt(); - - java.math.BigInteger sz_bi - = new java.math.BigInteger(new Integer(sz).toString()); - - if ( rem.subtract(four).compareTo(sz_bi) < 0 ) - { - throw new ValueError( - "map(string,string) is improperly structured (file a bug)"); - } - - ByteArray key = new ByteArray(getAttrValueBytes(pos+4,sz)); - key.setDefaultEncoding(defaultStringEncoding); - - rem = rem.subtract(four).subtract(sz_bi); - pos = value_sz.subtract(rem).longValue(); - - sz = java.nio.ByteBuffer.wrap(getAttrValueBytes(pos,4)).order( - java.nio.ByteOrder.LITTLE_ENDIAN).getInt(); - - sz_bi = new java.math.BigInteger(new Integer(sz).toString()); - - if ( rem.subtract(four).compareTo(sz_bi) < 0 ) - { - throw new ValueError( - "map(string,string) is improperly structured (file a bug)"); - } - - ByteArray val = new ByteArray(getAttrValueBytes(pos+4,sz)); - val.setDefaultEncoding(defaultStringEncoding); - - rem = rem.subtract(four).subtract(sz_bi); - pos = value_sz.subtract(rem).longValue(); - - map.put(key,val); - - map_sz += 1; - } - - if ( map_sz < Integer.MAX_VALUE && rem.compareTo(java.math.BigInteger.ZERO) > 0 ) - { - throw new ValueError("map(string,string) contains excess data (file a bug)"); - } - - return map; - } - - private java.lang.Object getAttrMapStringLongValue(String defaultStringEncoding) - throws ValueError - { - ByteArrayKeyedSortedMap map - = new ByteArrayKeyedSortedMap(); - - // Interpret return value of getValue_sz() as unsigned - // - java.math.BigInteger value_sz - = new java.math.BigInteger(1,java.nio.ByteBuffer.allocate(8).order( - java.nio.ByteOrder.BIG_ENDIAN).putLong(getValue_sz()).array()); - - java.math.BigInteger rem = new java.math.BigInteger(value_sz.toString()); - long pos = 0; - - java.math.BigInteger four = new java.math.BigInteger("4"); - java.math.BigInteger eight = new java.math.BigInteger("8"); - - int map_sz = 0; - - while ( rem.compareTo(four) >= 0 && map_sz <= Integer.MAX_VALUE ) - { - int sz - = java.nio.ByteBuffer.wrap(getAttrValueBytes(pos,4)).order( - java.nio.ByteOrder.LITTLE_ENDIAN).getInt(); - - java.math.BigInteger sz_bi - = new java.math.BigInteger(new Integer(sz).toString()); - - if ( rem.subtract(four).compareTo(sz_bi) < 0 ) - { - throw new ValueError( - "map(string,int64) is improperly structured (file a bug)"); - } - - ByteArray key = new ByteArray(getAttrValueBytes(pos+4,sz)); - key.setDefaultEncoding(defaultStringEncoding); - - rem = rem.subtract(four).subtract(sz_bi); - pos = value_sz.subtract(rem).longValue(); - - if ( rem.compareTo(eight) < 0 ) - { - throw new ValueError( - "map(string,int64) is improperly structured (file a bug)"); - } - - Long val = new Long(java.nio.ByteBuffer.wrap(getAttrValueBytes(pos,8)).order( - java.nio.ByteOrder.LITTLE_ENDIAN).getLong()); - - rem = rem.subtract(eight); - pos = value_sz.subtract(rem).longValue(); - - map.put(key,val); - - map_sz += 1; - } - - if ( map_sz < Integer.MAX_VALUE && rem.compareTo(java.math.BigInteger.ZERO) > 0 ) - { - throw new ValueError("map(string,int64) contains excess data (file a bug)"); - } - - return map; - } - - private java.lang.Object getAttrMapStringDoubleValue(String defaultStringEncoding) - throws ValueError - { - ByteArrayKeyedSortedMap - map = new ByteArrayKeyedSortedMap(); - - // Interpret return value of getValue_sz() as unsigned - // - java.math.BigInteger value_sz - = new java.math.BigInteger(1,java.nio.ByteBuffer.allocate(8).order( - java.nio.ByteOrder.BIG_ENDIAN).putLong(getValue_sz()).array()); - - java.math.BigInteger rem = new java.math.BigInteger(value_sz.toString()); - long pos = 0; - - java.math.BigInteger four = new java.math.BigInteger("4"); - java.math.BigInteger eight = new java.math.BigInteger("8"); - - int map_sz = 0; - - while ( rem.compareTo(four) >= 0 && map_sz <= Integer.MAX_VALUE ) - { - int sz - = java.nio.ByteBuffer.wrap(getAttrValueBytes(pos,4)).order( - java.nio.ByteOrder.LITTLE_ENDIAN).getInt(); - - java.math.BigInteger sz_bi - = new java.math.BigInteger(new Integer(sz).toString()); - - if ( rem.subtract(four).compareTo(sz_bi) < 0 ) - { - throw new ValueError( - "map(string,float) is improperly structured (file a bug)"); - } - - ByteArray key = new ByteArray(getAttrValueBytes(pos+4,sz)); - key.setDefaultEncoding(defaultStringEncoding); - - rem = rem.subtract(four).subtract(sz_bi); - pos = value_sz.subtract(rem).longValue(); - - if ( rem.compareTo(eight) < 0 ) - { - throw new ValueError( - "map(string,float) is improperly structured (file a bug)"); - } - - Double val = new Double(java.nio.ByteBuffer.wrap(getAttrValueBytes(pos,8)).order( - java.nio.ByteOrder.LITTLE_ENDIAN).getDouble()); - - rem = rem.subtract(eight); - pos = value_sz.subtract(rem).longValue(); - - map.put(key,val); - - map_sz += 1; - } - - if ( map_sz < Integer.MAX_VALUE && rem.compareTo(java.math.BigInteger.ZERO) > 0 ) - { - throw new ValueError("map(string,float) contains excess data (file a bug)"); - } - - return map; - } - - private java.lang.Object getAttrMapLongStringValue(String defaultStringEncoding) - throws ValueError - { - java.util.HashMap map = new java.util.HashMap(); - - // Interpret return value of getValue_sz() as unsigned - // - java.math.BigInteger value_sz - = new java.math.BigInteger(1,java.nio.ByteBuffer.allocate(8).order( - java.nio.ByteOrder.BIG_ENDIAN).putLong(getValue_sz()).array()); - - java.math.BigInteger rem = new java.math.BigInteger(value_sz.toString()); - long pos = 0; - - java.math.BigInteger four = new java.math.BigInteger("4"); - java.math.BigInteger eight = new java.math.BigInteger("8"); - - int map_sz = 0; - - while ( rem.compareTo(eight) >= 0 && map_sz <= Integer.MAX_VALUE ) - { - Long key = new Long(java.nio.ByteBuffer.wrap(getAttrValueBytes(pos,8)).order( - java.nio.ByteOrder.LITTLE_ENDIAN).getLong()); - - rem = rem.subtract(eight); - pos = value_sz.subtract(rem).longValue(); - - int sz - = java.nio.ByteBuffer.wrap(getAttrValueBytes(pos,4)).order( - java.nio.ByteOrder.LITTLE_ENDIAN).getInt(); - - java.math.BigInteger sz_bi - = new java.math.BigInteger(new Integer(sz).toString()); - - if ( rem.subtract(four).compareTo(sz_bi) < 0 ) - { - throw new ValueError( - "map(int64,string) is improperly structured (file a bug)"); - } - - ByteArray val = new ByteArray(getAttrValueBytes(pos+4,sz)); - val.setDefaultEncoding(defaultStringEncoding); - - rem = rem.subtract(four).subtract(sz_bi); - pos = value_sz.subtract(rem).longValue(); - - map.put(key,val); - - map_sz += 1; - } - - if ( map_sz < Integer.MAX_VALUE && rem.compareTo(java.math.BigInteger.ZERO) > 0 ) - { - throw new ValueError("map(int64,string) contains excess data (file a bug)"); - } - - return map; - } - - private java.lang.Object getAttrMapLongLongValue() throws ValueError - { - java.util.HashMap map = new java.util.HashMap(); - - // Interpret return value of getValue_sz() as unsigned - // - java.math.BigInteger value_sz - = new java.math.BigInteger(1,java.nio.ByteBuffer.allocate(8).order( - java.nio.ByteOrder.BIG_ENDIAN).putLong(getValue_sz()).array()); - - java.math.BigInteger rem = new java.math.BigInteger(value_sz.toString()); - long pos = 0; - - java.math.BigInteger eight = new java.math.BigInteger("8"); - java.math.BigInteger sixteen = new java.math.BigInteger("16"); - - int map_sz = 0; - - while ( rem.compareTo(sixteen) >= 0 && map_sz <= Integer.MAX_VALUE ) - { - Long key = new Long(java.nio.ByteBuffer.wrap(getAttrValueBytes(pos,8)).order( - java.nio.ByteOrder.LITTLE_ENDIAN).getLong()); - - rem = rem.subtract(eight); - pos = value_sz.subtract(rem).longValue(); - - Long val = new Long(java.nio.ByteBuffer.wrap(getAttrValueBytes(pos,8)).order( - java.nio.ByteOrder.LITTLE_ENDIAN).getLong()); - - rem = rem.subtract(eight); - pos = value_sz.subtract(rem).longValue(); - - map.put(key,val); - - map_sz += 1; - } - - if ( map_sz < Integer.MAX_VALUE && rem.compareTo(java.math.BigInteger.ZERO) > 0 ) - { - throw new ValueError("map(int64,int64) contains excess data (file a bug)"); - } - - return map; - } - - private java.lang.Object getAttrMapLongDoubleValue() throws ValueError - { - java.util.HashMap map = new java.util.HashMap(); - - // Interpret return value of getValue_sz() as unsigned - // - java.math.BigInteger value_sz - = new java.math.BigInteger(1,java.nio.ByteBuffer.allocate(8).order( - java.nio.ByteOrder.BIG_ENDIAN).putLong(getValue_sz()).array()); - - java.math.BigInteger rem = new java.math.BigInteger(value_sz.toString()); - long pos = 0; - - java.math.BigInteger eight = new java.math.BigInteger("8"); - java.math.BigInteger sixteen = new java.math.BigInteger("16"); - - int map_sz = 0; - - while ( rem.compareTo(sixteen) >= 0 && map_sz <= Integer.MAX_VALUE ) - { - Long key = new Long(java.nio.ByteBuffer.wrap(getAttrValueBytes(pos,8)).order( - java.nio.ByteOrder.LITTLE_ENDIAN).getLong()); - - rem = rem.subtract(eight); - pos = value_sz.subtract(rem).longValue(); - - Double val = new Double(java.nio.ByteBuffer.wrap(getAttrValueBytes(pos,8)).order( - java.nio.ByteOrder.LITTLE_ENDIAN).getDouble()); - - rem = rem.subtract(eight); - pos = value_sz.subtract(rem).longValue(); - - map.put(key,val); - - map_sz += 1; - } - - if ( map_sz < Integer.MAX_VALUE && rem.compareTo(java.math.BigInteger.ZERO) > 0 ) - { - throw new ValueError("map(int64,float) contains excess data (file a bug)"); - } - - return map; - } - - private java.lang.Object getAttrMapDoubleStringValue(String defaultStringEncoding) - throws ValueError - { - java.util.HashMap map = new java.util.HashMap(); - - // Interpret return value of getValue_sz() as unsigned - // - java.math.BigInteger value_sz - = new java.math.BigInteger(1,java.nio.ByteBuffer.allocate(8).order( - java.nio.ByteOrder.BIG_ENDIAN).putLong(getValue_sz()).array()); - - java.math.BigInteger rem = new java.math.BigInteger(value_sz.toString()); - long pos = 0; - - java.math.BigInteger four = new java.math.BigInteger("4"); - java.math.BigInteger eight = new java.math.BigInteger("8"); - - int map_sz = 0; - - while ( rem.compareTo(eight) >= 0 && map_sz <= Integer.MAX_VALUE ) - { - Double key = new Double(java.nio.ByteBuffer.wrap(getAttrValueBytes(pos,8)).order( - java.nio.ByteOrder.LITTLE_ENDIAN).getDouble()); - - rem = rem.subtract(eight); - pos = value_sz.subtract(rem).longValue(); - - int sz - = java.nio.ByteBuffer.wrap(getAttrValueBytes(pos,4)).order( - java.nio.ByteOrder.LITTLE_ENDIAN).getInt(); - - java.math.BigInteger sz_bi - = new java.math.BigInteger(new Integer(sz).toString()); - - if ( rem.subtract(four).compareTo(sz_bi) < 0 ) - { - throw new ValueError( - "map(float,string) is improperly structured (file a bug)"); - } - - ByteArray val = new ByteArray(getAttrValueBytes(pos+4,sz)); - val.setDefaultEncoding(defaultStringEncoding); - - rem = rem.subtract(four).subtract(sz_bi); - pos = value_sz.subtract(rem).longValue(); - - map.put(key,val); - - map_sz += 1; - } - - if ( map_sz < Integer.MAX_VALUE && rem.compareTo(java.math.BigInteger.ZERO) > 0 ) - { - throw new ValueError("map(float,string) contains excess data (file a bug)"); - } - - return map; - } - - private java.lang.Object getAttrMapDoubleLongValue() throws ValueError - { - java.util.HashMap map = new java.util.HashMap(); - - // Interpret return value of getValue_sz() as unsigned - // - java.math.BigInteger value_sz - = new java.math.BigInteger(1,java.nio.ByteBuffer.allocate(8).order( - java.nio.ByteOrder.BIG_ENDIAN).putLong(getValue_sz()).array()); - - java.math.BigInteger rem = new java.math.BigInteger(value_sz.toString()); - long pos = 0; - - java.math.BigInteger eight = new java.math.BigInteger("8"); - java.math.BigInteger sixteen = new java.math.BigInteger("16"); - - int map_sz = 0; - - while ( rem.compareTo(sixteen) >= 0 && map_sz <= Integer.MAX_VALUE ) - { - Double key = new Double(java.nio.ByteBuffer.wrap(getAttrValueBytes(pos,8)).order( - java.nio.ByteOrder.LITTLE_ENDIAN).getDouble()); - - rem = rem.subtract(eight); - pos = value_sz.subtract(rem).longValue(); - - Long val = new Long(java.nio.ByteBuffer.wrap(getAttrValueBytes(pos,8)).order( - java.nio.ByteOrder.LITTLE_ENDIAN).getLong()); - - rem = rem.subtract(eight); - pos = value_sz.subtract(rem).longValue(); - - map.put(key,val); - - map_sz += 1; - } - - if ( map_sz < Integer.MAX_VALUE && rem.compareTo(java.math.BigInteger.ZERO) > 0 ) - { - throw new ValueError("map(float,int64) contains excess data (file a bug)"); - } - - return map; - } - - private java.lang.Object getAttrMapDoubleDoubleValue() throws ValueError - { - java.util.HashMap map = new java.util.HashMap(); - - // Interpret return value of getValue_sz() as unsigned - // - java.math.BigInteger value_sz - = new java.math.BigInteger(1,java.nio.ByteBuffer.allocate(8).order( - java.nio.ByteOrder.BIG_ENDIAN).putLong(getValue_sz()).array()); - - java.math.BigInteger rem = new java.math.BigInteger(value_sz.toString()); - long pos = 0; - - java.math.BigInteger eight = new java.math.BigInteger("8"); - java.math.BigInteger sixteen = new java.math.BigInteger("16"); - - int map_sz = 0; - - while ( rem.compareTo(sixteen) >= 0 && map_sz <= Integer.MAX_VALUE ) - { - Double key = new Double(java.nio.ByteBuffer.wrap(getAttrValueBytes(pos,8)).order( - java.nio.ByteOrder.LITTLE_ENDIAN).getDouble()); - - rem = rem.subtract(eight); - pos = value_sz.subtract(rem).longValue(); - - Double val = new Double(java.nio.ByteBuffer.wrap(getAttrValueBytes(pos,8)).order( - java.nio.ByteOrder.LITTLE_ENDIAN).getDouble()); - - rem = rem.subtract(eight); - pos = value_sz.subtract(rem).longValue(); - - map.put(key,val); - - map_sz += 1; - } - - if ( map_sz < Integer.MAX_VALUE && rem.compareTo(java.math.BigInteger.ZERO) > 0 ) - { - throw new ValueError("map(float,float) contains excess data (file a bug)"); - } - - return map; - } - - public java.lang.Object getAttrValue(String defaultStringEncoding) throws ValueError - { - switch(getDatatype()) - { - case HYPERDATATYPE_STRING: - return getAttrStringValue(defaultStringEncoding); - - case HYPERDATATYPE_INT64: - return getAttrLongValue(); - - case HYPERDATATYPE_FLOAT: - return getAttrDoubleValue(); - - case HYPERDATATYPE_LIST_STRING: - ByteArrayVector ls = new ByteArrayVector(); - getAttrCollectionStringValue(ls,defaultStringEncoding); - return ls; - - case HYPERDATATYPE_LIST_INT64: - java.util.Vector li = new java.util.Vector(); - getAttrCollectionLongValue(li); - return li; - - case HYPERDATATYPE_LIST_FLOAT: - java.util.Vector lf = new java.util.Vector(); - getAttrCollectionDoubleValue(lf); - return lf; - - case HYPERDATATYPE_SET_STRING: - ByteArraySortedSet ss = new ByteArraySortedSet(); - getAttrCollectionStringValue(ss,defaultStringEncoding); - return ss; - - case HYPERDATATYPE_SET_INT64: - java.util.HashSet si = new java.util.HashSet(); - getAttrCollectionLongValue(si); - return si; - - case HYPERDATATYPE_SET_FLOAT: - java.util.HashSet sf = new java.util.HashSet(); - getAttrCollectionDoubleValue(sf); - return sf; - - case HYPERDATATYPE_MAP_STRING_STRING: - return getAttrMapStringStringValue(defaultStringEncoding); - - case HYPERDATATYPE_MAP_STRING_INT64: - return getAttrMapStringLongValue(defaultStringEncoding); - - case HYPERDATATYPE_MAP_STRING_FLOAT: - return getAttrMapStringDoubleValue(defaultStringEncoding); - - case HYPERDATATYPE_MAP_INT64_STRING: - return getAttrMapLongStringValue(defaultStringEncoding); - - case HYPERDATATYPE_MAP_INT64_INT64: - return getAttrMapLongLongValue(); - - case HYPERDATATYPE_MAP_INT64_FLOAT: - return getAttrMapLongDoubleValue(); - - case HYPERDATATYPE_MAP_FLOAT_STRING: - return getAttrMapDoubleStringValue(defaultStringEncoding); - - case HYPERDATATYPE_MAP_FLOAT_INT64: - return getAttrMapDoubleLongValue(); - - case HYPERDATATYPE_MAP_FLOAT_FLOAT: - return getAttrMapDoubleDoubleValue(); - - default: - throw new ValueError("Server returned garbage value (file a bug)"); - } - } %} From ae9c27c23a0013dd7776e127160e3962e14b4210 Mon Sep 17 00:00:00 2001 From: Nick Tolomiczenko Date: Wed, 16 Jan 2013 21:36:07 -0500 Subject: [PATCH 12/39] Java bindings passed preliminary test examples Signed-off-by: Nick Tolomiczenko --- client/java/examples/HyperAllTypes.java | 125 ------------------ client/java/examples/HyperBinaryTest.java | 15 ++- .../java/examples/HyperSearchFloatRange.java | 25 ++-- client/java/examples/HyperTest.java | 21 ++- client/java/examples/README.txt | 1 - client/java/extra_src/DeferredCondPut.java | 2 +- client/java/proxies/HyperClient.i | 6 +- configure.ac | 5 - 8 files changed, 44 insertions(+), 156 deletions(-) delete mode 100644 client/java/examples/HyperAllTypes.java diff --git a/client/java/examples/HyperAllTypes.java b/client/java/examples/HyperAllTypes.java deleted file mode 100644 index baf8f67f..00000000 --- a/client/java/examples/HyperAllTypes.java +++ /dev/null @@ -1,125 +0,0 @@ -/* This example uses the following space definition */ -/* -space alltypes -dimensions keystr, str, int (int64), flt (float), list_str (list (string)), list_int (list (int64)), list_flt (list (float)), set_str (set (string)), set_int (set (int64)), set_flt (set (float)), str2str (map (string, string)), str2int (map (string, int64)), str2flt (map (string, float)), int2str (map (int64, string)), int2int (map (int64, int64)), int2flt (map (int64, float)), flt2str (map (float, string)), flt2int (map (float, int64)), flt2flt (map (float, float)) -key keystr auto 1 3 -*/ - - -import hyperclient.*; -import java.util.*; - -public class HyperAllTypes -{ - public static void main(String[] args) throws Exception - { - HashMap attrs = new HashMap(); - - // Primitive types - // - attrs.put("str","Hello"); - attrs.put("int", 42L); - attrs.put("flt", 3.14D); - - // List types - // - Vector list_str = new Vector(); - list_str.add("dog"); - list_str.add("cat"); - list_str.add("horse"); - attrs.put("list_str", list_str); - - Vector list_int = new Vector(); - list_int.add(1L); - list_int.add(2L); - list_int.add(3L); - attrs.put("list_int", list_int); - - Vector list_flt = new Vector(); - list_flt.add(0.1D); - list_flt.add(0.01D); - list_flt.add(0.001D); - attrs.put("list_flt", list_flt); - - // Set types - // - TreeSet set_str = new TreeSet(); - set_str.add("dog"); - set_str.add("cat"); - set_str.add("horse"); - attrs.put("set_str", set_str); - - TreeSet set_int = new TreeSet(); - set_int.add(1L); - set_int.add(2L); - set_int.add(3L); - attrs.put("set_int", set_int); - - TreeSet set_flt = new TreeSet(); - set_flt.add(0.1D); - set_flt.add(0.01D); - set_flt.add(0.001D); - attrs.put("set_flt", set_flt); - - // Map types - // - HashMap str2str = new HashMap(); - str2str.put("dog", "canine"); - str2str.put("cat", "feline"); - str2str.put("horse", "equine"); - attrs.put("str2str", str2str); - - HashMap str2int = new HashMap(); - str2int.put("one", 1L); - str2int.put("two", 2L); - str2int.put("three", 3L); - attrs.put("str2int", str2int); - - HashMap str2flt = new HashMap(); - str2flt.put("tenth", 0.1); - str2flt.put("hundredth", 0.01); - str2flt.put("thousandth", 0.001); - attrs.put("str2flt", str2flt); - - HashMap int2str = new HashMap(); - int2str.put(1L, "canine"); - int2str.put(2L, "feline"); - int2str.put(3L, "equine"); - attrs.put("int2str", int2str); - - HashMap int2int = new HashMap(); - int2int.put(1L, 1L); - int2int.put(2L, 2L); - int2int.put(3L, 3L); - attrs.put("int2int", int2int); - - HashMap int2flt = new HashMap(); - int2flt.put(1L, 0.1D); - int2flt.put(2L, 0.01D); - int2flt.put(3L, 0.001D); - attrs.put("int2flt", int2flt); - - HashMap flt2str = new HashMap(); - flt2str.put(0.1D, "canine"); - flt2str.put(0.01D, "feline"); - flt2str.put(0.001D, "equine"); - attrs.put("flt2str", flt2str); - - HashMap flt2int = new HashMap(); - flt2int.put(0.1D, 1L); - flt2int.put(0.01D, 2L); - flt2int.put(0.001D, 3L); - attrs.put("flt2int", flt2int); - - HashMap flt2flt = new HashMap(); - flt2flt.put(0.1D, 0.1D); - flt2flt.put(0.01D, 0.01D); - flt2flt.put(0.001D, 0.001D); - attrs.put("flt2flt", flt2flt); - - HyperClient c = new HyperClient("127.0.0.1",1234); - - System.out.println("put: " + c.put("alltypes","key1",attrs)); - System.out.println("get: " + c.get("alltypes","key1")); - } -} diff --git a/client/java/examples/HyperBinaryTest.java b/client/java/examples/HyperBinaryTest.java index 13162963..e12d26b9 100644 --- a/client/java/examples/HyperBinaryTest.java +++ b/client/java/examples/HyperBinaryTest.java @@ -1,8 +1,6 @@ import hyperclient.*; import java.util.*; -/* This example uses the phonebook space from the tutorial */ - public class HyperBinaryTest { public static void printRawBytes(byte[] bytes) @@ -22,7 +20,16 @@ public static void printRawBytes(byte[] bytes) public static void main(String[] args) throws Exception { - HyperClient c = new HyperClient("127.0.0.1",1234); + HyperClient c = new HyperClient("127.0.0.1",1982); + + String desc = "space phonebook " + + "key username " + + "attributes first, last, int phone " + + "subspace first, last, phone"; + + System.out.println("\n\nSpace Description:\n\n" + desc + "\n\n"); + + c.add_space(desc); HashMap attrs = new HashMap(); @@ -90,5 +97,7 @@ public static void main(String[] args) throws Exception System.out.println("put success: " + c.put("phonebook", "mykey", attrs)); map = c.get("phonebook", "mykey"); System.out.println(((ByteArray)(map.get("first"))).decode("UTF-16BE")); + + c.rm_space("phonebook"); } } diff --git a/client/java/examples/HyperSearchFloatRange.java b/client/java/examples/HyperSearchFloatRange.java index 308f4bcc..9d6c8465 100644 --- a/client/java/examples/HyperSearchFloatRange.java +++ b/client/java/examples/HyperSearchFloatRange.java @@ -1,12 +1,3 @@ -/* This example uses space definition */ -/* -space weightbook -dimensions username, first, last, weight (float) -key username auto 1 3 -subspace first, last, weight auto 2 3 -*/ - - import hyperclient.*; import java.util.*; @@ -16,7 +7,16 @@ public static void main(String[] args) throws Exception { HashMap values = new HashMap(); - HyperClient c = new HyperClient("127.0.0.1",1234); + HyperClient c = new HyperClient("127.0.0.1",1982); + + String desc = "space weightbook " + + "key username " + + "attributes first, last, float weight " + + "subspace first, last, weight"; + + System.out.println("\n\nSpace Description:\n\n" + desc + "\n\n"); + + c.add_space(desc); values.put("first","Nick"); values.put("last","Tolomiczenko"); @@ -57,11 +57,11 @@ public static void main(String[] args) throws Exception System.out.println(s.next()); } - // Now add a range stipulation on the weight: [160.1,165) + // Now add a range stipulation on the weight: [160.1,165] values.put("weight", new AbstractMap.SimpleEntry(160.1D,165D)); - System.out.println("\nSearching for last name of 'Tolomiczenko'\n\n AND\n\nweight in the range [160.1,165):\n"); + System.out.println("\nSearching for last name of 'Tolomiczenko'\n\n AND\n\nweight in the range [160.1,165]:\n"); // Do the search again s = c.search("weightbook",values); @@ -86,5 +86,6 @@ public static void main(String[] args) throws Exception System.out.println(s.next()); } + c.rm_space("weightbook"); } } diff --git a/client/java/examples/HyperTest.java b/client/java/examples/HyperTest.java index da9480a8..a13b6239 100644 --- a/client/java/examples/HyperTest.java +++ b/client/java/examples/HyperTest.java @@ -1,5 +1,3 @@ -/* This example uses the phonebook space from the tutorial */ - import hyperclient.*; import java.util.*; @@ -9,7 +7,16 @@ public static void main(String[] args) throws Exception { HashMap values = new HashMap(); - HyperClient c = new HyperClient("127.0.0.1",1234); + HyperClient c = new HyperClient("127.0.0.1",1982); + + String desc = "space phonebook " + + "key username " + + "attributes first, last, int phone " + + "subspace first, last, phone"; + + System.out.println("\n\nSpace Description:\n\n" + desc + "\n\n"); + + c.add_space(desc); values.put("first","Nick"); values.put("last","Tolomiczenko"); @@ -50,11 +57,11 @@ public static void main(String[] args) throws Exception System.out.println(s.next()); } - // Now add a range stipulation on the phone number: [4165551024,4165551026) + // Now add a range stipulation on the phone number: [4165551024,4165551026] values.put("phone", new AbstractMap.SimpleEntry(4165551024L,4165551026L)); - System.out.println("\nSearching for last name of 'Tolomiczenko'\n\n AND\n\nphone number in the range [4165551024,4165551026):\n"); + System.out.println("\nSearching for last name of 'Tolomiczenko'\n\n AND\n\nphone number in the range [4165551024,4165551026]:\n"); // Do the search again s = c.search("phonebook",values); @@ -138,7 +145,7 @@ public static void main(String[] args) throws Exception System.out.println(s.next()); } - System.out.println("\nGroup delete for last name of 'Tolomiczenko'\n\n AND\n\nphone number in the range [4165551024,4165551026):\n"); + System.out.println("\nGroup delete for last name of 'Tolomiczenko'\n\n AND\n\nphone number in the range [4165551024,4165551026]:\n"); values.put("phone", new AbstractMap.SimpleEntry(4165551024L,4165551026L)); System.out.println("result: " + c.group_del("phonebook",values)); @@ -152,5 +159,7 @@ public static void main(String[] args) throws Exception { System.out.println(s.next()); } + + c.rm_space("phonebook"); } } diff --git a/client/java/examples/README.txt b/client/java/examples/README.txt index 92c2ede2..e845bc13 100644 --- a/client/java/examples/README.txt +++ b/client/java/examples/README.txt @@ -2,6 +2,5 @@ INSTRUCTIONS: . source_me javac HyperTest.java -javac HyperAllTypes.java javac HyperBinaryTest.java javac HyperSearchFloatRange.java diff --git a/client/java/extra_src/DeferredCondPut.java b/client/java/extra_src/DeferredCondPut.java index dc2e4d2c..d7414229 100644 --- a/client/java/extra_src/DeferredCondPut.java +++ b/client/java/extra_src/DeferredCondPut.java @@ -33,7 +33,7 @@ public DeferredCondPut(HyperClient client, Object space, Object key, attrs_sz = value.size(); attrs = client.dict_to_attrs(value); - reqId = client.condput(client.getBytes(space,true), + reqId = client.cond_put(client.getBytes(space,true), client.getBytes(key), condattrs, condattrs_sz, attrs, attrs_sz, diff --git a/client/java/proxies/HyperClient.i b/client/java/proxies/HyperClient.i index f623af0a..9497e952 100644 --- a/client/java/proxies/HyperClient.i +++ b/client/java/proxies/HyperClient.i @@ -1361,14 +1361,14 @@ return (java.util.Map)(d.waitFor()); } - public boolean condput(Object space, Object key, java.util.Map condition, + public boolean cond_put(Object space, Object key, java.util.Map condition, java.util.Map value) throws HyperClientException, TypeError, MemoryError, ValueError { - Deferred d = (DeferredCondPut)(async_condput(space, key, condition, value)); + Deferred d = (DeferredCondPut)(async_cond_put(space, key, condition, value)); return ((Boolean)(d.waitFor())).booleanValue(); } @@ -1772,7 +1772,7 @@ return new DeferredGet(this,space, key); } - public Deferred async_condput(Object space, Object key, java.util.Map condition, + public Deferred async_cond_put(Object space, Object key, java.util.Map condition, java.util.Map value) throws HyperClientException, TypeError, diff --git a/configure.ac b/configure.ac index 87cc7e2d..57a8f3dc 100644 --- a/configure.ac +++ b/configure.ac @@ -28,11 +28,6 @@ need_swig=no JNI_CPPFLAGS= if test x"${java_bindings}" = xyes; then - AC_MSG_ERROR([ -------------------------------------------------- -The Java bindings are temporarily unavailable. -They will be available in HyperDex 1.0 --------------------------------------------------]) AC_PROG_JAR AC_PROG_JAVAC AC_JNI_INCLUDE_DIR From bc6b9d3393e6b4c5fc5a90bb283a07d4fabdf5c9 Mon Sep 17 00:00:00 2001 From: Robert Escriva Date: Wed, 16 Jan 2013 11:06:11 -0500 Subject: [PATCH 13/39] Python shell --- doc/03.quickstart.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/03.quickstart.rst b/doc/03.quickstart.rst index 17c04e76..204e61a1 100644 --- a/doc/03.quickstart.rst +++ b/doc/03.quickstart.rst @@ -105,7 +105,7 @@ In HyperDex, each such collection of data is called a *space* and is conceptually similar to *tables* in traditional databases. Let's see how we can instruct HyperDex to create a suitable space for holding such objects. -First, let's connect to HyperDex: +First, let's open a Python shell and connect to HyperDex: .. sourcecode:: pycon From c238ec16115517cb261f5c60b2f818aec4b39df5 Mon Sep 17 00:00:00 2001 From: Robert Escriva Date: Wed, 16 Jan 2013 19:29:32 -0500 Subject: [PATCH 14/39] Documentation touchups --- doc/03.quickstart.rst | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/doc/03.quickstart.rst b/doc/03.quickstart.rst index 204e61a1..fdcaecdf 100644 --- a/doc/03.quickstart.rst +++ b/doc/03.quickstart.rst @@ -107,8 +107,10 @@ instruct HyperDex to create a suitable space for holding such objects. First, let's open a Python shell and connect to HyperDex: + .. sourcecode:: pycon + $ python >>> import hyperclient >>> c = hyperclient.Client('127.0.0.1', 1982) @@ -139,18 +141,13 @@ default, HyperDex treats every attribute as an opaque bytestring, but provides other types as well. Here, we specify that the phone number be treated as an integer. The available datatypes are discussed in :ref:`Chapter 4 `. -Note that, under the covers, HyperDex treats all objects as points in a -multidimensional hyperspace. Since the ``phonebook`` space has four attributes -per object, HyperDex can map the data to points in a four-dimensional space. -The resulting four-dimensional space is hard to visualize, but you can see why -the name HyperDex is so apt. - -Note that, under the covers, HyperDex will not necessarily create one giant -hyperdimensional space. Doing so would cause lots of problems when trying to map -objects with large numbers of attributes. Instead, we will typically want to -create *subspaces* consisting of smaller numbers of dimensions. The lower number -of dimensions enable the mapping from points in space to nodes in the cluster to -be more efficient; in particular, fewer nodes need to be contacted during search +Even though the objects we specify will have many attributes, HyperDex will not +necessarily create one giant hyperdimensional space spanning as many dimensions +as attributes. Doing so would not be desirable, as the space would be to large +to efficiently map onto a cluster. Instead, we will typically want to create +*subspaces* consisting of smaller numbers of dimensions. The lower number of +dimensions enable the mapping from points in space to nodes in the cluster to be +more efficient; in particular, fewer nodes need to be contacted during search operations. In this simple example, we create a 3-dimensional subspace for the ``first``, ``last`` and ``phone`` attributes. HyperDex always implicitly creates a 1-dimensional subspace for the key of objects. From 733a5b618bbce5e3f080c94997041243863446fd Mon Sep 17 00:00:00 2001 From: Robert Escriva Date: Fri, 18 Jan 2013 09:10:09 -0500 Subject: [PATCH 15/39] Make the java bindings build from tarball --- Makefile.am | 47 ++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 42 insertions(+), 5 deletions(-) diff --git a/Makefile.am b/Makefile.am index 02998cd6..c8e3d3f4 100644 --- a/Makefile.am +++ b/Makefile.am @@ -245,7 +245,44 @@ libhypercoordinator_la_LIBADD = -le libhyperclient_extra_dist = \ client/keyop_info.gperf \ - hyperclient/java/hyperclient.i \ + client/java/examples/HyperBinaryTest.java \ + client/java/examples/HyperSearchFloatRange.java \ + client/java/examples/HyperTest.java \ + client/java/examples/README.txt \ + client/java/examples/source_me \ + client/java/extra_src/AttributeError.java \ + client/java/extra_src/ByteArray.java \ + client/java/extra_src/ByteArrayKeyedMap.java \ + client/java/extra_src/ByteArrayKeyedSortedMap.java \ + client/java/extra_src/ByteArraySortedSet.java \ + client/java/extra_src/ByteArrayVector.java \ + client/java/extra_src/DeferredCondPut.java \ + client/java/extra_src/DeferredCount.java \ + client/java/extra_src/DeferredDelete.java \ + client/java/extra_src/DeferredFromAttrs.java \ + client/java/extra_src/DeferredGet.java \ + client/java/extra_src/DeferredGroupDel.java \ + client/java/extra_src/Deferred.java \ + client/java/extra_src/DeferredMapOp.java \ + client/java/extra_src/GreaterEqual.java \ + client/java/extra_src/HyperClientException.java \ + client/java/extra_src/LessEqual.java \ + client/java/extra_src/MapOp.java \ + client/java/extra_src/MemoryError.java \ + client/java/extra_src/Pending.java \ + client/java/extra_src/Predicate.java \ + client/java/extra_src/Range.java \ + client/java/extra_src/SearchBase.java \ + client/java/extra_src/Search.java \ + client/java/extra_src/SimpleOp.java \ + client/java/extra_src/SortedSearch.java \ + client/java/extra_src/TypeError.java \ + client/java/extra_src/ValueError.java \ + client/java/hyperclient.i \ + client/java/proxies/hyperclient_attribute_check.i \ + client/java/proxies/hyperclient_attribute.i \ + client/java/proxies/HyperClient.i \ + client/java/proxies/hyperclient_map_attribute.i \ client/python/hyperclient.pyx libhyperclient_includedir = $(includedir) @@ -331,15 +368,15 @@ libhyperclient_java_la_LIBADD = \ client/java/hyperclient_wrap.cxx: client/java/hyperclient.i client/hyperclient.h mkdir -p client/java/hyperclient - ${SWIG} -java -package hyperclient -outdir $(abs_builddir)/client/java/hyperclient \ + ${SWIG} -I${abs_srcdir} -java -package hyperclient -outdir $(abs_builddir)/client/java/hyperclient \ -o $(abs_builddir)/client/java/hyperclient_wrap.cxx -w518 $(abs_top_srcdir)/client/java/hyperclient.i hyperclient_jarfile = hyperclient-$(VERSION).jar hyperclient-java.stamp: client/java/hyperclient_wrap.cxx - cp client/java/extra_src/*.java client/java/hyperclient/. - javac client/java/hyperclient/*.java - date > hyperclient-java.stamp + cp ${abs_srcdir}/client/java/extra_src/*.java ${abs_builddir}/client/java/hyperclient/. + javac ${abs_builddir}/client/java/hyperclient/*.java + date > ${abs_builddir}/hyperclient-java.stamp $(hyperclient_jarfile): hyperclient-java.stamp cd $(abs_top_builddir)/client/java ; $(JAR) cvf $(JARFLAGS) $(abs_top_builddir)/$(hyperclient_jarfile) hyperclient/*.class From 5628b6baa648f0f55d494e679b7af6a325c992b7 Mon Sep 17 00:00:00 2001 From: Robert Escriva Date: Fri, 18 Jan 2013 09:10:37 -0500 Subject: [PATCH 16/39] Set version as 1.0.dev --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 57a8f3dc..7b8a06af 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ([2.66]) -AC_INIT([HyperDex], [0.5.dev], [escriva@cs.cornell.edu]) +AC_INIT([HyperDex], [1.0.dev], [escriva@cs.cornell.edu]) AM_INIT_AUTOMAKE([foreign subdir-objects dist-bzip2 dist-lzma]) m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) AM_PATH_PYTHON([2.6]) From a3fb1439927f466359dfdf6146e6369d851e58bb Mon Sep 17 00:00:00 2001 From: Robert Escriva Date: Fri, 18 Jan 2013 09:14:19 -0500 Subject: [PATCH 17/39] Update the YCSB benchmark --- ycsb/hyperclient/HyperClientYCSB.java | 33 ++++++++++++--------------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/ycsb/hyperclient/HyperClientYCSB.java b/ycsb/hyperclient/HyperClientYCSB.java index a5ba5ab1..d1477688 100644 --- a/ycsb/hyperclient/HyperClientYCSB.java +++ b/ycsb/hyperclient/HyperClientYCSB.java @@ -59,7 +59,7 @@ public class HyperClientYCSB extends DB public void init() throws DBException { String host = getProperties().getProperty("hyperclient.host", "127.0.0.1"); - Integer port = Integer.parseInt(getProperties().getProperty("hyperclient.port", "1234")); + Integer port = Integer.parseInt(getProperties().getProperty("hyperclient.port", "1982")); m_client = new HyperClient(host, port); m_pat = Pattern.compile("([a-zA-Z]*)([0-9]*)"); m_mat = m_pat.matcher("user1"); @@ -98,7 +98,6 @@ public int read(String table, String key, Set fields, HashMap fiel * @param values A HashMap of field/value pairs to update in the record * @return Zero on success, a non-zero error code on error. See this class's description for a discussion of error codes. */ - public int update(String table, String key, HashMap values) + public int update(String table, String key, HashMap _values) { - Map res = new HashMap(); - convert_from_java(values, res); + HashMap values = new HashMap(); + + for (Map.Entry entry : _values.entrySet()) + { + values.put(entry.getKey(), entry.getValue().toArray()); + } if (m_scannable) { @@ -183,16 +186,17 @@ public int update(String table, String key, HashMap values) } long num = Long.parseLong(m_mat.group(2)); - res.put("recno", new Long(num << 32)); + values.put("recno", new Long(num << 32)); } try { - Boolean ret = m_client.put(table, key, res); - return ret.booleanValue()?0:1; + m_client.put(table, key, values); + return 0; } catch(Exception e) { + System.out.println(e.toString()); return 1; } } @@ -222,8 +226,8 @@ public int delete(String table, String key) { try { - boolean ret = m_client.del(table, key); - return ret?0:1; + m_client.del(table, key); + return 0; } catch(Exception e) { @@ -247,13 +251,4 @@ private void convert_to_java(Set fields, Map interres, HashMap result, Map interres) - { - Map r = result; - for (Map.Entry entry : r.entrySet()) - { - interres.put(entry.getKey(), entry.getValue().toString()); - } - } } From 7494c55edc93e2b62b4eac80f9778540331ef262 Mon Sep 17 00:00:00 2001 From: Robert Escriva Date: Fri, 18 Jan 2013 11:43:42 -0500 Subject: [PATCH 18/39] Fix a deadlock between background and network threads. We paused the background threads by holding a lock across reconfiguration. Unfortunately, we did so by first blocking the threads and then the network traffic. If a packet arrived that required passing info to the background threads (by holding their lock), we could deadlock. Fix this by adding another set of condition variables between the background threads and the thread that performs reconfiguration. --- daemon/communication.cc | 20 +------ daemon/communication.h | 12 +--- daemon/daemon.cc | 30 ++++------ daemon/datalayer.cc | 85 ++++++++++++++++------------ daemon/datalayer.h | 17 +++--- daemon/replication_manager.cc | 96 +++++++++++++++++++------------- daemon/replication_manager.h | 26 ++++----- daemon/search_manager.cc | 21 +------ daemon/search_manager.h | 12 +--- daemon/state_transfer_manager.cc | 58 ++++++++++++------- daemon/state_transfer_manager.h | 17 +++--- 11 files changed, 191 insertions(+), 203 deletions(-) diff --git a/daemon/communication.cc b/daemon/communication.cc index 69d0fb77..42762c44 100644 --- a/daemon/communication.cc +++ b/daemon/communication.cc @@ -105,15 +105,7 @@ communication :: teardown() { } -reconfigure_returncode -communication :: prepare(const configuration&, - const configuration&, - const server_id&) -{ - return RECONFIGURE_SUCCESS; -} - -reconfigure_returncode +void communication :: reconfigure(const configuration&, const configuration& new_config, const server_id&) @@ -137,16 +129,6 @@ communication :: reconfigure(const configuration&, { m_early_messages.push(em); } - - return RECONFIGURE_SUCCESS; -} - -reconfigure_returncode -communication :: cleanup(const configuration&, - const configuration&, - const server_id&) -{ - return RECONFIGURE_SUCCESS; } bool diff --git a/daemon/communication.h b/daemon/communication.h index 8421f39e..e375ff75 100644 --- a/daemon/communication.h +++ b/daemon/communication.h @@ -81,15 +81,9 @@ class communication bool setup(const po6::net::location& bind_to, unsigned threads); void teardown(); - reconfigure_returncode prepare(const configuration& old_config, - const configuration& new_config, - const server_id& us); - reconfigure_returncode reconfigure(const configuration& old_config, - const configuration& new_config, - const server_id& us); - reconfigure_returncode cleanup(const configuration& old_config, - const configuration& new_config, - const server_id& us); + void reconfigure(const configuration& old_config, + const configuration& new_config, + const server_id& us); public: bool send_client(const virtual_server_id& from, diff --git a/daemon/daemon.cc b/daemon/daemon.cc index 7ad1305d..95f9c951 100644 --- a/daemon/daemon.cc +++ b/daemon/daemon.cc @@ -286,19 +286,11 @@ daemon :: run(bool daemonize, continue; } - // XXX we really should check the reconfigure_returncode even though - // nothing right now fails when reconfiguring - - LOG(INFO) << "received new configuration version=" << new_config.version(); - // prepare - m_data.prepare(old_config, new_config, m_us); - m_comm.prepare(old_config, new_config, m_us); - m_repl.prepare(old_config, new_config, m_us); - m_stm.prepare(old_config, new_config, m_us); - m_sm.prepare(old_config, new_config, m_us); - // reconfigure - LOG(INFO) << "preparations for reconfiguration done; pausing network communication"; - // this line to the "unpause" below are mutually exclusive with network workers + LOG(INFO) << "received new configuration version=" << new_config.version() + << "; pausing all activity while we reconfigure"; + m_stm.pause(); + m_repl.pause(); + m_data.pause(); m_comm.pause(); m_data.reconfigure(old_config, new_config, m_us); m_comm.reconfigure(old_config, new_config, m_us); @@ -307,13 +299,11 @@ daemon :: run(bool daemonize, m_sm.reconfigure(old_config, new_config, m_us); m_config = new_config; m_comm.unpause(); - LOG(INFO) << "reconfiguration complete; unpausing network communication"; - // cleanup - m_sm.cleanup(old_config, new_config, m_us); - m_stm.cleanup(old_config, new_config, m_us); - m_repl.cleanup(old_config, new_config, m_us); - m_comm.cleanup(old_config, new_config, m_us); - m_data.cleanup(old_config, new_config, m_us); + m_data.unpause(); + m_repl.unpause(); + m_stm.unpause(); + LOG(INFO) << "reconfiguration complete; resuming normal operation"; + // let the coordinator know we've moved to this config m_coord.ack_config(new_config.version()); } diff --git a/daemon/datalayer.cc b/daemon/datalayer.cc index 474791ad..23339c90 100644 --- a/daemon/datalayer.cc +++ b/daemon/datalayer.cc @@ -73,8 +73,11 @@ datalayer :: datalayer(daemon* d) , m_cleaner(std::tr1::bind(&datalayer::cleaner, this)) , m_block_cleaner() , m_wakeup_cleaner(&m_block_cleaner) + , m_wakeup_reconfigurer(&m_block_cleaner) , m_need_cleaning(false) , m_shutdown(true) + , m_need_pause(false) + , m_paused(false) , m_state_transfer_captures() { } @@ -383,20 +386,38 @@ datalayer :: clear_dirty() } } -reconfigure_returncode -datalayer :: prepare(const configuration&, - const configuration&, - const server_id&) +void +datalayer :: pause() { - m_block_cleaner.lock(); - return RECONFIGURE_SUCCESS; + po6::threads::mutex::hold hold(&m_block_cleaner); + assert(!m_need_pause); + m_need_pause = true; } -reconfigure_returncode +void +datalayer :: unpause() +{ + po6::threads::mutex::hold hold(&m_block_cleaner); + assert(m_need_pause); + m_wakeup_cleaner.broadcast(); + m_need_pause = false; +} + +void datalayer :: reconfigure(const configuration&, const configuration& new_config, const server_id& us) { + { + po6::threads::mutex::hold hold(&m_block_cleaner); + assert(m_need_pause); + + while (!m_paused) + { + m_wakeup_reconfigurer.wait(); + } + } + std::vector captures; new_config.captures(&captures); std::vector regions; @@ -412,18 +433,6 @@ datalayer :: reconfigure(const configuration&, std::sort(regions.begin(), regions.end()); m_counters.adopt(regions); - return RECONFIGURE_SUCCESS; -} - -reconfigure_returncode -datalayer :: cleanup(const configuration&, - const configuration&, - const server_id&) -{ - m_wakeup_cleaner.broadcast(); - m_need_cleaning = true; - m_block_cleaner.unlock(); - return RECONFIGURE_SUCCESS; } datalayer::returncode @@ -1777,11 +1786,19 @@ datalayer :: cleaner() { po6::threads::mutex::hold hold(&m_block_cleaner); - while (!m_need_cleaning && - m_state_transfer_captures.empty() && - !m_shutdown) + while ((!m_need_cleaning && + m_state_transfer_captures.empty() && + !m_shutdown) || m_need_pause) { + m_paused = true; + + if (m_need_pause) + { + m_wakeup_reconfigurer.signal(); + } + m_wakeup_cleaner.wait(); + m_paused = false; } if (m_shutdown) @@ -1845,23 +1862,17 @@ datalayer :: cleaner() m_daemon->m_stm.report_wiped(cached_cid); - // If this is not a region we need to keep, we need to iterate and - // delete + if (!m_daemon->m_config.is_captured_region(capture_id(cid))) { - po6::threads::mutex::hold hold(&m_block_cleaner); - - if (!m_daemon->m_config.is_captured_region(capture_id(cid))) - { - cached_cid = capture_id(cid); - continue; - } + cached_cid = capture_id(cid); + continue; + } - if (state_transfer_captures.find(capture_id(cid)) != state_transfer_captures.end()) - { - cached_cid = capture_id(cid); - state_transfer_captures.erase(cached_cid); - continue; - } + if (state_transfer_captures.find(capture_id(cid)) != state_transfer_captures.end()) + { + cached_cid = capture_id(cid); + state_transfer_captures.erase(cached_cid); + continue; } std::vector backing; diff --git a/daemon/datalayer.h b/daemon/datalayer.h index f8f8ad68..f7fcae2a 100644 --- a/daemon/datalayer.h +++ b/daemon/datalayer.h @@ -97,15 +97,11 @@ class datalayer const po6::net::hostname& coordinator); // clears the "dirty" bit bool clear_dirty(); - reconfigure_returncode prepare(const configuration& old_config, - const configuration& new_config, - const server_id& us); - reconfigure_returncode reconfigure(const configuration& old_config, - const configuration& new_config, - const server_id& us); - reconfigure_returncode cleanup(const configuration& old_config, - const configuration& new_config, - const server_id& us); + void pause(); + void unpause(); + void reconfigure(const configuration& old_config, + const configuration& new_config, + const server_id& us); public: returncode get(const region_id& ri, @@ -246,8 +242,11 @@ class datalayer po6::threads::thread m_cleaner; po6::threads::mutex m_block_cleaner; po6::threads::cond m_wakeup_cleaner; + po6::threads::cond m_wakeup_reconfigurer; bool m_need_cleaning; bool m_shutdown; + bool m_need_pause; + bool m_paused; std::set m_state_transfer_captures; }; diff --git a/daemon/replication_manager.cc b/daemon/replication_manager.cc index 6bd19c97..b4ab5dd6 100755 --- a/daemon/replication_manager.cc +++ b/daemon/replication_manager.cc @@ -72,13 +72,16 @@ replication_manager :: replication_manager(daemon* d) , m_counters() , m_shutdown(true) , m_retransmitter(std::tr1::bind(&replication_manager::retransmitter, this)) - , m_block_retransmitter() - , m_wakeup_retransmitter(&m_block_retransmitter) - , m_need_retransmit(false) , m_garbage_collector(std::tr1::bind(&replication_manager::garbage_collector, this)) - , m_block_garbage_collector() - , m_wakeup_garbage_collector(&m_block_garbage_collector) + , m_block_both() + , m_wakeup_retransmitter(&m_block_both) + , m_wakeup_garbage_collector(&m_block_both) + , m_wakeup_reconfigurer(&m_block_both) + , m_need_retransmit(false) , m_lower_bounds() + , m_need_pause(false) + , m_paused_retransmitter(false) + , m_paused_garbage_collector(false) { } @@ -90,8 +93,7 @@ replication_manager :: ~replication_manager() throw () bool replication_manager :: setup() { - po6::threads::mutex::hold holdr(&m_block_retransmitter); - po6::threads::mutex::hold holdg(&m_block_garbage_collector); + po6::threads::mutex::hold holdr(&m_block_both); m_retransmitter.start(); m_garbage_collector.start(); m_shutdown = false; @@ -104,20 +106,39 @@ replication_manager :: teardown() shutdown(); } -reconfigure_returncode -replication_manager :: prepare(const configuration&, - const configuration&, - const server_id&) +void +replication_manager :: pause() { - m_block_retransmitter.lock(); - return RECONFIGURE_SUCCESS; + po6::threads::mutex::hold hold(&m_block_both); + assert(!m_need_pause); + m_need_pause = true; } -reconfigure_returncode +void +replication_manager :: unpause() +{ + po6::threads::mutex::hold hold(&m_block_both); + assert(m_need_pause); + m_wakeup_retransmitter.broadcast(); + m_wakeup_garbage_collector.broadcast(); + m_need_pause = false; +} + +void replication_manager :: reconfigure(const configuration&, const configuration& new_config, const server_id&) { + { + po6::threads::mutex::hold hold(&m_block_both); + assert(m_need_pause); + + while (!m_paused_retransmitter || !m_paused_garbage_collector) + { + m_wakeup_reconfigurer.wait(); + } + } + std::map seq_ids; std::vector transfers_in; new_config.transfer_in_regions(m_daemon->m_us, &transfers_in); @@ -171,19 +192,6 @@ replication_manager :: reconfigure(const configuration&, bool found = m_counters.take_max(regions[i], non_counter_max); assert(found); } - - return RECONFIGURE_SUCCESS; -} - -reconfigure_returncode -replication_manager :: cleanup(const configuration&, - const configuration&, - const server_id&) -{ - m_wakeup_retransmitter.broadcast(); - m_need_retransmit = true; - m_block_retransmitter.unlock(); - return RECONFIGURE_SUCCESS; } void @@ -558,7 +566,7 @@ replication_manager :: chain_ack(const virtual_server_id& from, void replication_manager :: chain_gc(const region_id& reg_id, uint64_t seq_id) { - po6::threads::mutex::hold hold(&m_block_garbage_collector); + po6::threads::mutex::hold hold(&m_block_both); m_wakeup_garbage_collector.broadcast(); m_lower_bounds.push_back(std::make_pair(reg_id, seq_id)); } @@ -566,7 +574,7 @@ replication_manager :: chain_gc(const region_id& reg_id, uint64_t seq_id) void replication_manager :: trip_periodic() { - po6::threads::mutex::hold hold(&m_block_retransmitter); + po6::threads::mutex::hold hold(&m_block_both); m_wakeup_retransmitter.broadcast(); m_need_retransmit = true; } @@ -1010,11 +1018,19 @@ replication_manager :: retransmitter() while (true) { { - po6::threads::mutex::hold hold(&m_block_retransmitter); + po6::threads::mutex::hold hold(&m_block_both); - while (!m_need_retransmit && !m_shutdown) + while ((!m_need_retransmit && !m_shutdown) || m_need_pause) { + m_paused_retransmitter = true; + + if (m_need_pause) + { + m_wakeup_reconfigurer.signal(); + } + m_wakeup_retransmitter.wait(); + m_paused_retransmitter = false; } if (m_shutdown) @@ -1029,14 +1045,12 @@ replication_manager :: retransmitter() std::map seq_id_lower_bounds; { - po6::threads::mutex::hold hold(&m_block_retransmitter); m_counters.peek(&seq_id_lower_bounds); } for (keyholder_map_t::iterator it = m_keyholders.begin(); it != m_keyholders.end(); it.next()) { - po6::threads::mutex::hold hold(&m_block_retransmitter); region_id ri(it.key().region); if (region_cache.find(ri) != region_cache.end() || @@ -1102,7 +1116,6 @@ replication_manager :: retransmitter() } then = now; - po6::threads::mutex::hold hold(&m_block_retransmitter); std::vector > cluster_members; m_daemon->m_config.get_all_addresses(&cluster_members); @@ -1155,11 +1168,19 @@ replication_manager :: garbage_collector() std::list > lower_bounds; { - po6::threads::mutex::hold hold(&m_block_garbage_collector); + po6::threads::mutex::hold hold(&m_block_both); - while (m_lower_bounds.empty() && !m_shutdown) + while ((m_lower_bounds.empty() && !m_shutdown) || m_need_pause) { + m_paused_garbage_collector = true; + + if (m_need_pause) + { + m_wakeup_reconfigurer.signal(); + } + m_wakeup_garbage_collector.wait(); + m_paused_garbage_collector = false; } if (m_shutdown) @@ -1199,8 +1220,7 @@ replication_manager :: shutdown() bool is_shutdown; { - po6::threads::mutex::hold holdr(&m_block_retransmitter); - po6::threads::mutex::hold holdg(&m_block_garbage_collector); + po6::threads::mutex::hold holdr(&m_block_both); m_wakeup_retransmitter.broadcast(); m_wakeup_garbage_collector.broadcast(); is_shutdown = m_shutdown; diff --git a/daemon/replication_manager.h b/daemon/replication_manager.h index ed483899..98bc77f9 100755 --- a/daemon/replication_manager.h +++ b/daemon/replication_manager.h @@ -68,15 +68,11 @@ class replication_manager public: bool setup(); void teardown(); - reconfigure_returncode prepare(const configuration& old_config, - const configuration& new_config, - const server_id& us); - reconfigure_returncode reconfigure(const configuration& old_config, - const configuration& new_config, - const server_id& us); - reconfigure_returncode cleanup(const configuration& old_config, - const configuration& new_config, - const server_id& us); + void pause(); + void unpause(); + void reconfigure(const configuration& old_config, + const configuration& new_config, + const server_id& us); // Network workers call these methods. public: @@ -190,13 +186,17 @@ class replication_manager counter_map m_counters; bool m_shutdown; po6::threads::thread m_retransmitter; - po6::threads::mutex m_block_retransmitter; - po6::threads::cond m_wakeup_retransmitter; - bool m_need_retransmit; po6::threads::thread m_garbage_collector; - po6::threads::mutex m_block_garbage_collector; + po6::threads::mutex m_block_both; + po6::threads::cond m_wakeup_retransmitter; po6::threads::cond m_wakeup_garbage_collector; + po6::threads::cond m_wakeup_reconfigurer; + bool m_need_retransmit; std::list > m_lower_bounds; + bool m_need_pause; + bool m_paused; + bool m_paused_retransmitter; + bool m_paused_garbage_collector; }; } // namespace hyperdex diff --git a/daemon/search_manager.cc b/daemon/search_manager.cc index 06e38c0a..64cdf97d 100644 --- a/daemon/search_manager.cc +++ b/daemon/search_manager.cc @@ -163,29 +163,12 @@ search_manager :: teardown() { } -reconfigure_returncode -search_manager :: prepare(const configuration&, - const configuration&, - const server_id&) -{ - return RECONFIGURE_SUCCESS; -} - -reconfigure_returncode +void search_manager :: reconfigure(const configuration&, const configuration&, const server_id&) { - return RECONFIGURE_SUCCESS; -} - -reconfigure_returncode -search_manager :: cleanup(const configuration&, - const configuration&, - const server_id&) -{ - // XXX - return RECONFIGURE_SUCCESS; + // XXX cleanup dead or old searches } void diff --git a/daemon/search_manager.h b/daemon/search_manager.h index 81c06412..6808ea64 100644 --- a/daemon/search_manager.h +++ b/daemon/search_manager.h @@ -52,15 +52,9 @@ class search_manager public: bool setup(); void teardown(); - reconfigure_returncode prepare(const configuration& old_config, - const configuration& new_config, - const server_id& us); - reconfigure_returncode reconfigure(const configuration& old_config, - const configuration& new_config, - const server_id& us); - reconfigure_returncode cleanup(const configuration& old_config, - const configuration& new_config, - const server_id& us); + void reconfigure(const configuration& old_config, + const configuration& new_config, + const server_id& us); public: void start(const server_id& from, diff --git a/daemon/state_transfer_manager.cc b/daemon/state_transfer_manager.cc index 0fed7418..07dc7473 100644 --- a/daemon/state_transfer_manager.cc +++ b/daemon/state_transfer_manager.cc @@ -51,8 +51,11 @@ state_transfer_manager :: state_transfer_manager(daemon* d) , m_kickstarter(std::tr1::bind(&state_transfer_manager::kickstarter, this)) , m_block_kickstarter() , m_wakeup_kickstarter(&m_block_kickstarter) + , m_wakeup_reconfigurer(&m_block_kickstarter) , m_need_kickstart(false) , m_shutdown(true) + , m_need_pause(false) + , m_paused(false) { } @@ -78,13 +81,21 @@ state_transfer_manager :: teardown() shutdown(); } -reconfigure_returncode -state_transfer_manager :: prepare(const configuration&, - const configuration&, - const server_id&) +void +state_transfer_manager :: pause() { - m_block_kickstarter.lock(); - return RECONFIGURE_SUCCESS; + po6::threads::mutex::hold hold(&m_block_kickstarter); + assert(!m_need_pause); + m_need_pause = true; +} + +void +state_transfer_manager :: unpause() +{ + po6::threads::mutex::hold hold(&m_block_kickstarter); + assert(m_need_pause); + m_wakeup_kickstarter.broadcast(); + m_need_pause = false; } template @@ -139,11 +150,21 @@ setup_transfer_state(const char* desc, tmp.swap(*transfer_states); } -reconfigure_returncode +void state_transfer_manager :: reconfigure(const configuration&, const configuration& new_config, const server_id&) { + { + po6::threads::mutex::hold hold(&m_block_kickstarter); + assert(m_need_pause); + + while (!m_paused) + { + m_wakeup_reconfigurer.wait(); + } + } + leveldb_snapshot_ptr snap = m_daemon->m_data.make_raw_snapshot(); // Setup transfers in @@ -157,19 +178,6 @@ state_transfer_manager :: reconfigure(const configuration&, new_config.transfer_out_regions(m_daemon->m_us, &transfers_out); std::sort(transfers_out.begin(), transfers_out.end()); setup_transfer_state("outgoing", &m_daemon->m_data, snap, transfers_out, &m_transfers_out); - - return RECONFIGURE_SUCCESS; -} - -reconfigure_returncode -state_transfer_manager :: cleanup(const configuration&, - const configuration&, - const server_id&) -{ - m_wakeup_kickstarter.broadcast(); - m_need_kickstart = true; - m_block_kickstarter.unlock(); - return RECONFIGURE_SUCCESS; } void @@ -612,9 +620,17 @@ state_transfer_manager :: kickstarter() { po6::threads::mutex::hold hold(&m_block_kickstarter); - while (!m_need_kickstart && !m_shutdown) + while ((!m_need_kickstart && !m_shutdown) || m_need_pause) { + m_paused = true; + + if (m_need_pause) + { + m_wakeup_reconfigurer.signal(); + } + m_wakeup_kickstarter.wait(); + m_paused = false; } if (m_shutdown) diff --git a/daemon/state_transfer_manager.h b/daemon/state_transfer_manager.h index 18ffe0d7..3721e7f6 100644 --- a/daemon/state_transfer_manager.h +++ b/daemon/state_transfer_manager.h @@ -57,15 +57,11 @@ class state_transfer_manager public: bool setup(); void teardown(); - reconfigure_returncode prepare(const configuration& old_config, - const configuration& new_config, - const server_id& us); - reconfigure_returncode reconfigure(const configuration& old_config, - const configuration& new_config, - const server_id& us); - reconfigure_returncode cleanup(const configuration& old_config, - const configuration& new_config, - const server_id& us); + void pause(); + void unpause(); + void reconfigure(const configuration& old_config, + const configuration& new_config, + const server_id& us); public: void xfer_op(const virtual_server_id& from, @@ -112,8 +108,11 @@ class state_transfer_manager po6::threads::thread m_kickstarter; po6::threads::mutex m_block_kickstarter; po6::threads::cond m_wakeup_kickstarter; + po6::threads::cond m_wakeup_reconfigurer; bool m_need_kickstart; bool m_shutdown; + bool m_need_pause; + bool m_paused; }; } // namespace hyperdex From 87548f6948c087220b7688167c16a53e97a315fb Mon Sep 17 00:00:00 2001 From: Robert Escriva Date: Fri, 18 Jan 2013 14:23:24 -0500 Subject: [PATCH 19/39] Fix an off-by-one that made the client aggressively request new configurations --- client/coordinator_link.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/client/coordinator_link.cc b/client/coordinator_link.cc index 461281a2..b58dd168 100644 --- a/client/coordinator_link.cc +++ b/client/coordinator_link.cc @@ -357,8 +357,7 @@ coordinator_link :: make_rpc(const char* func, bool coordinator_link :: initiate_wait_for_config(hyperclient_returncode* status) { - uint64_t wait_for = m_config.version() > 0 ? m_config.version() - 1 : 0; - m_wait_config_id = m_repl.wait("hyperdex", "config", wait_for, &m_wait_config_status); + m_wait_config_id = m_repl.wait("hyperdex", "config", m_config.version(), &m_wait_config_status); if (m_wait_config_id < 0) { From 5071b430aaa88333237c3a6a93c7bfb749ce1174 Mon Sep 17 00:00:00 2001 From: Nick Tolomiczenko Date: Sat, 19 Jan 2013 15:19:52 -0500 Subject: [PATCH 20/39] Put rm_space() in a 'finally' block Signed-off-by: Nick Tolomiczenko --- client/java/examples/HyperTest.java | 318 ++++++++++++++++------------ 1 file changed, 179 insertions(+), 139 deletions(-) diff --git a/client/java/examples/HyperTest.java b/client/java/examples/HyperTest.java index a13b6239..2ad939be 100644 --- a/client/java/examples/HyperTest.java +++ b/client/java/examples/HyperTest.java @@ -18,148 +18,188 @@ public static void main(String[] args) throws Exception c.add_space(desc); - values.put("first","Nick"); - values.put("last","Tolomiczenko"); - values.put("phone",4165551024L); - - System.out.println("ntolomic put: " + c.put("phonebook","ntolomic",values)); - - values.put("first","George"); - values.put("last","Tolomiczenko"); - values.put("phone",4165551025L); - - System.out.println("gtolomic put: " + c.put("phonebook","gtolomic",values)); - - values.put("first","Paul"); - values.put("last","Tolomiczenko"); - values.put("phone",4165551026L); - - System.out.println("ptolomic put: " + c.put("phonebook","ptolomic",values)); - - System.out.println("\nAbout retrieve key ntolomic:\n"); - - Map row = c.get("phonebook","ntolomic"); - - System.out.println("Got back: " + row); - - System.out.println("\nSearching for last name of 'Tolomiczenko':\n"); - - // We'll use the same 'values' map for our search predicate - - // This leaves us with only the last name entry - values.remove("first"); - values.remove("phone"); - - SearchBase s = c.search("phonebook",values); - - while(s.hasNext()) - { - System.out.println(s.next()); - } - - // Now add a range stipulation on the phone number: [4165551024,4165551026] - values.put("phone", - new AbstractMap.SimpleEntry(4165551024L,4165551026L)); - - System.out.println("\nSearching for last name of 'Tolomiczenko'\n\n AND\n\nphone number in the range [4165551024,4165551026]:\n"); - - // Do the search again - s = c.search("phonebook",values); - - while(s.hasNext()) - { - System.out.println(s.next()); - } - - // Stipulate the range with a List of size 2 instead. - // In particular, use in Vector. - Vector vrange = new Vector(2); - vrange.add(4165551024L); vrange.add(4165551026L); - values.put("phone",vrange); - - System.out.println("\nDo the search again using a List of size 2 for the range stipulation:\n"); - // Do the search again - s = c.search("phonebook",values); - - while(s.hasNext()) - { - System.out.println(s.next()); - } - - System.out.println("\nLet's use the same predicate, values, to get the count:"); - System.out.println("count = " + c.count("phonebook",values)); - - // Now do an async_put, which should work by definition. - // - values.put("first","Stavroula"); - values.put("phone",4165551027L); - - Deferred d = c.async_put("phonebook","stolomic",values);; - - System.out.println("\nJust called async_put:\n"); - System.out.println("\nAbout to call waitFor():\n"); - - System.out.println(d.waitFor()); - - values.remove("first"); values.remove("phone"); - - // Do the search again - s = c.search("phonebook",values); - - while(s.hasNext()) + try { - System.out.println(s.next()); + values.put("first","Nick"); + values.put("last","Tolomiczenko"); + values.put("phone",4165551024L); + + System.out.println("ntolomic put: " + c.put("phonebook","ntolomic",values)); + + values.put("first","George"); + values.put("last","Tolomiczenko"); + values.put("phone",4165551025L); + + System.out.println("gtolomic put: " + c.put("phonebook","gtolomic",values)); + + values.put("first","Paul"); + values.put("last","Tolomiczenko"); + values.put("phone",4165551026L); + + System.out.println("ptolomic put: " + c.put("phonebook","ptolomic",values)); + + System.out.println("\nAbout retrieve key ntolomic:\n"); + + Map row = c.get("phonebook","ntolomic"); + + System.out.println("Got back: " + row); + + System.out.println("\nSearching for last name of 'Tolomiczenko':\n"); + + // We'll use the same 'values' map for our search predicate + + // This leaves us with only the last name entry + values.remove("first"); + values.remove("phone"); + + SearchBase s = c.search("phonebook",values); + + while(s.hasNext()) + { + System.out.println(s.next()); + } + + // Now add a range stipulation on the phone number: [4165551024,4165551026] + values.put("phone", + new AbstractMap.SimpleEntry(4165551024L,4165551026L)); + + System.out.println("\nSearching for last name of 'Tolomiczenko'\n\n AND\n\nphone number in the range [4165551024,4165551026]:\n"); + + // Do the search again + s = c.search("phonebook",values); + + while(s.hasNext()) + { + System.out.println(s.next()); + } + + // Stipulate the range with a List of size 2 instead. + // In particular, use a Vector. + // + Vector vrange = new Vector(2); + vrange.add(4165551024L); vrange.add(4165551026L); + values.put("phone",vrange); + + System.out.println("\nDo the search again using a List of size 2 for the range stipulation:\n"); + // Do the search again + s = c.search("phonebook",values); + + while(s.hasNext()) + { + System.out.println(s.next()); + } + + // Stipulate the range with the Range class. + // + Range range = new Range(4165551024L,4165551026L); + values.put("phone",range); + + System.out.println("\nDo the search again using Range class:\n"); + // Do the search again + s = c.search("phonebook",values); + + while(s.hasNext()) + { + System.out.println(s.next()); + } + + /* + // Stipulate the range using a List of Predicate based classes. + // In particular, use a Vector implementation of a List and LessEqual, + // GreaterEqual both derived from Predicate. + // + Vector predicates = new Vector(2); + predicates.add(new GreaterEqual(4165551024L)); + predicates.add(new LessEqual(4165551026L)); + values.put("phone",predicates); + + System.out.println("\nDo the search again using a List of Predicate based classes.:\n"); + // Do the search again + s = c.search("phonebook",values); + + while(s.hasNext()) + { + System.out.println(s.next()); + } + */ + + System.out.println("\nLet's use the same predicate, values, to get the count:"); + System.out.println("count = " + c.count("phonebook",values)); + + // Now do an async_put, which should work by definition. + // + values.put("first","Stavroula"); + values.put("phone",4165551027L); + + Deferred d = c.async_put("phonebook","stolomic",values);; + + System.out.println("\nJust called async_put:\n"); + System.out.println("\nAbout to call waitFor():\n"); + + System.out.println(d.waitFor()); + + values.remove("first"); values.remove("phone"); + + // Do the search again + s = c.search("phonebook",values); + + while(s.hasNext()) + { + System.out.println(s.next()); + } + + // Do a sorted search by first name ascending + + System.out.println("\nDo a sorted search by first name ascending:"); + + s = c.sorted_search("phonebook",values,"first",-1,false); + + while(s.hasNext()) + { + System.out.println(s.next()); + } + + // Do the same sorted search, but with a limit of 2 + + System.out.println("\nDo the same sorted search, but with a limit of 2:"); + + s = c.sorted_search("phonebook",values,"first",2,false); + + while(s.hasNext()) + { + System.out.println(s.next()); + } + + System.out.println("\nAbout delete stolomic:\n"); + + System.out.println("result: " + c.del("phonebook","stolomic")); + + // Do the search again + s = c.search("phonebook",values); + + while(s.hasNext()) + { + System.out.println(s.next()); + } + + System.out.println("\nGroup delete for last name of 'Tolomiczenko'\n\n AND\n\nphone number in the range [4165551024,4165551026]:\n"); + values.put("phone", + new AbstractMap.SimpleEntry(4165551024L,4165551026L)); + System.out.println("result: " + c.group_del("phonebook",values)); + + values.remove("phone"); + + // Do the search again + s = c.search("phonebook",values); + + while(s.hasNext()) + { + System.out.println(s.next()); + } } - - // Do a sorted search by first name ascending - - System.out.println("\nDo a sorted search by first name ascending:"); - - s = c.sorted_search("phonebook",values,"first",-1,false); - - while(s.hasNext()) + finally { - System.out.println(s.next()); + c.rm_space("phonebook"); } - - // Do the same sorted search, but with a limit of 2 - - System.out.println("\nDo the same sorted search, but with a limit of 2:"); - - s = c.sorted_search("phonebook",values,"first",2,false); - - while(s.hasNext()) - { - System.out.println(s.next()); - } - - System.out.println("\nAbout delete stolomic:\n"); - - System.out.println("result: " + c.del("phonebook","stolomic")); - - // Do the search again - s = c.search("phonebook",values); - - while(s.hasNext()) - { - System.out.println(s.next()); - } - - System.out.println("\nGroup delete for last name of 'Tolomiczenko'\n\n AND\n\nphone number in the range [4165551024,4165551026]:\n"); - values.put("phone", - new AbstractMap.SimpleEntry(4165551024L,4165551026L)); - System.out.println("result: " + c.group_del("phonebook",values)); - - values.remove("phone"); - - // Do the search again - s = c.search("phonebook",values); - - while(s.hasNext()) - { - System.out.println(s.next()); - } - - c.rm_space("phonebook"); } } From ac4b4d07a900de27fef61e010570e974d342cf55 Mon Sep 17 00:00:00 2001 From: Scott Dunlop Date: Sat, 19 Jan 2013 21:35:41 -0800 Subject: [PATCH 21/39] removed deprecated dist-lzma from autoconf Upstream autotools has removed dist-lzma as of 2.69; this prevents generating a configure script with the following error: $ autoreconf -i configure.ac:6: error: support for lzma-compressed distribution archives has been removed ... ellided install output ... autoreconf: automake failed with exit status: 1 See the automake manpages for more info. --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 7b8a06af..bd4260fd 100644 --- a/configure.ac +++ b/configure.ac @@ -3,7 +3,7 @@ AC_PREREQ([2.66]) AC_INIT([HyperDex], [1.0.dev], [escriva@cs.cornell.edu]) -AM_INIT_AUTOMAKE([foreign subdir-objects dist-bzip2 dist-lzma]) +AM_INIT_AUTOMAKE([foreign subdir-objects dist-bzip2]) m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) AM_PATH_PYTHON([2.6]) LT_PREREQ([2.2]) From c0b6e00c1d7ff96a3e9082bc0fef0eac3a48de42 Mon Sep 17 00:00:00 2001 From: Scott Dunlop Date: Sat, 19 Jan 2013 21:53:54 -0800 Subject: [PATCH 22/39] Documented dependency on gperf for git builds When building from scratch on Arch Linux, autoreconf and configure will pass without gperf installed. This should be documented as a requirement for building from git. --- doc/02.installation.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/02.installation.rst b/doc/02.installation.rst index d5abcd5e..9a6cbd42 100644 --- a/doc/02.installation.rst +++ b/doc/02.installation.rst @@ -319,6 +319,7 @@ Required Dependencies: * Bison_: Used for building internal parsers. Required for all builds. * Cython_: Used for building Python bindings. Required for ``--enable-python-bindings``. Recommmended version: >= 0.15. + * Gperf_: Generates perfect hashes. .. _Autoconf: http://www.gnu.org/software/autoconf/ .. _Automake: http://www.gnu.org/software/automake/ @@ -327,6 +328,7 @@ Required Dependencies: .. _Flex: http://flex.sourceforge.net/ .. _Bison: http://www.gnu.org/software/bison/ .. _Cython: http://cython.org/ +.. _Gperf: http://www.gnu.org/software/gperf/ After all dependencies are installed, run the ``autoreconf`` command to bootstrap the repository: From ca94be6c16b10d9e60705ec827dd80a3fd27aa47 Mon Sep 17 00:00:00 2001 From: Scott Dunlop Date: Sat, 19 Jan 2013 22:03:56 -0800 Subject: [PATCH 23/39] Added a pkg-config file for hyperclient --- .gitignore | 2 ++ Makefile.am | 8 +++++++- configure.ac | 2 +- hyperclient.pc.in | 12 ++++++++++++ 4 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 hyperclient.pc.in diff --git a/.gitignore b/.gitignore index 1259b68f..7a9ba683 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,7 @@ doc/HyperDex-0.5.dev.pdf doc/man/hyperdex-daemon.1 doc/man/hyperdex-replication-stress-test.1 doc/man/hyperdex-simple-consistency-stress-test.1 +hyperclient.pc hyperdex hyperdex-add-space hyperdex-binary-test @@ -66,3 +67,4 @@ ylwrap client/java/hyperclient *.jar *.stamp + diff --git a/Makefile.am b/Makefile.am index c8e3d3f4..c8afc8c2 100644 --- a/Makefile.am +++ b/Makefile.am @@ -71,6 +71,8 @@ noinst_PROGRAMS = \ dist_man_MANS = doc/man/hyperdex-daemon.1 +CONFIG_CLEAN_FILES = hyperclient.pc + CLEANFILES = \ $(java_bindings_cleanfiles) \ doc/HyperDex-$(VERSION).pdf \ @@ -78,6 +80,9 @@ CLEANFILES = \ JAVAROOT = $(abs_top_srcdir) +pkgconfigdir = $(libdir)/pkgconfig +pkgconfig_DATA = hyperclient.pc + .PHONY: coverage doc/HyperDex-$(VERSION).pdf dist-hook: @@ -283,7 +288,8 @@ libhyperclient_extra_dist = \ client/java/proxies/hyperclient_attribute.i \ client/java/proxies/HyperClient.i \ client/java/proxies/hyperclient_map_attribute.i \ - client/python/hyperclient.pyx + client/python/hyperclient.pyx \ + hyperclient.pc libhyperclient_includedir = $(includedir) libhyperclient_include_HEADERS = \ diff --git a/configure.ac b/configure.ac index bd4260fd..dc85cf94 100644 --- a/configure.ac +++ b/configure.ac @@ -149,5 +149,5 @@ AM_CONDITIONAL([ENABLE_JAVA_BINDINGS], [test x"${java_bindings}" = xyes]) AM_CONDITIONAL([ENABLE_PYTHON_BINDINGS], [test x"${python_bindings}" = xyes]) AM_CONDITIONAL([ENABLE_YCSB_DRIVER], [test x"${enable_ycsb}" = xyes]) -AC_CONFIG_FILES([Makefile]) +AC_CONFIG_FILES([Makefile hyperclient.pc]) AC_OUTPUT diff --git a/hyperclient.pc.in b/hyperclient.pc.in new file mode 100644 index 00000000..f7a38356 --- /dev/null +++ b/hyperclient.pc.in @@ -0,0 +1,12 @@ +prefix=@prefix@ +exec_prefix=@exec_prefix@ +libdir=@libdir@ +includedir=@includedir@ + +Name: hyperclient +Description: Client library for the HyperDex scalable high performance key-value store +Version: @VERSION@ + +Requires: libpo6 >= 0.2.3 libe >= 0.2.7 +Libs: -L${libdir} -lhyperclient +Cflags: -I${includedir} From 43c080cf46307bb496da9591ffa6604b18ba1379 Mon Sep 17 00:00:00 2001 From: Scott Dunlop Date: Sat, 19 Jan 2013 22:15:09 -0800 Subject: [PATCH 24/39] removed spurious requirements from hyperclient.pc --- hyperclient.pc.in | 1 - 1 file changed, 1 deletion(-) diff --git a/hyperclient.pc.in b/hyperclient.pc.in index f7a38356..d4a0a4a7 100644 --- a/hyperclient.pc.in +++ b/hyperclient.pc.in @@ -7,6 +7,5 @@ Name: hyperclient Description: Client library for the HyperDex scalable high performance key-value store Version: @VERSION@ -Requires: libpo6 >= 0.2.3 libe >= 0.2.7 Libs: -L${libdir} -lhyperclient Cflags: -I${includedir} From daf052dd73c42e0f382d8c08dac6d398eb58507d Mon Sep 17 00:00:00 2001 From: Robert Escriva Date: Sun, 20 Jan 2013 14:41:45 -0500 Subject: [PATCH 25/39] make error messages for "hyperdex" more useful --- hyperdex.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hyperdex.cc b/hyperdex.cc index 2f4ddf80..500637b2 100644 --- a/hyperdex.cc +++ b/hyperdex.cc @@ -139,7 +139,7 @@ main(int argc, const char* argv[]) if (execv(p->path, const_cast(args)) < 0) { - std::cerr << "failed to exec " << p->name << ": " << strerror(errno) << std::endl; + std::cerr << "failed to exec " << p->name << " from " << p->path << ": " << strerror(errno) << std::endl; return EXIT_FAILURE; } @@ -148,6 +148,6 @@ main(int argc, const char* argv[]) } } - std::cerr << "unknown command " << args[0] << "\n" << std::endl; + std::cerr << "hyperdex: '" << args[0] << "' is not a HyperDex command. See 'hyperdex --help'\n" << std::endl; return help(poptcon); } From dab578ecedecc216b46a8bdca6fb4b15829d003b Mon Sep 17 00:00:00 2001 From: Scott Dunlop Date: Mon, 21 Jan 2013 13:21:42 -0800 Subject: [PATCH 26/39] This commit and all previous that I have authored are signed off by me Signed-off-by: Scott Dunlop From c076d8281ba3288dd306d25ad1a193ceb5a7bb5a Mon Sep 17 00:00:00 2001 From: Malthe Borch Date: Wed, 23 Jan 2013 11:00:27 +0100 Subject: [PATCH 27/39] This needs to be in quotes, otherwise autoconf seems to interpret it as a tuple value and we get a type comparison error. --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 7b8a06af..c8e978d1 100644 --- a/configure.ac +++ b/configure.ac @@ -135,7 +135,7 @@ AC_ARG_ENABLE([python_bindings], [AS_HELP_STRING([--enable-python-bindings], [build Python bindings @<:@default: no@:>@])], [python_bindings=${enableval}], [python_bindings=no]) if test x"${python_bindings}" = xyes; then - AC_PYTHON_DEVEL([>= 2.6]) + AC_PYTHON_DEVEL([>= '2.6']) fi AC_ARG_ENABLE([log-all-messages], [AS_HELP_STRING([--enable-log-all-messages], From fbfc2ba836ce9b736d777e7a51a226785b177831 Mon Sep 17 00:00:00 2001 From: Robert Escriva Date: Wed, 23 Jan 2013 11:38:55 -0500 Subject: [PATCH 28/39] Increase write buffer size --- daemon/datalayer.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/daemon/datalayer.cc b/daemon/datalayer.cc index 23339c90..700cc489 100644 --- a/daemon/datalayer.cc +++ b/daemon/datalayer.cc @@ -95,6 +95,7 @@ datalayer :: setup(const po6::pathname& path, po6::net::hostname* saved_coordinator) { leveldb::Options opts; + opts.write_buffer_size = 64ULL * 1024ULL * 1024ULL; opts.create_if_missing = true; opts.filter_policy = leveldb::NewBloomFilterPolicy(10); std::string name(path.get()); From f701e95950d07bfe710259541fcd9d1d6b143a4c Mon Sep 17 00:00:00 2001 From: Robert Escriva Date: Wed, 23 Jan 2013 11:39:02 -0500 Subject: [PATCH 29/39] Reorder ACK with I/O where we can get away with it Nodes inside the chain don't need to ack before putting an object to disk --- daemon/replication_manager.cc | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/daemon/replication_manager.cc b/daemon/replication_manager.cc index b4ab5dd6..bbb0f624 100755 --- a/daemon/replication_manager.cc +++ b/daemon/replication_manager.cc @@ -507,6 +507,12 @@ replication_manager :: chain_ack(const virtual_server_id& from, } pend->acked = true; + bool is_head = m_daemon->m_config.head_of_region(ri) == to; + + if (!is_head && m_daemon->m_config.version() == pend->recv_config_version) + { + send_ack(to, pend->recv, false, reg_id, seq_id, version, key); + } if (kh->version_on_disk() < version) { @@ -555,7 +561,8 @@ replication_manager :: chain_ack(const virtual_server_id& from, { respond_to_client(to, pend->client, pend->nonce, NET_SUCCESS); } - else if (m_daemon->m_config.version() == pend->recv_config_version) + + if (is_head && m_daemon->m_config.version() == pend->recv_config_version) { send_ack(to, pend->recv, false, reg_id, seq_id, version, key); } From e5b736961b07d1eb670f209cdb8d0f46eec9659a Mon Sep 17 00:00:00 2001 From: Robert Escriva Date: Wed, 23 Jan 2013 11:39:46 -0500 Subject: [PATCH 30/39] Fix perms --- daemon/replication_manager.cc | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 daemon/replication_manager.cc diff --git a/daemon/replication_manager.cc b/daemon/replication_manager.cc old mode 100755 new mode 100644 From 5b79822e5c83097869d024ea2fa535086a33824f Mon Sep 17 00:00:00 2001 From: Malthe Borch Date: Wed, 23 Jan 2013 18:46:52 +0100 Subject: [PATCH 31/39] This commit and all previous that I have authored are signed off by me Signed-off-by: Malthe Borch From 1b3103281ef44947322b51278e502e649bd83161 Mon Sep 17 00:00:00 2001 From: Robert Escriva Date: Wed, 23 Jan 2013 13:12:16 -0500 Subject: [PATCH 32/39] Fix commit 7494c55edc93e2b62b4eac80f9778540331ef262 Threads would not wakeup after reconfigure. --- daemon/datalayer.cc | 1 + daemon/replication_manager.cc | 1 + daemon/state_transfer_manager.cc | 1 + 3 files changed, 3 insertions(+) diff --git a/daemon/datalayer.cc b/daemon/datalayer.cc index 700cc489..1b0fc996 100644 --- a/daemon/datalayer.cc +++ b/daemon/datalayer.cc @@ -402,6 +402,7 @@ datalayer :: unpause() assert(m_need_pause); m_wakeup_cleaner.broadcast(); m_need_pause = false; + m_need_cleaning = true; } void diff --git a/daemon/replication_manager.cc b/daemon/replication_manager.cc index bbb0f624..e4dc2666 100644 --- a/daemon/replication_manager.cc +++ b/daemon/replication_manager.cc @@ -122,6 +122,7 @@ replication_manager :: unpause() m_wakeup_retransmitter.broadcast(); m_wakeup_garbage_collector.broadcast(); m_need_pause = false; + m_need_retransmit = true; } void diff --git a/daemon/state_transfer_manager.cc b/daemon/state_transfer_manager.cc index 07dc7473..4a5aa59b 100644 --- a/daemon/state_transfer_manager.cc +++ b/daemon/state_transfer_manager.cc @@ -96,6 +96,7 @@ state_transfer_manager :: unpause() assert(m_need_pause); m_wakeup_kickstarter.broadcast(); m_need_pause = false; + m_need_kickstart = true; } template From 186ad8c32da40cdde7be0f973a945edc2e49613c Mon Sep 17 00:00:00 2001 From: Robert Escriva Date: Thu, 24 Jan 2013 09:25:36 -0500 Subject: [PATCH 33/39] Finish implementing workload E --- ycsb/hyperclient/HyperClientYCSB.java | 37 ++++++++++++--------------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/ycsb/hyperclient/HyperClientYCSB.java b/ycsb/hyperclient/HyperClientYCSB.java index d1477688..c589bb13 100644 --- a/ycsb/hyperclient/HyperClientYCSB.java +++ b/ycsb/hyperclient/HyperClientYCSB.java @@ -113,49 +113,46 @@ public int read(String table, String key, Set fields, HashMap fields, Vector> result) { - /* // XXX I'm going to be lazy and not support "fields" for now. Patches // welcome. + if (!m_scannable) { - return 1000; + return 1; } m_mat.reset(startkey); if (!m_mat.matches()) { - return 1001; + return 2; } long base = Long.parseLong(m_mat.group(2)); long lower = base << 32; long upper = (base + recordcount) << 32; - HyperVector res = new HyperVector(); - - ReturnCode ret = m_client.range_search(table, "recno", lower, upper, res); + HashMap values = new HashMap(); + Vector vrange = new Vector(2); + vrange.add(lower); + vrange.add(upper); + values.put("recno", vrange); - for (int i = 0; i < m_retries && ret != ReturnCode.SUCCESS; ++i) + try { - ret = m_client.range_search(table, "recno", lower, upper, res); - } + SearchBase s = m_client.search(table, values); - if (ret == ReturnCode.SUCCESS) - { - for (int i = 0; i < res.size(); ++i) + while (s.hasNext()) { - HyperMap e = HyperMap.dynamic_cast(res.get(i)); - HashMap e2 = new HashMap(); - convert_to_java(fields, e, e2); - result.add(e2); + s.next(); } + return 0; } - - return ret.swigValue(); - */ - return 0; + catch(Exception e) + { + return 3; + } } /** From f1deb531600b28304a3ff187042b7273ebdbb3e8 Mon Sep 17 00:00:00 2001 From: Robert Escriva Date: Sun, 27 Jan 2013 15:16:36 -0500 Subject: [PATCH 34/39] Remove unused variable from replication_manager --- daemon/replication_manager.h | 1 - 1 file changed, 1 deletion(-) diff --git a/daemon/replication_manager.h b/daemon/replication_manager.h index 98bc77f9..859d7f0e 100755 --- a/daemon/replication_manager.h +++ b/daemon/replication_manager.h @@ -194,7 +194,6 @@ class replication_manager bool m_need_retransmit; std::list > m_lower_bounds; bool m_need_pause; - bool m_paused; bool m_paused_retransmitter; bool m_paused_garbage_collector; }; From bd684004fc1e0dd04051461e4051248fb9e97eee Mon Sep 17 00:00:00 2001 From: Nick Tolomiczenko Date: Sun, 27 Jan 2013 23:38:06 -0500 Subject: [PATCH 35/39] Allowing predicate values to be a list as well Signed-off-by: Nick Tolomiczenko --- client/java/examples/HyperBinaryTest.java | 141 +++++++++--------- .../java/examples/HyperSearchFloatRange.java | 140 +++++++++-------- client/java/examples/HyperTest.java | 20 +-- client/java/examples/source_me | 2 +- client/java/extra_src/Predicate.java | 2 +- client/java/proxies/HyperClient.i | 120 ++++++++------- 6 files changed, 212 insertions(+), 213 deletions(-) diff --git a/client/java/examples/HyperBinaryTest.java b/client/java/examples/HyperBinaryTest.java index e12d26b9..acae149e 100644 --- a/client/java/examples/HyperBinaryTest.java +++ b/client/java/examples/HyperBinaryTest.java @@ -31,73 +31,78 @@ public static void main(String[] args) throws Exception c.add_space(desc); - HashMap attrs = new HashMap(); - - // Make up some arbitrary bytes - byte[] inBytes = new byte[256]; - - for (int i=0; i attrs = new HashMap(); + + // Make up some arbitrary bytes + byte[] inBytes = new byte[256]; + + for (int i=0; i(160.1D,165D)); - - System.out.println("\nSearching for last name of 'Tolomiczenko'\n\n AND\n\nweight in the range [160.1,165]:\n"); - - // Do the search again - s = c.search("weightbook",values); - - while(s.hasNext()) + try { - System.out.println(s.next()); + values.put("first","Nick"); + values.put("last","Tolomiczenko"); + values.put("weight",175.2); + + System.out.println("ntolomic put: " + c.put("weightbook","ntolomic",values)); + + values.put("first","George"); + values.put("last","Tolomiczenko"); + values.put("weight",160.7); + + System.out.println("gtolomic put: " + c.put("weightbook","gtolomic",values)); + + values.put("first","Paul"); + values.put("last","Tolomiczenko"); + values.put("weight",162.3); + + System.out.println("ptolomic put: " + c.put("weightbook","ptolomic",values)); + + System.out.println("\nAbout retrieve key ntolomic:\n"); + + Map row = c.get("weightbook","ntolomic"); + + System.out.println("Got back: " + row); + + System.out.println("\nSearching for last name of 'Tolomiczenko':\n"); + + // We'll use the same 'values' map for our search predicate + + // This leaves us with only the last name entry + values.remove("first"); + values.remove("weight"); + + SearchBase s = c.search("weightbook",values); + + while(s.hasNext()) + { + System.out.println(s.next()); + } + + // Now add a range stipulation on the weight: [160.1,165] + values.put("weight", + new AbstractMap.SimpleEntry(160.1D,165D)); + + System.out.println("\nSearching for last name of 'Tolomiczenko'\n\n AND\n\nweight in the range [160.1,165]:\n"); + + // Do the search again + s = c.search("weightbook",values); + + while(s.hasNext()) + { + System.out.println(s.next()); + } + + // Stipulate the range using a List of Predicate based classes. + // In particular, use a Vector implementation of a List and LessEqual, + // GreaterEqual both derived from Predicate. + // + Vector predicates = new Vector(2); + predicates.add(new GreaterEqual(160.1D)); + predicates.add(new LessEqual(165D)); + values.put("weight",predicates); + + System.out.println("\nDo the search again using a List of Predicate based classes.:\n"); + // Do the search again + s = c.search("weightbook",values); + + while(s.hasNext()) + { + System.out.println(s.next()); + } } - - // Stipulate the range with a List of size 2 instead. - // In particular, use in Vector. - Vector vrange = new Vector(2); - vrange.add(160.1D); vrange.add(165D); - values.put("weight",vrange); - - System.out.println("\nDo the search again using a List of size 2 for the range stipulation:\n"); - // Do the search again - s = c.search("weightbook",values); - - while(s.hasNext()) - { - System.out.println(s.next()); + finally + { + c.rm_space("weightbook"); } - - c.rm_space("weightbook"); } } diff --git a/client/java/examples/HyperTest.java b/client/java/examples/HyperTest.java index 2ad939be..ce0c6969 100644 --- a/client/java/examples/HyperTest.java +++ b/client/java/examples/HyperTest.java @@ -73,22 +73,6 @@ public static void main(String[] args) throws Exception System.out.println(s.next()); } - // Stipulate the range with a List of size 2 instead. - // In particular, use a Vector. - // - Vector vrange = new Vector(2); - vrange.add(4165551024L); vrange.add(4165551026L); - values.put("phone",vrange); - - System.out.println("\nDo the search again using a List of size 2 for the range stipulation:\n"); - // Do the search again - s = c.search("phonebook",values); - - while(s.hasNext()) - { - System.out.println(s.next()); - } - // Stipulate the range with the Range class. // Range range = new Range(4165551024L,4165551026L); @@ -103,12 +87,11 @@ public static void main(String[] args) throws Exception System.out.println(s.next()); } - /* // Stipulate the range using a List of Predicate based classes. // In particular, use a Vector implementation of a List and LessEqual, // GreaterEqual both derived from Predicate. // - Vector predicates = new Vector(2); + Vector predicates = new Vector(2); predicates.add(new GreaterEqual(4165551024L)); predicates.add(new LessEqual(4165551026L)); values.put("phone",predicates); @@ -121,7 +104,6 @@ public static void main(String[] args) throws Exception { System.out.println(s.next()); } - */ System.out.println("\nLet's use the same predicate, values, to get the count:"); System.out.println("count = " + c.count("phonebook",values)); diff --git a/client/java/examples/source_me b/client/java/examples/source_me index 0956df63..b553f0b9 100644 --- a/client/java/examples/source_me +++ b/client/java/examples/source_me @@ -1,2 +1,2 @@ -export CLASSPATH=.:../../../hyperclient-0.5.dev.jar +export CLASSPATH=.:../../../hyperclient-1.0.dev.jar export LD_LIBRARY_PATH=../../../.libs diff --git a/client/java/extra_src/Predicate.java b/client/java/extra_src/Predicate.java index 494cdd6d..3c5d56db 100644 --- a/client/java/extra_src/Predicate.java +++ b/client/java/extra_src/Predicate.java @@ -2,7 +2,7 @@ import java.util.*; -class Predicate +public abstract class Predicate { protected List> raw; diff --git a/client/java/proxies/HyperClient.i b/client/java/proxies/HyperClient.i index 9497e952..c5a9f3d3 100644 --- a/client/java/proxies/HyperClient.i +++ b/client/java/proxies/HyperClient.i @@ -493,6 +493,60 @@ } } + // Convenience method used when pred is not an instance of List + // + private java.util.Vector< + java.util.Map.Entry< + ByteArray,java.util.Map.Entry>> + getRawChecks(ByteArray attr, String errStr, Object pred) throws AttributeError, + TypeError + { + java.util.Vector< + java.util.Map.Entry< + ByteArray,java.util.Map.Entry>> rawChecks + = new java.util.Vector< + java.util.Map.Entry< + ByteArray,java.util.Map.Entry>>(); + + if ( isBytes(pred) || pred instanceof Long || pred instanceof Double ) + { + rawChecks.add(new java.util.AbstractMap.SimpleEntry< + ByteArray,java.util.Map.Entry>( + attr, + new java.util.AbstractMap.SimpleEntry< + hyperpredicate,Object>( + hyperpredicate.HYPERPREDICATE_EQUALS, pred))); + } + else if ( pred instanceof java.util.Map.Entry ) + { + Object lower = ((java.util.Map.Entry)pred).getKey(); + Object upper = ((java.util.Map.Entry)pred).getValue(); + + if ( ! ( lower instanceof Long && upper instanceof Long ) && + ! ( lower instanceof Double && upper instanceof Double ) ) + { + throw + new TypeError( + String.format(errStr,pred.getClass().getName())); + } + + rawChecks.addAll( + (new Range(lower,upper)).getRawChecks(attr)); + } + else if ( pred instanceof Predicate ) + { + rawChecks.addAll( + ((Predicate)pred).getRawChecks(attr)); + } + else + { + throw + new TypeError(String.format(errStr,pred.getClass().getName())); + } + + return rawChecks; + } + // Using a Vector retvals to return multiple values. // // retvals at 0 - hacs (hyperclient_attribute_check type) @@ -531,70 +585,20 @@ throw new TypeError("Cannot search with a null criteria"); - String errStr = "Attribute '" + attrStr + "' has incorrect type ( expected Predicate, Long, Double, String, Map.Entry, Map.Entry, List or List (List being of size 2), but got %s"; + String errStr = "Attribute '" + attrStr + "' has incorrect type ( expected Predicate, Long, Double, String, Map.Entry, Map.Entry or List, but got %s"; - if ( isBytes(params) || params instanceof Long || params instanceof Double ) + if ( ! (params instanceof java.util.List) ) { - rawChecks.add(new java.util.AbstractMap.SimpleEntry< - ByteArray,java.util.Map.Entry>( - new ByteArray(attrBytes), - new java.util.AbstractMap.SimpleEntry< - hyperpredicate,Object>( - hyperpredicate.HYPERPREDICATE_EQUALS, params))); + rawChecks.addAll(getRawChecks(new ByteArray(attrBytes),errStr,params)); } - else if ( params instanceof java.util.Map.Entry ) - { - Object lower = ((java.util.Map.Entry)params).getKey(); - Object upper = ((java.util.Map.Entry)params).getValue(); - - if ( ! ( lower instanceof Long && upper instanceof Long ) && - ! ( lower instanceof Double && upper instanceof Double ) ) - { - throw - new TypeError( - String.format(errStr,params.getClass().getName())); - } - - rawChecks.addAll( - (new Range(lower,upper)).getRawChecks(new ByteArray(attrBytes))); - } - else if ( params instanceof java.util.List ) + else { - try - { - java.util.List listParams = (java.util.List)params; - - if ( listParams.size() != 2 ) - throw new TypeError("Attribute '" + attrStr + "': using a List to specify a range requires its size to be 2, but got size " + listParams.size()); - } - catch (TypeError te) - { - throw te; - } - - Object lower = ((java.util.List)params).get(0); - Object upper = ((java.util.List)params).get(1); - - if ( ! ( lower instanceof Long && upper instanceof Long ) && - ! ( lower instanceof Double && upper instanceof Double ) ) + for (java.util.Iterator paramsIt = ((java.util.List)params).iterator(); + paramsIt.hasNext();) { - throw - new TypeError( - String.format(errStr,params.getClass().getName())); + rawChecks.addAll( + getRawChecks(new ByteArray(attrBytes),errStr,paramsIt.next())); } - - rawChecks.addAll( - (new Range(lower,upper)).getRawChecks(new ByteArray(attrBytes))); - } - else if ( params instanceof Predicate ) - { - rawChecks.addAll( - ((Predicate)params).getRawChecks(new ByteArray(attrBytes))); - } - else - { - throw - new TypeError(String.format(errStr,params.getClass().getName())); } } From 831c30ac74f5fb7c706efc823ef7304e4e27cb19 Mon Sep 17 00:00:00 2001 From: Nick Tolomiczenko Date: Mon, 28 Jan 2013 21:20:00 -0500 Subject: [PATCH 36/39] Changed how a search range is constructed for the ycsb test Specifying a range predicate using a Vector -- or any List derived instance -- of size 2 has been deprecated to allow for predicate List's. Signed-off-by: Nick Tolomiczenko --- client/java/examples/HyperSearchFloatRange.java | 2 +- client/java/examples/HyperTest.java | 2 +- ycsb/hyperclient/HyperClientYCSB.java | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/client/java/examples/HyperSearchFloatRange.java b/client/java/examples/HyperSearchFloatRange.java index 2f7ae930..7fb030aa 100644 --- a/client/java/examples/HyperSearchFloatRange.java +++ b/client/java/examples/HyperSearchFloatRange.java @@ -82,7 +82,7 @@ public static void main(String[] args) throws Exception predicates.add(new LessEqual(165D)); values.put("weight",predicates); - System.out.println("\nDo the search again using a List of Predicate based classes.:\n"); + System.out.println("\nDo the search again using a List of Predicate based classes:\n"); // Do the search again s = c.search("weightbook",values); diff --git a/client/java/examples/HyperTest.java b/client/java/examples/HyperTest.java index ce0c6969..179789ad 100644 --- a/client/java/examples/HyperTest.java +++ b/client/java/examples/HyperTest.java @@ -96,7 +96,7 @@ public static void main(String[] args) throws Exception predicates.add(new LessEqual(4165551026L)); values.put("phone",predicates); - System.out.println("\nDo the search again using a List of Predicate based classes.:\n"); + System.out.println("\nDo the search again using a List of Predicate based classes:\n"); // Do the search again s = c.search("phonebook",values); diff --git a/ycsb/hyperclient/HyperClientYCSB.java b/ycsb/hyperclient/HyperClientYCSB.java index c589bb13..35092dee 100644 --- a/ycsb/hyperclient/HyperClientYCSB.java +++ b/ycsb/hyperclient/HyperClientYCSB.java @@ -35,6 +35,7 @@ import java.util.Map; import java.util.Set; import java.util.Vector; +import java.util.AbstractMap; import java.util.regex.*; import com.yahoo.ycsb.DB; @@ -133,10 +134,9 @@ public int scan(String table, String startkey, int recordcount, Set fiel long upper = (base + recordcount) << 32; HashMap values = new HashMap(); - Vector vrange = new Vector(2); - vrange.add(lower); - vrange.add(upper); - values.put("recno", vrange); + AbstractMap.SimpleEntry range + = new AbstractMap.SimpleEntry(lower,upper); + values.put("recno", range); try { From 4b80a4d84a4e31e84dd1b64e8f95733de6108118 Mon Sep 17 00:00:00 2001 From: Nick Tolomiczenko Date: Tue, 29 Jan 2013 11:39:32 -0500 Subject: [PATCH 37/39] Modified the error message for an invalid predicate Signed-off-by: Nick Tolomiczenko --- client/java/proxies/HyperClient.i | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/client/java/proxies/HyperClient.i b/client/java/proxies/HyperClient.i index c5a9f3d3..9e9c04c6 100644 --- a/client/java/proxies/HyperClient.i +++ b/client/java/proxies/HyperClient.i @@ -498,9 +498,12 @@ private java.util.Vector< java.util.Map.Entry< ByteArray,java.util.Map.Entry>> - getRawChecks(ByteArray attr, String errStr, Object pred) throws AttributeError, + getRawChecks(ByteArray attr, Object pred) throws AttributeError, TypeError { + + String errStr = "Attribute '" + attr + "' has an invalid predicate type ( expected Predicate, Long, Double, String, Map.Entry, Map.Entry, Map.Entry or a List of any of the former )"; + java.util.Vector< java.util.Map.Entry< ByteArray,java.util.Map.Entry>> rawChecks @@ -526,8 +529,7 @@ ! ( lower instanceof Double && upper instanceof Double ) ) { throw - new TypeError( - String.format(errStr,pred.getClass().getName())); + new TypeError(errStr); } rawChecks.addAll( @@ -541,7 +543,7 @@ else { throw - new TypeError(String.format(errStr,pred.getClass().getName())); + new TypeError(errStr); } return rawChecks; @@ -577,19 +579,14 @@ byte[] attrBytes = getBytes(attrObject); - String attrStr = ByteArray.decode(attrBytes,defaultStringEncoding); - Object params = predicate.get(attrObject); if ( params == null ) throw new TypeError("Cannot search with a null criteria"); - - String errStr = "Attribute '" + attrStr + "' has incorrect type ( expected Predicate, Long, Double, String, Map.Entry, Map.Entry or List, but got %s"; - if ( ! (params instanceof java.util.List) ) { - rawChecks.addAll(getRawChecks(new ByteArray(attrBytes),errStr,params)); + rawChecks.addAll(getRawChecks(new ByteArray(attrBytes),params)); } else { @@ -597,7 +594,7 @@ paramsIt.hasNext();) { rawChecks.addAll( - getRawChecks(new ByteArray(attrBytes),errStr,paramsIt.next())); + getRawChecks(new ByteArray(attrBytes),paramsIt.next())); } } } From 835da1292ad2bb9128266e3e43fb2c002d8cb497 Mon Sep 17 00:00:00 2001 From: Robert Escriva Date: Fri, 1 Feb 2013 11:07:21 -0500 Subject: [PATCH 38/39] Fix a crash in the index encoding Numbers can be left at 0 bytes (a zero) rather than having 8 \x00 bytes. The index encode didn't consider this. --- daemon/datalayer.cc | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/daemon/datalayer.cc b/daemon/datalayer.cc index 1b0fc996..33f53734 100644 --- a/daemon/datalayer.cc +++ b/daemon/datalayer.cc @@ -1370,6 +1370,8 @@ datalayer :: encode_index(const region_id& ri, + sizeof(uint64_t) + sizeof(uint16_t); char* ptr = NULL; + char buf_i[sizeof(int64_t)]; + char buf_d[sizeof(double)]; int64_t tmp_i; double tmp_d; @@ -1389,7 +1391,9 @@ datalayer :: encode_index(const region_id& ri, ptr = e::pack8be('i', ptr); ptr = e::pack64be(ri.get(), ptr); ptr = e::pack16be(attr, ptr); - e::unpack64le(value.data(), &tmp_i); + memset(buf_i, 0, sizeof(int64_t)); + memmove(buf_i, value.data(), std::min(value.size(), sizeof(int64_t))); + e::unpack64le(buf_i, &tmp_i); ptr = index_encode_int64(tmp_i, ptr); break; case HYPERDATATYPE_FLOAT: @@ -1398,7 +1402,9 @@ datalayer :: encode_index(const region_id& ri, ptr = e::pack8be('i', ptr); ptr = e::pack64be(ri.get(), ptr); ptr = e::pack16be(attr, ptr); - e::unpackdoublele(value.data(), &tmp_d); + memset(buf_d, 0, sizeof(double)); + memmove(buf_d, value.data(), std::min(value.size(), sizeof(double))); + e::unpackdoublele(buf_d, &tmp_d); ptr = index_encode_double(tmp_d, ptr); break; case HYPERDATATYPE_GENERIC: @@ -1441,6 +1447,8 @@ datalayer :: encode_index(const region_id& ri, + sizeof(uint64_t) + sizeof(uint16_t); char* ptr = NULL; + char buf_i[sizeof(int64_t)]; + char buf_d[sizeof(double)]; int64_t tmp_i; double tmp_d; @@ -1464,7 +1472,9 @@ datalayer :: encode_index(const region_id& ri, ptr = e::pack8be('i', ptr); ptr = e::pack64be(ri.get(), ptr); ptr = e::pack16be(attr, ptr); - e::unpack64le(value.data(), &tmp_i); + memset(buf_i, 0, sizeof(int64_t)); + memmove(buf_i, value.data(), std::min(value.size(), sizeof(int64_t))); + e::unpack64le(buf_i, &tmp_i); ptr = index_encode_int64(tmp_i, ptr); memmove(ptr, key.data(), key.size()); break; @@ -1474,7 +1484,9 @@ datalayer :: encode_index(const region_id& ri, ptr = e::pack8be('i', ptr); ptr = e::pack64be(ri.get(), ptr); ptr = e::pack16be(attr, ptr); - e::unpackdoublele(value.data(), &tmp_d); + memset(buf_d, 0, sizeof(double)); + memmove(buf_d, value.data(), std::min(value.size(), sizeof(double))); + e::unpackdoublele(buf_d, &tmp_d); ptr = index_encode_double(tmp_d, ptr); memmove(ptr, key.data(), key.size()); break; From 601b9531242ca28acf8352f2daa77ced11d29cac Mon Sep 17 00:00:00 2001 From: Robert Escriva Date: Sun, 3 Feb 2013 22:19:56 -0500 Subject: [PATCH 39/39] Add documentation about transactions. --- doc/07.transactions.rst | 294 ++++++++++++++++++++++++++++++++++++++++ doc/index.rst | 1 + 2 files changed, 295 insertions(+) create mode 100644 doc/07.transactions.rst diff --git a/doc/07.transactions.rst b/doc/07.transactions.rst new file mode 100644 index 00000000..0de6b180 --- /dev/null +++ b/doc/07.transactions.rst @@ -0,0 +1,294 @@ +.. _transactions: + +Transactions +============ + +HyperDex Warp is a superset of HyperDex that provides transactions. For this +chapter, you'll need to install the hyperdex-warp demo package available at +http://hyperdex.org/ + +By the end of this chapter you'll be familiar with HyperDex Warp's transactions. + +Setup +----- + +As in the previous chapter, the first step is to deploy the cluster and connect +a client. First we launch and initialize the coordinator: + +.. sourcecode:: console + + $ hyperdex coordinator -f -l 127.0.0.1 -p 1982 + +Next, let's launch a few daemon processes to store data. Execute the following +commands (note that each instance binds to a different port and has a different ``/path/to/data``): + +.. sourcecode:: console + + $ hyperdex daemon -f --listen=127.0.0.1 --listen-port=2012 \ + --coordinator=127.0.0.1 --coordinator-port=1982 --data=/path/to/data1 + $ hyperdex daemon -f --listen=127.0.0.1 --listen-port=2013 \ + --coordinator=127.0.0.1 --coordinator-port=1982 --data=/path/to/data2 + $ hyperdex daemon -f --listen=127.0.0.1 --listen-port=2014 \ + --coordinator=127.0.0.1 --coordinator-port=1982 --data=/path/to/data3 + + +This brings up three different daemons ready to serve in the HyperDex cluster. +Finally, we create a space which makes use of all three systems in the cluster. +In this example, let's create a space that may be suitable for storing virtual +currency in a banking application: + +.. sourcecode:: console + + >>> import hyperclient + >>> c = hyperclient.Client('127.0.0.1', 1982) + >>> c.add_space(''' + ... space accounts + ... key id + ... attribute + ... int balance + ... ''') + +Let's create some accounts with some starting balances: + +.. sourcecode:: pycon + + >>> c.put('accounts', 'joe', {'balance': 100}) + True + >>> c.put('accounts', 'brian', {'balance': 25}) + True + +Using Transactions +------------------ + +The prototypical example of where transactions come in handy is a standard bank +debit/credit scenario. If Joe wanted to transfer currency to Brian, it would be +bad for Joe and Brian if the money were to leave Joe's account and not find its +way into Brian's. It would be bad for the bank if money were to be created out +of thin air by a deposit in Brian's account without a corresponding withdrawal +from Joe's -- manipulating markets in such a fashion is reserved for those who +are too big to fail. + +In the prototypical, "Hello World," example involving transactions, we transfer +$10 from Joe to Brian: + +.. sourcecode:: pycon + + >>> x = c.begin_transaction() + >>> brian = x.get('accounts', 'brian')['balance'] + >>> joe = x.get('accounts', 'joe')['balance'] + >>> x.put('accounts', 'brian', {'balance': brian + 10}) + True + >>> x.put('accounts', 'joe', {'balance': joe - 10}) + True + >>> x.commit() + True + >>> print "Brian's balance =", c.get('accounts', 'brian')['balance'] + Brian's balance = 35 + >>> print "Joe's balance =", c.get('accounts', 'joe')['balance'] + Joe's balance = 90 + +This transaction is relatively straight-forward. There are three critical +properties that will distinguish this transaction. One, either both of these +operations execute, or neither do. It is impossible for half of a transaction +to be lost. + +Two, there is no eventual consistency. All transaction results are immediately +visible to all future events. There is no need to reconcile or maintain +multiple versions. There is no inconsistency. + +Finally, HyperDex Warp is fault tolerant. A user-configurable fault tolerance +threshold, ``f`` allows HyperDex Warp to remain available in the presence of +concurrent failures. + +For a real bank application, we'd likely want to charge Joe an (excessive) +overdraft fee if he didn't have the money to cover the transfer. With +transactions, implementing this case is trivial: + +.. sourcecode:: pycon + + >>> x = c.begin_transaction() + >>> brian = x.get('accounts', 'brian')['balance'] + >>> joe = x.get('accounts', 'joe')['balance'] + >>> x.put('accounts', 'brian', {'balance': brian + 10}) + True + >>> if joe < 10: + ... # assess an overdraft fee on Joe + ... joe -= 35 + ... + >>> x.put('accounts', 'joe', {'balance': joe - 10}) + True + >>> x.commit() + True + >>> print "Brian's balance =", c.get('accounts', 'brian')['balance'] + Brian's balance = 45 + >>> print "Joe's balance =", c.get('accounts', 'joe')['balance'] + Joe's balance = 80 + +Here, we've successfully transferred money from Joe to Brian and cover the case +where the bank wishes to charge Joe a fee for not having enough money in the +first place. + +The astute reader will notice that the example above is very different from +doing the following: + +.. sourcecode:: pycon + + >>> # an example of broken code + >>> c.atomic_add('accounts', 'brian', {'balance': 10}) + True + >>> c.atomic_sub('accounts', 'joe', {'balance': 10}) + True + >>> print "Brian's balance =", c.get('accounts', 'brian')['balance'] + Brian's balance = 55 + >>> print "Joe's balance =", c.get('accounts', 'joe')['balance'] + Joe's balance = 70 + +This example is broken in multiple ways. First, even though each individual +operation is atomic, the two operations are not indivisible, and will be +interspersed with all other operations on the accounts. Second, the client is a +single point of failure for this transaction. Imagine if the server executing +these operations failed between the two atomic operations. No matter what the +order of operations, someone would lose money. + +Transactions are especially useful when multiple competing processes are +executing transactions concurrently. HyperDex Warp guarantees *one-copy +serializablility*. Any number of transactions may execute simultaneously. +The final state of the database will always be identical to executing the +committed transactions sequentially without concurrency. + +Let's illustrate HyperDex's behavior with concurrent transactions. +In the same terminal, let's begin a new transaction to pay Joe his yearly +5% interest payment: + +.. sourcecode:: pycon + + >>> x = c.begin_transaction() + >>> balance = x.get('accounts', 'joe')['balance'] + >>> balance = int(balance * 1.05) + >>> x.put('accounts', 'joe', {'balance': balance}) + True + +At this point, transaction ``x`` has not committed. Any modifications performed +within the context of a transaction are visible only within the transaction. No +other clients or transactions may observe the state. Verify this for yourself +with: + +.. sourcecode:: pycon + + >>> print "Joe's balance =", c.get('accounts', 'joe')['balance'] + Joe's balance = 70 + >>> print "Joe's balance =", x.get('accounts', 'joe')['balance'] + Joe's balance = 73 + +Joe's balance is still the same as before. Now, open another window and start a +second, concurrent transaction where Joe deposits cash at a branch location: + +.. sourcecode:: pycon + + >>> import hyperclient + >>> c2 = hyperclient.Client('127.0.0.1', 1982) + >>> y = c2.begin_transaction() + >>> balance = y.get('accounts', 'joe')['balance'] + >>> balance += 386 + >>> y.put('accounts', 'joe', {'balance': balance}) + True + +At this point, both ``x`` and ``y`` are ready to commit, but neither has +actually altered the account balance. In fact, there is no way for ``x`` and +``y`` to commit that doesn't violate consistency. If ``x`` commits before +``y``, the ending balance will be $466, shorting Joe of the $4 in interest he +is due. If, however, ``y`` commits before ``x``, then the ending balance of $84 +will cause Joe's cash deposit of $386 to be lost into the ether. In this +situation, one transaction must abort. If ``y`` commits first, then ``x`` will +abort: + +In the second terminal, commit ``y``: + +.. sourcecode:: pycon + + >>> y.commit() + True + >>> print "Joe's balance =", c2.get('accounts', 'joe')['balance'] + Joe's balance = 456 + +Trying to commit ``x`` in the first terminal will result in the transaction +being aborted by HyperDex Warp: + +.. sourcecode:: pycon + + >>> x.commit() + Traceback (most recent call last): + HyperClientException: HyperClient(HYPERCLIENT_ABORTED, Transaction was aborted) + +As expected, transaction ``x`` will not alter Joe's balance: + +.. sourcecode:: pycon + + >>> print "Joe's balance =", c2.get('accounts', 'joe')['balance'] + Joe's balance = 456 + +A transaction will only abort if it was executed simultaneously with another +transation that operates on the same data. The typical process for handling +aborted transactions is to repeatedly try the transaction until it succeeds: + +.. sourcecode:: pycon + + >>> committed = False + >>> while not committed: + ... try: + ... x = c.begin_transaction() + ... balance = x.get('accounts', 'joe')['balance'] + ... balance = int(balance * 1.05) + ... x.put('accounts', 'joe', {'balance': balance}) + ... committed = x.commit() + ... except hyperclient.HyperClientException as e: + ... if e.status != hyperclient.HYPERCLIENT_ABORTED: + ... raise e + ... + True + +Rich API +-------- + +HyperDex Warp Transactions expose the full set of atomic and asynchronous +operations from the HyperDex API. All operations performed under a transaction +are fully transactional. The following example shows asynchronous atomic math +operations combined with an asynchronous commit: + +.. sourcecode:: pycon + + >>> t = c.begin_transaction() + >>> d1 = t.async_atomic_add('accounts', 'brian', {'balance': 10}) + >>> d2 = t.async_atomic_sub('accounts', 'joe', {'balance': 10}) + >>> d1.wait() + True + >>> d2.wait() + True + >>> d3 = t.async_commit() + >>> d3.wait() + True + +All transactions operate on the most recent copy of the data. When a +transaction commits, the transaction is cosistently applied. HyperDex Warp +really means it when it says a transaction has committed. There are no +conflicting operations to reconcile later (after the system commits), and there +is no eventual consistency. All effects are immediate and permanent. + +Summary +------- + +HyperDex Warp provides decentralized, distributed ACID transactions. Committed +transactions are one-copy serializable, making it extremely easy to reason about +the concurrent operations. The rich API coupled with transactional guarantees +provide a rare combination of ease-of-use, correctness, and performance that +other NoSQL systems cannot provide. + +Get your copy of `HyperDex Warp`_ today. + +.. _HyperDex Warp: http://hyperdex.org/Warp + +.. todo:: + + .. sourcecode:: pycon + + >>> c.rm_space('accounts') diff --git a/doc/index.rst b/doc/index.rst index eda1c747..1bd4c3cc 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -12,6 +12,7 @@ Welcome to the HyperDex Reference Manual 04.datatypes 05.asynchronous 06.faults + 07.transactions api/python api/c glossary