Permalink
Browse files

improvements to support more type conversions for sharding purposes.

Signed-off-by: gburgett <gordon.burgett@gmail.com>
  • Loading branch information...
1 parent a397dc0 commit 237d41d8180acd3ed57b444e5dd2f72e701fd67c @gburgett committed Feb 20, 2013
@@ -18,6 +18,8 @@
import org.xflatdb.xflat.query.IntervalProvider;
import org.jdom2.xpath.XPathExpression;
import org.jdom2.xpath.XPathFactory;
+import org.xflatdb.xflat.query.XPathQuery;
+import org.xflatdb.xflat.util.XPathExpressionEqualityMatcher;
/**
* Represents the configuration for a sharded table. A sharded table is one which
@@ -34,64 +36,115 @@ private ShardsetConfig(ShardsetConfig other){
this.shardPropertyClass = other.shardPropertyClass;
this.shardPropertySelector = other.shardPropertySelector;
this.intervalProvider = other.intervalProvider;
+
}
private Class<T> shardPropertyClass;
+ /**
+ * Gets the class to which the string value of the property selected by the
+ * shard property selector must be convertible. The converted value is fed
+ * to the Interval Provider to determine the correct shard for a data row.
+ */
public Class<T> getShardPropertyClass(){
return shardPropertyClass;
}
private XPathExpression<?> shardPropertySelector;
+ /**
+ * Gets the XPath expression that selects a property of a row which determines
+ * the shard where the row will be placed.
+ */
public XPathExpression<?> getShardPropertySelector(){
return shardPropertySelector;
}
private IntervalProvider<T> intervalProvider;
+ /**
+ * Gets the interval provider used to choose which shard a row will be placed in.
+ */
public IntervalProvider<T> getIntervalProvider(){
return intervalProvider;
}
+ private boolean isId = false;
+ /**
+ * Gets a flag indicating whether the shard property selector is the {@link org.xflatdb.xflat.query.XPathQuery#Id} expression.
+ * @return
+ */
+ public boolean isShardedById(){
+ return this.isId;
+ }
+
+ /**
+ * Creates a ShardsetConfig describing a table sharded by ID. The shard selector
+ * will use the {@link XPathQuery#Id} expression.
+ * @param <U> The generic type of the ID property; all objects in the table must have the same property.
+ * @param idClass The class of the property selected by the xpath expression.
+ * @param idIntervalProvider A IntervalProvider that determines the static ranges for the
+ * property, each range will have its own file.
+ * @return A new shardset config.
+ */
+ public static <U> ShardsetConfig<U> byId(Class<U> idClass, IntervalProvider<U> idIntervalProvider){
+ if(idClass == null){
+ throw new IllegalArgumentException("idClass cannot be null");
+ }
+ if(idIntervalProvider == null){
+ throw new IllegalArgumentException("idIntervalProvider cannot be null");
+ }
+
+ ShardsetConfig<U> ret = new ShardsetConfig<>();
+ ret.shardPropertySelector = XPathQuery.Id;
+ ret.shardPropertyClass = idClass;
+ ret.intervalProvider = idIntervalProvider;
+ ret.isId = true;
+
+ return ret;
+ }
/**
* Creates a ShardsetConfig with the minimum configuration necessary for static-range sharding.
* The remainder can be left default or set as desired.
- * @param <U>
+ * @param <U> The generic type of the ID property; all objects in the table must have the same property.
* @param xpathProperty An XPath expression selecting a property of the data to shard on.
* @param propertyClass The class of the property selected by the xpath expression.
- * @param intervalProvider A RangeProvider that determines the static ranges for the
+ * @param intervalProvider A IntervalProvider that determines the static ranges for the
* property, each range will have its own file.
* @return A new shardset config.
*/
- public static <U> ShardsetConfig<U> create(String xpathProperty, Class<U> propertyClass, IntervalProvider<U> intervalProvider){
- return create(XPathFactory.instance().compile(xpathProperty), propertyClass, intervalProvider);
+ public static <U> ShardsetConfig<U> by(String xpathProperty, Class<U> propertyClass, IntervalProvider<U> intervalProvider){
+ return by(XPathFactory.instance().compile(xpathProperty), propertyClass, intervalProvider);
}
/**
* Creates a ShardsetConfig with the minimum configuration necessary for static-range sharding.
* The remainder can be left default or set as desired.
- * @param <U>
+ * @param <U> The generic type of the ID property; all objects in the table must have the same property.
* @param xpathProperty An XPath expression selecting a property of the data to shard on.
* @param propertyClass The class of the property selected by the xpath expression.
- * @param rangeProvider A RangeProvider that determines the static ranges for the
+ * @param intervalProvider An {@link IntervalProvider} that determines the static ranges for the
* property, each range will have its own file.
* @return A new shardset config.
*/
- public static <U> ShardsetConfig<U> create(XPathExpression<?> xpathProperty, Class<U> propertyClass, IntervalProvider<U> rangeProvider){
+ public static <U> ShardsetConfig<U> by(XPathExpression<?> xpathProperty, Class<U> propertyClass, IntervalProvider<U> intervalProvider){
if(xpathProperty == null){
throw new IllegalArgumentException("xpathProperty cannot be null");
}
if(propertyClass == null){
throw new IllegalArgumentException("propertyClass cannot be null");
}
- if(rangeProvider == null){
- throw new IllegalArgumentException("rangeProvider cannot be null");
+ if(intervalProvider == null){
+ throw new IllegalArgumentException("intervalProvider cannot be null");
}
ShardsetConfig<U> ret = new ShardsetConfig<>();
ret.shardPropertySelector = xpathProperty;
ret.shardPropertyClass = propertyClass;
- ret.intervalProvider = rangeProvider;
+ ret.intervalProvider = intervalProvider;
+
+ if(XPathExpressionEqualityMatcher.equals(xpathProperty, XPathQuery.Id)){
+ ret.isId = true;
+ }
return ret;
}
@@ -28,6 +28,7 @@
* <code>ConversionException</code> without detail message.
*/
public ConversionException(String msg, Throwable cause) {
+ super(msg, cause);
}
/**
@@ -0,0 +1,41 @@
+/*
+ * To change this template, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package org.xflatdb.xflat.convert.converters;
+
+import java.util.Date;
+import org.xflatdb.xflat.convert.ConversionException;
+import org.xflatdb.xflat.convert.ConversionService;
+import org.xflatdb.xflat.convert.Converter;
+
+/**
+ * Contains a number of converters to and from Date.
+ * @author Gordon
+ */
+public class DateConverters {
+
+
+ /**
+ * Registers all the date converters to the given ConversionService.
+ * @param service
+ */
+ public static void registerTo(ConversionService service){
+ service.addConverter(Date.class, Long.class, DateToLongConverter);
+ service.addConverter(Long.class, Date.class, LongToDateConverter);
+ }
+
+ public static final Converter<Date, Long> DateToLongConverter = new Converter<Date, Long>() {
+ @Override
+ public Long convert(Date source) throws ConversionException {
+ return source.getTime();
+ }
+ };
+
+ public static final Converter<Long, Date> LongToDateConverter = new Converter<Long, Date>() {
+ @Override
+ public Date convert(Long source) throws ConversionException {
+ return new Date(source);
+ }
+ };
+}
@@ -28,14 +28,7 @@
import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.stream.XMLInputFactory;
import javax.xml.stream.XMLStreamException;
-import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.xflatdb.xflat.convert.ConversionException;
-import org.xflatdb.xflat.convert.ConversionNotSupportedException;
-import org.xflatdb.xflat.convert.ConversionService;
-import org.xflatdb.xflat.convert.Converter;
-import org.xflatdb.xflat.convert.PojoConverter;
-import org.xflatdb.xflat.db.IdAccessor;
import org.jdom2.Document;
import org.jdom2.Element;
import org.jdom2.Namespace;
@@ -45,6 +38,12 @@
import org.jdom2.output.XMLOutputter;
import org.jdom2.xpath.XPathExpression;
import org.jdom2.xpath.XPathFactory;
+import org.xflatdb.xflat.convert.ConversionException;
+import org.xflatdb.xflat.convert.ConversionNotSupportedException;
+import org.xflatdb.xflat.convert.ConversionService;
+import org.xflatdb.xflat.convert.Converter;
+import org.xflatdb.xflat.convert.PojoConverter;
+import org.xflatdb.xflat.db.IdAccessor;
/**
* A PojoConverter that extends a ConversionService to convert all unknown
@@ -170,12 +169,15 @@ public boolean canConvert(Class<?> source, Class<?> target) {
}
}
catch(ConversionException ex){
+ LogFactory.getLog(getClass())
+ .trace("ConversionException encountered when attempting to map class with JAXB", ex);
return false;
}
}
@Override
public <T> T convert(Object source, Class<T> target) throws ConversionException {
+
try{
return base.convert(source, target);
}
@@ -199,7 +201,7 @@ public boolean canConvert(Class<?> source, Class<?> target) {
}
//try again now that we successfully made JAXB converters
- return convert(source, target);
+ return base.convert(source, target);
}
}
@@ -229,13 +231,6 @@ private boolean makeJaxbConverters(Class<?> target)
return false;
}
- //catch some of the most common errors
- XmlRootElement rootAnnotation = target.getAnnotation(XmlRootElement.class);
- if(rootAnnotation == null){
- this.cannotMap.add(target);
- return false;
- }
-
try {
JAXBContext context = JAXBContext.newInstance(target);
@@ -24,6 +24,7 @@
import java.util.WeakHashMap;
import java.util.logging.Level;
import java.util.logging.Logger;
+import org.apache.commons.logging.LogFactory;
import org.xflatdb.xflat.Cursor;
import org.xflatdb.xflat.DuplicateKeyException;
import org.xflatdb.xflat.KeyNotFoundException;
@@ -136,6 +137,13 @@ private T convert(Element rowData){
//cache the ID
this.idMap.put(ret, sId);
}
+ else{
+ try {
+ this.accessor.setIdValue(ret, this.getIdGenerator().stringToId(sId, this.accessor.getIdType()));
+ } catch (IllegalAccessException | InvocationTargetException ex) {
+ LogFactory.getLog(getClass()).warn("Exception setting ID value " + sId + " on type " + this.accessor.getIdType(), ex);
+ }
+ }
return ret;
}
@@ -271,9 +279,6 @@ private Element findOneElement(XPathQuery query){
ret.add(convert(e));
}
}
- catch(Exception ex){
- throw new XFlatException("Unable to close cursor", ex);
- }
return ret;
}
@@ -17,7 +17,6 @@
import java.util.HashSet;
import java.util.Iterator;
-import java.util.List;
import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
@@ -34,7 +33,6 @@
import org.xflatdb.xflat.db.EngineBase.RowData;
import org.xflatdb.xflat.transaction.Transaction;
import org.xflatdb.xflat.transaction.TransactionException;
-import org.xflatdb.xflat.transaction.TransactionManager;
import org.xflatdb.xflat.transaction.TransactionOptions;
/**
@@ -198,6 +196,20 @@ protected void setTransactionManager(EngineTransactionManager transactionManager
this.transactionManager = transactionManager;
}
+ private IdGenerator idGenerator;
+ /**
+ * Gets the ID generator which generates IDs for this table.
+ */
+ protected IdGenerator getIdGenerator(){
+ return idGenerator;
+ }
+ /**
+ * Sets the ID generator which generates IDs for this table.
+ */
+ protected void setIdGenerator(IdGenerator idGenerator){
+ this.idGenerator = idGenerator;
+ }
+
//</editor-fold>
@@ -110,7 +110,18 @@ protected ShardedEngineBase(File file, String tableName, ShardsetConfig<T> confi
T converted;
if(value == null || !this.config.getShardPropertyClass().isAssignableFrom(value.getClass())){
try {
- converted = this.getConversionService().convert(value, this.config.getShardPropertyClass());
+ if(this.config.isShardedById() && value != null){
+ //if its sharded by ID, then getInterval ought to be selecting the ID attribute of the row,
+ //which ought to be convertible to string and then run through the ID generator's conversion.
+ String idVal = value instanceof String ?
+ (String)value :
+ this.getConversionService().convert(value, String.class);
+
+ converted = (T)this.getIdGenerator().stringToId(idVal, this.config.getShardPropertyClass());
+ }
+ else{
+ converted = this.getConversionService().convert(value, this.config.getShardPropertyClass());
+ }
} catch (ConversionException ex) {
throw new XFlatException("Data cannot be sharded: sharding expression " + config.getShardPropertySelector().getExpression() +
" selected non-convertible value " + value, ex);
@@ -180,6 +180,7 @@ private EngineBase makeNewEngine(File file){
ret.setConversionService(db.getConversionService());
ret.setExecutorService(db.getExecutorService());
ret.setTransactionManager(db.getEngineTransactionManager());
+ ret.setIdGenerator(this.idGenerator);
if(ret instanceof ShardedEngineBase){
//give it a metadata factory centered in its own file. If it uses this,
@@ -43,6 +43,7 @@
import org.xflatdb.xflat.convert.ConversionService;
import org.xflatdb.xflat.convert.DefaultConversionService;
import org.xflatdb.xflat.convert.PojoConverter;
+import org.xflatdb.xflat.convert.converters.DateConverters;
import org.xflatdb.xflat.convert.converters.JDOMConverters;
import org.xflatdb.xflat.convert.converters.StringConverters;
import org.xflatdb.xflat.engine.DefaultEngineFactory;
@@ -191,6 +192,7 @@ public XFlatDatabase(File directory, ScheduledExecutorService executorService){
this.conversionService = new DefaultConversionService();
StringConverters.registerTo(conversionService);
JDOMConverters.registerTo(conversionService);
+ DateConverters.registerTo(conversionService);
this.executorService = executorService;
Oops, something went wrong.

0 comments on commit 237d41d

Please sign in to comment.